Merge pull request #313 from Lightricks/pr-2026-01-07
Update ComfyUI-LTXVideo - 2026-01-07
This commit is contained in:
Michael Kupchick
@@ -64,17 +64,17 @@ Download the following models:
|
||||
* [`ltx-2-19b-dev.safetensors`](https://huggingface.co/Lightricks/LTX-2/blob/main/ltx-2-19b-dev.safetensors)
|
||||
* [`ltx-2-19b-distilled.safetensors`](https://huggingface.co/Lightricks/LTX-2/blob/main/ltx-2-19b-distilled.safetensors)
|
||||
|
||||
**Spatial Upsampler** - Required for current two-stage pipeline implementations in this repository
|
||||
* [`ltx-2-spatial-upsampler-x2-1.0.safetensors`](https://huggingface.co/Lightricks/LTX-2/blob/main/ltx-2-spatial-upsampler-x2-1.0.safetensors)
|
||||
**Spatial Upscaler** - Required for current two-stage pipeline implementations in this repository
|
||||
* [`ltx-2-spatial-upscaler-x2-1.0.safetensors`](https://huggingface.co/Lightricks/LTX-2/blob/main/ltx-2-spatial-upscaler-x2-1.0.safetensors)
|
||||
|
||||
**Temporal Upsampler** - Required for current two-stage pipeline implementations in this repository
|
||||
* [`ltx-2-temporal-upsampler-x2-1.0.safetensors`](https://huggingface.co/Lightricks/LTX-2/blob/main/ltx-2-temporal-upsampler-x2-1.0.safetensors)
|
||||
**Temporal Upscaler** - Required for current two-stage pipeline implementations in this repository
|
||||
* [`ltx-2-temporal-upscaler-x2-1.0.safetensors`](https://huggingface.co/Lightricks/LTX-2/blob/main/ltx-2-temporal-upscaler-x2-1.0.safetensors)
|
||||
|
||||
**Distilled LoRA** - Required for current two-stage pipeline implementations in this repository (except DistilledPipeline and ICLoraPipeline)
|
||||
* [`ltx-2-19b-distilled-lora-384.safetensors`](https://huggingface.co/Lightricks/LTX-2/blob/main/ltx-2-19b-distilled-lora-384.safetensors)
|
||||
|
||||
**Gemma Text Encoder** (download all assets from the repository)
|
||||
* [`Gemma 3`](https://huggingface.co/google/gemma-3-12b-it-qat-q4_0-unquantized/)
|
||||
* [`Gemma 3`](https://huggingface.co/google/gemma-3-12b-it-qat-q4_0-unquantized)
|
||||
|
||||
**LoRAs**
|
||||
* [`ltx-2-19b-ic-lora-canny-control.safetensors`](https://huggingface.co/Lightricks/LTX-2-19b-IC-LoRA-Canny-Control/blob/main/ltx-2-19b-ic-lora-canny-control.safetensors)
|
||||
|
||||
@@ -7,7 +7,6 @@ from .easy_samplers import (
|
||||
LTXVInContextSampler,
|
||||
)
|
||||
from .gemma_encoder import LTXVGemmaCLIPModelLoader, LTXVGemmaEnhancePrompt
|
||||
from .gemma_encoder_mgpu_loader import LTXVGemmaCLIPModelLoaderMGPU
|
||||
from .guide import LTXVAddGuideAdvanced
|
||||
from .guiders import GuiderParametersNode, MultimodalGuiderNode
|
||||
from .latent_norm import (
|
||||
@@ -83,7 +82,6 @@ NODE_CLASS_MAPPINGS = {
|
||||
"LTXVStatNormLatent": LTXVStatNormLatent,
|
||||
"LTXVPerStepStatNormPatcher": LTXVPerStepStatNormPatcher,
|
||||
"LTXVGemmaCLIPModelLoader": LTXVGemmaCLIPModelLoader,
|
||||
"LTXVGemmaCLIPModelLoaderMGPU": LTXVGemmaCLIPModelLoaderMGPU,
|
||||
"LTXVGemmaEnhancePrompt": LTXVGemmaEnhancePrompt,
|
||||
"DynamicConditioning": DynamicConditioning,
|
||||
"LowVRAMCheckpointLoader": LowVRAMCheckpointLoader,
|
||||
|
||||
@@ -284,7 +284,8 @@ class Embeddings1DConnector(nn.Module):
|
||||
)
|
||||
learnable_registers = torch.tile(
|
||||
self.learnable_registers, (num_registers_duplications, 1)
|
||||
)
|
||||
).to(hidden_states.device)
|
||||
|
||||
attention_mask_binary = (
|
||||
attention_mask.squeeze(1).squeeze(1).unsqueeze(-1) >= -9000.0
|
||||
).int()
|
||||
|
||||
@@ -283,44 +283,6 @@
|
||||
"ltx-2-19b-distilled.safetensors"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5178,
|
||||
"type": "LTXVGemmaCLIPModelLoader",
|
||||
"pos": [
|
||||
-3158.5490398695288,
|
||||
7158.821506282883
|
||||
],
|
||||
"size": [
|
||||
421.93656760943577,
|
||||
106
|
||||
],
|
||||
"flags": {},
|
||||
"order": 4,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "clip",
|
||||
"type": "CLIP",
|
||||
"links": [
|
||||
13924,
|
||||
13926
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LTXVGemmaCLIPModelLoader",
|
||||
"widget_ue_connectable": {
|
||||
"model_path": true,
|
||||
"max_length": true
|
||||
}
|
||||
},
|
||||
"widgets_values": [
|
||||
"gemma-3-12b-it-qat-q4_0-unquantized/model-00001-of-00005.safetensors",
|
||||
"ltx-2-19b-distilled.safetensors",
|
||||
1024
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5188,
|
||||
"type": "LTXVAudioVAELoader",
|
||||
@@ -333,7 +295,7 @@
|
||||
58
|
||||
],
|
||||
"flags": {},
|
||||
"order": 5,
|
||||
"order": 4,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
@@ -561,7 +523,7 @@
|
||||
58
|
||||
],
|
||||
"flags": {},
|
||||
"order": 6,
|
||||
"order": 5,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
@@ -582,58 +544,6 @@
|
||||
"ltx-2-spatial-upscaler-x2-1.0.safetensors"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5212,
|
||||
"type": "MarkdownNote",
|
||||
"pos": [
|
||||
-3624.6205106722414,
|
||||
6715.620851300959
|
||||
],
|
||||
"size": [
|
||||
421.1303542521987,
|
||||
436.3332627074051
|
||||
],
|
||||
"flags": {
|
||||
"collapsed": false
|
||||
},
|
||||
"order": 7,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
"title": "Model Links",
|
||||
"properties": {},
|
||||
"widgets_values": [
|
||||
"## Model Links\n\n**LTX-2 Model Weights**\n\n- [ltx-2-19b-distilled.safetensors](https://huggingface.co/Lightricks/LTX-2/resolve/main/ltx-2-19b-distilled.safetensors)\n- [ ltx-2-spatial-upscaler-x2-1.0.safetensors ](https://huggingface.co/Lightricks/LTX-2/tree/main/ltx-2-spatial-upscaler-x2-1.0.safetensors)\n\n**Text Encoder**\n- [Google Gemma 3](https://huggingface.co/google/gemma-3-12b-pt)\n\n**Model Storage Location**\n\n```\n📂 ComfyUI/\n├── 📂 models/\n│ ├── 📂 text_encoders/\n│ │ ├── comfy_gemma_3_12B_it.safetensors\n│ ├── 📂 checkpoints/\n│ │ └── ltx-2-19b-distilled.safetensors\n│ ├── 📂 latent_upscale_models/ \n └── ltx-2-spatial-upscaler-x2-1.0.safetensors\n```\n\n## Report Issues\nTo report any issues when running this workflow, [go to GitHub](https://github.com/Lightricks/ComfyUI-LTXVideo/issues)\n"
|
||||
],
|
||||
"color": "#432",
|
||||
"bgcolor": "#653"
|
||||
},
|
||||
{
|
||||
"id": 5211,
|
||||
"type": "MarkdownNote",
|
||||
"pos": [
|
||||
-3623.5618947453854,
|
||||
7193.231548656596
|
||||
],
|
||||
"size": [
|
||||
420.9178919734818,
|
||||
209.33158232705227
|
||||
],
|
||||
"flags": {
|
||||
"collapsed": false
|
||||
},
|
||||
"order": 8,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
"title": "Prompting LTX-2",
|
||||
"properties": {},
|
||||
"widgets_values": [
|
||||
"## LTX-2 Prompting Tips\n1. **Core Actions**: Describe events and actions as they occur over time \n2. **Audio**: Describe sounds and dialogue needed for the scene \n3. **Reference Image**: Do not repeat details already present \n4. **Consistency**: Avoid instructions that do not match the reference image, as this will degrade results"
|
||||
],
|
||||
"color": "#432",
|
||||
"bgcolor": "#653"
|
||||
},
|
||||
{
|
||||
"id": 5213,
|
||||
"type": "MarkdownNote",
|
||||
@@ -648,7 +558,7 @@
|
||||
"flags": {
|
||||
"collapsed": false
|
||||
},
|
||||
"order": 9,
|
||||
"order": 6,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
@@ -674,7 +584,7 @@
|
||||
"flags": {
|
||||
"collapsed": false
|
||||
},
|
||||
"order": 10,
|
||||
"order": 7,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
@@ -745,6 +655,120 @@
|
||||
"randomize"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5178,
|
||||
"type": "LTXVGemmaCLIPModelLoader",
|
||||
"pos": [
|
||||
-3158.5490398695288,
|
||||
7158.821506282883
|
||||
],
|
||||
"size": [
|
||||
421.93656760943577,
|
||||
106
|
||||
],
|
||||
"flags": {},
|
||||
"order": 8,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "clip",
|
||||
"type": "CLIP",
|
||||
"links": [
|
||||
13924,
|
||||
13926
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LTXVGemmaCLIPModelLoader",
|
||||
"widget_ue_connectable": {
|
||||
"model_path": true,
|
||||
"max_length": true
|
||||
}
|
||||
},
|
||||
"widgets_values": [
|
||||
"gemma-3-12b-it-qat-q4_0-unquantized/model-00001-of-00005.safetensors",
|
||||
"ltx-2-19b-distilled.safetensors",
|
||||
1024
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5175,
|
||||
"type": "PrimitiveStringMultiline",
|
||||
"pos": [
|
||||
-1990.592849503922,
|
||||
6831.482572842548
|
||||
],
|
||||
"size": [
|
||||
281.93625497178095,
|
||||
424.68799816112823
|
||||
],
|
||||
"flags": {},
|
||||
"order": 9,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "STRING",
|
||||
"type": "STRING",
|
||||
"links": [
|
||||
13928
|
||||
]
|
||||
}
|
||||
],
|
||||
"title": "Positive Prompt",
|
||||
"properties": {
|
||||
"cnr_id": "comfy-core",
|
||||
"ver": "0.5.1",
|
||||
"Node name for S&R": "PrimitiveStringMultiline"
|
||||
},
|
||||
"widgets_values": [
|
||||
"a 3d animated movie scene shows the wise elderly owl on the left saying to a baby owl to his right, with a raspy deep voice: \"focus, close your eyes, and flap!\". we then see the little owl jump off the branch screaming in fear and disappear. the the elder owl remains standing on the branch with a disappointed face. the camera remains static in place"
|
||||
],
|
||||
"color": "#232",
|
||||
"bgcolor": "#353"
|
||||
},
|
||||
{
|
||||
"id": 5180,
|
||||
"type": "LoadImage",
|
||||
"pos": [
|
||||
-2403.4019881614336,
|
||||
6835.8261958245885
|
||||
],
|
||||
"size": [
|
||||
395.509854028056,
|
||||
433.14022877831576
|
||||
],
|
||||
"flags": {},
|
||||
"order": 10,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "IMAGE",
|
||||
"type": "IMAGE",
|
||||
"links": [
|
||||
13927,
|
||||
13940
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "MASK",
|
||||
"type": "MASK",
|
||||
"links": []
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"cnr_id": "comfy-core",
|
||||
"ver": "0.3.56",
|
||||
"Node name for S&R": "LoadImage"
|
||||
},
|
||||
"widgets_values": [
|
||||
"distilled image.png",
|
||||
"image"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5189,
|
||||
"type": "3eaa20c4-5842-4fe4-87df-c0a7e83a6a78",
|
||||
@@ -899,80 +923,56 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5175,
|
||||
"type": "PrimitiveStringMultiline",
|
||||
"id": 5211,
|
||||
"type": "MarkdownNote",
|
||||
"pos": [
|
||||
-1990.592849503922,
|
||||
6831.482572842548
|
||||
-3620.0939157608577,
|
||||
7287.881055902407
|
||||
],
|
||||
"size": [
|
||||
281.93625497178095,
|
||||
424.68799816112823
|
||||
420.9178919734818,
|
||||
209.33158232705227
|
||||
],
|
||||
"flags": {},
|
||||
"flags": {
|
||||
"collapsed": false
|
||||
},
|
||||
"order": 11,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "STRING",
|
||||
"type": "STRING",
|
||||
"links": [
|
||||
13928
|
||||
]
|
||||
}
|
||||
],
|
||||
"title": "Positive Prompt",
|
||||
"properties": {
|
||||
"cnr_id": "comfy-core",
|
||||
"ver": "0.5.1",
|
||||
"Node name for S&R": "PrimitiveStringMultiline"
|
||||
},
|
||||
"outputs": [],
|
||||
"title": "Prompting LTX-2",
|
||||
"properties": {},
|
||||
"widgets_values": [
|
||||
"a 3d animated movie scene shows the wise elderly owl on the left saying to a baby owl to his right, with a raspy deep voice: \"focus, close your eyes, and flap!\". we then see the little owl jump off the branch screaming in fear and disappear. the the elder owl remains standing on the branch with a disappointed face. the camera remains static in place"
|
||||
"## LTX-2 Prompting Tips\n1. **Core Actions**: Describe events and actions as they occur over time \n2. **Audio**: Describe sounds and dialogue needed for the scene \n3. **Reference Image**: Do not repeat details already present \n4. **Consistency**: Avoid instructions that do not match the reference image, as this will degrade results"
|
||||
],
|
||||
"color": "#232",
|
||||
"bgcolor": "#353"
|
||||
"color": "#432",
|
||||
"bgcolor": "#653"
|
||||
},
|
||||
{
|
||||
"id": 5180,
|
||||
"type": "LoadImage",
|
||||
"id": 5212,
|
||||
"type": "MarkdownNote",
|
||||
"pos": [
|
||||
-2403.4019881614336,
|
||||
6835.8261958245885
|
||||
-3627.789651297241,
|
||||
6716.012179425959
|
||||
],
|
||||
"size": [
|
||||
395.509854028056,
|
||||
433.14022877831576
|
||||
427.8617596510994,
|
||||
529.3430202216196
|
||||
],
|
||||
"flags": {},
|
||||
"flags": {
|
||||
"collapsed": false
|
||||
},
|
||||
"order": 12,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "IMAGE",
|
||||
"type": "IMAGE",
|
||||
"links": [
|
||||
13927,
|
||||
13940
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "MASK",
|
||||
"type": "MASK",
|
||||
"links": []
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"cnr_id": "comfy-core",
|
||||
"ver": "0.3.56",
|
||||
"Node name for S&R": "LoadImage"
|
||||
},
|
||||
"outputs": [],
|
||||
"title": "Model Links",
|
||||
"properties": {},
|
||||
"widgets_values": [
|
||||
"distilled image.png",
|
||||
"image"
|
||||
]
|
||||
"## Model Links\n\n**LTX-2 Model Weights**\n\n- [ltx-2-19b-distilled.safetensors](https://huggingface.co/Lightricks/LTX-2/resolve/main/ltx-2-19b-distilled.safetensors)\n- [ ltx-2-spatial-upscaler-x2-1.0.safetensors ](https://huggingface.co/Lightricks/LTX-2/resolve/main/ltx-2-spatial-upscaler-x2-1.0.safetensors)\n\n**Text Encoder**\n- [Google Gemma 3](https://huggingface.co/google/gemma-3-12b-it-qat-q4_0-unquantized)\n\nPlease download the entire folder - \n\n- Run: \n 1. cd models/text_encoders\n 2. git clone https://huggingface.co/google/gemma-3-12b-it-qat-q4_0-unquantized\n\n*Full documentation can be found [here](https://docs.ltx.video/open-source-model/integration-tools/comfy-ui#text-encoder)\n\n\n**Model Storage Location**\n\n```\n📂 ComfyUI/\n├── 📂 models/\n│ ├── 📂 text_encoders/\n│ │ ├── gemma-3-12b-it-qat-q4_0-unquantized/model-00001-of-00005.safetensors\n│ ├── 📂 checkpoints/\n│ │ └── ltx-2-19b-distilled.safetensors\n│ ├── 📂 latent_upscale_models/ \n └── ltx-2-spatial-upscaler-x2-1.0.safetensors\n```\n**Assets**\n- [distilled image](https://github.com/Lightricks/ComfyUI-LTXVideo/blob/master/example_workflows/assets/distilled%20image.png)\n\n## Report Issues\nTo report any issues when running this workflow, [go to GitHub](https://github.com/Lightricks/ComfyUI-LTXVideo/issues)\n"
|
||||
],
|
||||
"color": "#432",
|
||||
"bgcolor": "#653"
|
||||
}
|
||||
],
|
||||
"links": [
|
||||
@@ -1274,9 +1274,9 @@
|
||||
"flags": {},
|
||||
"nodes": [
|
||||
5176,
|
||||
5178,
|
||||
5188,
|
||||
5210
|
||||
5210,
|
||||
5178
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -2100,58 +2100,6 @@
|
||||
1
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5239,
|
||||
"type": "ImpactExecutionOrderController",
|
||||
"pos": [
|
||||
-2153.013232660581,
|
||||
6935.750091838874
|
||||
],
|
||||
"size": [
|
||||
254.63517702162235,
|
||||
47.506006012050875
|
||||
],
|
||||
"flags": {},
|
||||
"order": 19,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"localized_name": "signal",
|
||||
"name": "signal",
|
||||
"type": "*",
|
||||
"link": 13982
|
||||
},
|
||||
{
|
||||
"localized_name": "value",
|
||||
"name": "value",
|
||||
"type": "*",
|
||||
"link": 13983
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"localized_name": "signal",
|
||||
"name": "signal",
|
||||
"type": "*",
|
||||
"links": [
|
||||
13986,
|
||||
13987
|
||||
]
|
||||
},
|
||||
{
|
||||
"localized_name": "value",
|
||||
"name": "value",
|
||||
"type": "*",
|
||||
"links": [
|
||||
14000
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "ImpactExecutionOrderController"
|
||||
},
|
||||
"widgets_values": []
|
||||
},
|
||||
{
|
||||
"id": 5069,
|
||||
"type": "GetImageSize",
|
||||
@@ -2354,53 +2302,6 @@
|
||||
"auto"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5111,
|
||||
"type": "RandomNoise",
|
||||
"pos": [
|
||||
-746.4419600673787,
|
||||
7087.4135610931135
|
||||
],
|
||||
"size": [
|
||||
270,
|
||||
82
|
||||
],
|
||||
"flags": {
|
||||
"collapsed": true
|
||||
},
|
||||
"order": 14,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"localized_name": "noise_seed",
|
||||
"name": "noise_seed",
|
||||
"type": "INT",
|
||||
"widget": {
|
||||
"name": "noise_seed"
|
||||
},
|
||||
"link": 13979
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"localized_name": "NOISE",
|
||||
"name": "NOISE",
|
||||
"type": "NOISE",
|
||||
"links": [
|
||||
13783
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"cnr_id": "comfy-core",
|
||||
"ver": "0.3.71",
|
||||
"Node name for S&R": "RandomNoise"
|
||||
},
|
||||
"widgets_values": [
|
||||
42,
|
||||
"fixed"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5138,
|
||||
"type": "EmptyLTXVLatentVideo",
|
||||
@@ -2807,6 +2708,105 @@
|
||||
0.6,
|
||||
false
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5239,
|
||||
"type": "ImpactExecutionOrderController",
|
||||
"pos": [
|
||||
-2153.013232660581,
|
||||
6935.750091838874
|
||||
],
|
||||
"size": [
|
||||
254.63517702162235,
|
||||
47.506006012050875
|
||||
],
|
||||
"flags": {},
|
||||
"order": 19,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"localized_name": "signal",
|
||||
"name": "signal",
|
||||
"type": "*",
|
||||
"link": 13982
|
||||
},
|
||||
{
|
||||
"localized_name": "value",
|
||||
"name": "value",
|
||||
"type": "*",
|
||||
"link": 13983
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"localized_name": "signal",
|
||||
"name": "signal",
|
||||
"type": "*",
|
||||
"links": [
|
||||
13986,
|
||||
13987
|
||||
]
|
||||
},
|
||||
{
|
||||
"localized_name": "value",
|
||||
"name": "value",
|
||||
"type": "*",
|
||||
"links": [
|
||||
14000
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "ImpactExecutionOrderController"
|
||||
},
|
||||
"widgets_values": []
|
||||
},
|
||||
{
|
||||
"id": 5111,
|
||||
"type": "RandomNoise",
|
||||
"pos": [
|
||||
-746.4419600673787,
|
||||
7087.4135610931135
|
||||
],
|
||||
"size": [
|
||||
270,
|
||||
82
|
||||
],
|
||||
"flags": {
|
||||
"collapsed": true
|
||||
},
|
||||
"order": 14,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"localized_name": "noise_seed",
|
||||
"name": "noise_seed",
|
||||
"type": "INT",
|
||||
"widget": {
|
||||
"name": "noise_seed"
|
||||
},
|
||||
"link": 13979
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"localized_name": "NOISE",
|
||||
"name": "NOISE",
|
||||
"type": "NOISE",
|
||||
"links": [
|
||||
13783
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"cnr_id": "comfy-core",
|
||||
"ver": "0.3.71",
|
||||
"Node name for S&R": "RandomNoise"
|
||||
},
|
||||
"widgets_values": [
|
||||
42,
|
||||
"fixed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"groups": [
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -251,44 +251,6 @@
|
||||
"ltx-2-19b-distilled.safetensors"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5227,
|
||||
"type": "LTXVGemmaCLIPModelLoader",
|
||||
"pos": [
|
||||
-3298.5249619455894,
|
||||
7907.801436963024
|
||||
],
|
||||
"size": [
|
||||
421.93656760943577,
|
||||
106
|
||||
],
|
||||
"flags": {},
|
||||
"order": 4,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "clip",
|
||||
"type": "CLIP",
|
||||
"links": [
|
||||
13961,
|
||||
13962
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LTXVGemmaCLIPModelLoader",
|
||||
"widget_ue_connectable": {
|
||||
"model_path": true,
|
||||
"max_length": true
|
||||
}
|
||||
},
|
||||
"widgets_values": [
|
||||
"gemma-3-12b-it-qat-q4_0-unquantized/model-00001-of-00005.safetensors",
|
||||
"ltx-2-19b-distilled.safetensors",
|
||||
1024
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5219,
|
||||
"type": "LTXVAudioVAELoader",
|
||||
@@ -301,7 +263,7 @@
|
||||
58
|
||||
],
|
||||
"flags": {},
|
||||
"order": 5,
|
||||
"order": 4,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
@@ -551,42 +513,6 @@
|
||||
"auto"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5222,
|
||||
"type": "PrimitiveStringMultiline",
|
||||
"pos": [
|
||||
-2567.508915639494,
|
||||
7588.770792256889
|
||||
],
|
||||
"size": [
|
||||
337.7200214704494,
|
||||
515.3418506718763
|
||||
],
|
||||
"flags": {},
|
||||
"order": 6,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "STRING",
|
||||
"type": "STRING",
|
||||
"links": [
|
||||
13980
|
||||
]
|
||||
}
|
||||
],
|
||||
"title": "Positive Prompt",
|
||||
"properties": {
|
||||
"cnr_id": "comfy-core",
|
||||
"ver": "0.5.1",
|
||||
"Node name for S&R": "PrimitiveStringMultiline"
|
||||
},
|
||||
"widgets_values": [
|
||||
"A medium shot captures a doctor, clad in a white lab coat over a blue shirt, with a stethoscope draped around his neck, as he gently palpates the right knee of a patient. The patient, wearing a dark blue garment, sits on a light blue examination table, her left hand resting on her left thigh, and her right hand resting on her right thigh, just above the knee. A subtle red inflamed area is visible on the patient's right knee. The background is softly blurred, showing medical supplies on a counter. The camera slowly tilts up, revealing the doctor's face as he carefully flexes the patient's knee, testing its range of movement. The patient's expression shifts, her brow furrowing slightly, and she winces in discomfort. As the doctor continues to assess the knee, a nurse, dressed in scrubs and holding a notepad, enters the frame from the left, walking past the doctor and patient. She briefly pauses, extending the notepad towards the doctor, who glances at the notes and gives a subtle nod of acknowledgment before returning his attention to the patient's knee. The nurse continues walking out of frame to the right, and the doctor resumes his examination, his hands continuing to gently manipulate the patient's knee, who continues to wince.\n"
|
||||
],
|
||||
"color": "#232",
|
||||
"bgcolor": "#353"
|
||||
},
|
||||
{
|
||||
"id": 5244,
|
||||
"type": "LatentUpscaleModelLoader",
|
||||
@@ -599,7 +525,7 @@
|
||||
100.68012732230727
|
||||
],
|
||||
"flags": {},
|
||||
"order": 7,
|
||||
"order": 5,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
@@ -620,58 +546,6 @@
|
||||
"ltx-2-spatial-upscaler-x2-1.0.safetensors"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5246,
|
||||
"type": "MarkdownNote",
|
||||
"pos": [
|
||||
-3759.587881115107,
|
||||
7906.926955158114
|
||||
],
|
||||
"size": [
|
||||
418.0950915480007,
|
||||
208.54998853542747
|
||||
],
|
||||
"flags": {
|
||||
"collapsed": false
|
||||
},
|
||||
"order": 8,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
"title": "Prompting LTX-2",
|
||||
"properties": {},
|
||||
"widgets_values": [
|
||||
"## LTX-2 Prompting Tips\n1. **Core Actions**: Describe events and actions as they occur over time \n2. **Audio**: Describe sounds and dialogue needed for the scene \n3. **Consistency**: Avoid instructions that do not match the reference image, as this will degrade results\n"
|
||||
],
|
||||
"color": "#432",
|
||||
"bgcolor": "#653"
|
||||
},
|
||||
{
|
||||
"id": 5245,
|
||||
"type": "MarkdownNote",
|
||||
"pos": [
|
||||
-3759.592503870824,
|
||||
7426.935420507221
|
||||
],
|
||||
"size": [
|
||||
421.1303542521987,
|
||||
436.3332627074051
|
||||
],
|
||||
"flags": {
|
||||
"collapsed": false
|
||||
},
|
||||
"order": 9,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
"title": "Model Links",
|
||||
"properties": {},
|
||||
"widgets_values": [
|
||||
"## Model Links\n\n**LTX-2 Model Weights**\n\n- [ltx-2-19b-distilled.safetensors](https://huggingface.co/Lightricks/LTX-2/resolve/main/ltx-2-19b-distilled.safetensors)\n- [ ltx-2-spatial-upscaler-x2-1.0.safetensors ](https://huggingface.co/Lightricks/LTX-2/tree/main/ltx-2-spatial-upscaler-x2-1.0.safetensors)\n\n**Text Encoder**\n- [Google Gemma 3](https://huggingface.co/google/gemma-3-12b-pt)\n\n**Model Storage Location**\n\n```\n📂 ComfyUI/\n├── 📂 models/\n│ ├── 📂 text_encoders/\n│ │ ├── comfy_gemma_3_12B_it.safetensors\n│ ├── 📂 checkpoints/\n│ │ └── ltx-2-19b-distilled.safetensors\n│ ├── 📂 latent_upscale_models/ \n └── ltx-2-spatial-upscaler-x2-1.0.safetensors\n```\n\n## Report Issues\nTo report any issues when running this workflow, [go to GitHub](https://github.com/Lightricks/ComfyUI-LTXVideo/issues)\n"
|
||||
],
|
||||
"color": "#432",
|
||||
"bgcolor": "#653"
|
||||
},
|
||||
{
|
||||
"id": 5247,
|
||||
"type": "MarkdownNote",
|
||||
@@ -686,7 +560,7 @@
|
||||
"flags": {
|
||||
"collapsed": false
|
||||
},
|
||||
"order": 10,
|
||||
"order": 6,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
@@ -698,32 +572,6 @@
|
||||
"color": "#432",
|
||||
"bgcolor": "#653"
|
||||
},
|
||||
{
|
||||
"id": 5248,
|
||||
"type": "MarkdownNote",
|
||||
"pos": [
|
||||
-2581.888874206601,
|
||||
7104.237854538457
|
||||
],
|
||||
"size": [
|
||||
469.5872194653941,
|
||||
209.10385276580928
|
||||
],
|
||||
"flags": {
|
||||
"collapsed": false
|
||||
},
|
||||
"order": 11,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
"title": "LoRA",
|
||||
"properties": {},
|
||||
"widgets_values": [
|
||||
"# Using LoRAs\n**Ctrl + B to enable**\n\nFor the camera LoRA we recommend setting strength = 1\n\nYou can use LTX [Ready to use camera LoRAs](https://huggingface.co/collections/Lightricks/ltx-2)\n\n\n\n\nUse a second LoRa only when the details are important in the LoRa. \n\n## Training\nWe have made available a full trainer to help you to train your own LoRA. You can [find it here](https://github.com/Lightricks/LTX-2)."
|
||||
],
|
||||
"color": "#432",
|
||||
"bgcolor": "#653"
|
||||
},
|
||||
{
|
||||
"id": 5225,
|
||||
"type": "LTXVGemmaEnhancePrompt",
|
||||
@@ -778,9 +626,111 @@
|
||||
"",
|
||||
"You are a Creative Assistant. Given a user's raw input prompt describing a scene or concept, expand it into a detailed video generation prompt with specific visuals and integrated audio to guide a text-to-video model.\n\n#### Guidelines\n- Strictly follow all aspects of the user's raw input: include every element requested (style, visuals, motions, actions, camera movement, audio).\n - If the input is vague, invent concrete details: lighting, textures, materials, scene settings, etc.\n - For characters: describe gender, clothing, hair, expressions. DO NOT invent unrequested characters.\n- Use active language: present-progressive verbs (\"is walking,\" \"speaking\"). If no action specified, describe natural movements.\n- Maintain chronological flow: use temporal connectors (\"as,\" \"then,\" \"while\").\n- Audio layer: Describe complete soundscape (background audio, ambient sounds, SFX, speech/music when requested). Integrate sounds chronologically alongside actions. Be specific (e.g., \"soft footsteps on tile\"), not vague (e.g., \"ambient sound is present\").\n- Speech (only when requested): \n - For ANY speech-related input (talking, conversation, singing, etc.), ALWAYS include exact words in quotes with voice characteristics (e.g., \"The man says in an excited voice: 'You won't believe what I just saw!'\").\n - Specify language if not English and accent if relevant.\n- Style: Include visual style at the beginning: \"Style: <style>, <rest of prompt>.\" Default to cinematic-realistic if unspecified. Omit if unclear.\n- Visual and audio only: NO non-visual/auditory senses (smell, taste, touch).\n- Restrained language: Avoid dramatic/exaggerated terms. Use mild, natural phrasing.\n - Colors: Use plain terms (\"red dress\"), not intensified (\"vibrant blue,\" \"bright red\").\n - Lighting: Use neutral descriptions (\"soft overhead light\"), not harsh (\"blinding light\").\n - Facial features: Use delicate modifiers for subtle features (i.e., \"subtle freckles\").\n\n#### Important notes: \n- Analyze the user's raw input carefully. In cases of FPV or POV, exclude the description of the subject whose POV is requested.\n- Camera motion: DO NOT invent camera motion unless requested by the user.\n- Speech: DO NOT modify user-provided character dialogue unless it's a typo.\n- No timestamps or cuts: DO NOT use timestamps or describe scene cuts unless explicitly requested.\n- Format: DO NOT use phrases like \"The scene opens with...\". Start directly with Style (optional) and chronological scene description.\n- Format: DO NOT start your response with special characters.\n- DO NOT invent dialogue unless the user mentions speech/talking/singing/conversation.\n- If the user's raw input prompt is highly detailed, chronological and in the requested format: DO NOT make major edits or introduce new elements. Add/enhance audio descriptions if missing.\n\n#### Output Format (Strict):\n- Single continuous paragraph in natural language (English).\n- NO titles, headings, prefaces, code fences, or Markdown.\n- If unsafe/invalid, return original user prompt. Never ask questions or clarifications.\n\nYour output quality is CRITICAL. Generate visually rich, dynamic prompts with integrated audio for high-quality video generation.\n\n#### Example\nInput: \"A woman at a coffee shop talking on the phone\"\nOutput:\nStyle: realistic with cinematic lighting. In a medium close-up, a woman in her early 30s with shoulder-length brown hair sits at a small wooden table by the window. She wears a cream-colored turtleneck sweater, holding a white ceramic coffee cup in one hand and a smartphone to her ear with the other. Ambient cafe sounds fill the space—espresso machine hiss, quiet conversations, gentle clinking of cups. The woman listens intently, nodding slightly, then takes a sip of her coffee and sets it down with a soft clink. Her face brightens into a warm smile as she speaks in a clear, friendly voice, 'That sounds perfect! I'd love to meet up this weekend. How about Saturday afternoon?' She laughs softly—a genuine chuckle—and shifts in her chair. Behind her, other patrons move subtly in and out of focus. 'Great, I'll see you then,' she concludes cheerfully, lowering the phone.",
|
||||
512,
|
||||
true
|
||||
true,
|
||||
42,
|
||||
"randomize"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5222,
|
||||
"type": "PrimitiveStringMultiline",
|
||||
"pos": [
|
||||
-2567.508915639494,
|
||||
7588.770792256889
|
||||
],
|
||||
"size": [
|
||||
337.7200214704494,
|
||||
515.3418506718763
|
||||
],
|
||||
"flags": {},
|
||||
"order": 7,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "STRING",
|
||||
"type": "STRING",
|
||||
"links": [
|
||||
13980
|
||||
]
|
||||
}
|
||||
],
|
||||
"title": "Positive Prompt",
|
||||
"properties": {
|
||||
"cnr_id": "comfy-core",
|
||||
"ver": "0.5.1",
|
||||
"Node name for S&R": "PrimitiveStringMultiline"
|
||||
},
|
||||
"widgets_values": [
|
||||
"cinematic epic war movie scene shows a medium closeup shot of a fierce muscular Viking warrior standing in place, clad in weathered leather armor and gripping a battle-worn axe, his long, tangled hair billowing against the stormy sky. The scene is set under a foreboding, cloud-heavy sky, casting an eerie, dim light that reflects off his glinting axe. we hear him roar: \"prepare for battle, and eternal glory!\". the camera slowly zooms in on his determined face"
|
||||
],
|
||||
"color": "#232",
|
||||
"bgcolor": "#353"
|
||||
},
|
||||
{
|
||||
"id": 5227,
|
||||
"type": "LTXVGemmaCLIPModelLoader",
|
||||
"pos": [
|
||||
-3298.5249619455894,
|
||||
7907.801436963024
|
||||
],
|
||||
"size": [
|
||||
421.93656760943577,
|
||||
106
|
||||
],
|
||||
"flags": {},
|
||||
"order": 8,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "clip",
|
||||
"type": "CLIP",
|
||||
"links": [
|
||||
13961,
|
||||
13962
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LTXVGemmaCLIPModelLoader",
|
||||
"widget_ue_connectable": {
|
||||
"model_path": true,
|
||||
"max_length": true
|
||||
}
|
||||
},
|
||||
"widgets_values": [
|
||||
"gemma-3-12b-it-qat-q4_0-unquantized/model-00001-of-00005.safetensors",
|
||||
"ltx-2-19b-distilled.safetensors",
|
||||
1024
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5248,
|
||||
"type": "MarkdownNote",
|
||||
"pos": [
|
||||
-2581.888874206601,
|
||||
7104.237854538457
|
||||
],
|
||||
"size": [
|
||||
469.5872194653941,
|
||||
209.10385276580928
|
||||
],
|
||||
"flags": {
|
||||
"collapsed": false
|
||||
},
|
||||
"order": 9,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
"title": "LoRA",
|
||||
"properties": {},
|
||||
"widgets_values": [
|
||||
"# Using LoRAs\n**Ctrl + B to enable**\n\nFor the camera LoRA we recommend setting strength = 1\n\nYou can use LTX [Ready to use camera LoRAs](https://huggingface.co/collections/Lightricks/ltx-2)\n\n\n\n\nUse a second LoRa only when the details are important in the LoRa. \n\n## Training\nWe have made available a full trainer to help you to train your own LoRA. You can [find it here](https://github.com/Lightricks/LTX-2)"
|
||||
],
|
||||
"color": "#432",
|
||||
"bgcolor": "#653"
|
||||
},
|
||||
{
|
||||
"id": 5232,
|
||||
"type": "61915fab-cab7-41be-9727-d69a7e41f24a",
|
||||
@@ -912,8 +862,60 @@
|
||||
"widgets_values": [
|
||||
121,
|
||||
25,
|
||||
42
|
||||
45
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5246,
|
||||
"type": "MarkdownNote",
|
||||
"pos": [
|
||||
-3759.009376983867,
|
||||
7951.430451540103
|
||||
],
|
||||
"size": [
|
||||
418.0950915480007,
|
||||
208.54998853542747
|
||||
],
|
||||
"flags": {
|
||||
"collapsed": false
|
||||
},
|
||||
"order": 10,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
"title": "Prompting LTX-2",
|
||||
"properties": {},
|
||||
"widgets_values": [
|
||||
"## LTX-2 Prompting Tips\n1. **Core Actions**: Describe events and actions as they occur over time \n2. **Audio**: Describe sounds and dialogue needed for the scene \n3. **Consistency**: Avoid instructions that do not match the reference image, as this will degrade results\n"
|
||||
],
|
||||
"color": "#432",
|
||||
"bgcolor": "#653"
|
||||
},
|
||||
{
|
||||
"id": 5245,
|
||||
"type": "MarkdownNote",
|
||||
"pos": [
|
||||
-3759.592503870824,
|
||||
7426.935420507221
|
||||
],
|
||||
"size": [
|
||||
419.9816103344465,
|
||||
480.7830408486252
|
||||
],
|
||||
"flags": {
|
||||
"collapsed": false
|
||||
},
|
||||
"order": 11,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
"title": "Model Links",
|
||||
"properties": {},
|
||||
"widgets_values": [
|
||||
"## Model Links\n\n**LTX-2 Model Weights**\n\n- [ltx-2-19b-distilled.safetensors](https://huggingface.co/Lightricks/LTX-2/resolve/main/ltx-2-19b-distilled.safetensors)\n- [ ltx-2-spatial-upscaler-x2-1.0.safetensors ](https://huggingface.co/Lightricks/LTX-2/resolve/main/ltx-2-spatial-upscaler-x2-1.0.safetensors)\n\n**Text Encoder**\n- [Google Gemma 3](https://huggingface.co/google/gemma-3-12b-it-qat-q4_0-unquantized)\n\nPlease download the entire folder - \n\n- Run: \n 1. cd models/text_encoders\n 2. git clone https://huggingface.co/google/gemma-3-12b-it-qat-q4_0-unquantized\n\n*Full documentation can be found [here](https://docs.ltx.video/open-source-model/integration-tools/comfy-ui#text-encoder)\n\n**Model Storage Location**\n\n```\n📂 ComfyUI/\n├── 📂 models/\n│ ├── 📂 text_encoders/\n│ │ ├── gemma-3-12b-it-qat-q4_0-unquantized/model-00001-of-00005.safetensors\n│ ├── 📂 checkpoints/\n│ │ └── ltx-2-19b-distilled.safetensors\n│ ├── 📂 latent_upscale_models/ \n └── ltx-2-spatial-upscaler-x2-1.0.safetensors\n```\n\n## Report Issues\nTo report any issues when running this workflow, [go to GitHub](https://github.com/Lightricks/ComfyUI-LTXVideo/issues)\n"
|
||||
],
|
||||
"color": "#432",
|
||||
"bgcolor": "#653"
|
||||
}
|
||||
],
|
||||
"links": [
|
||||
@@ -1126,8 +1128,8 @@
|
||||
"nodes": [
|
||||
5224,
|
||||
5223,
|
||||
5222,
|
||||
5225
|
||||
5225,
|
||||
5222
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1163,9 +1165,9 @@
|
||||
"flags": {},
|
||||
"nodes": [
|
||||
5228,
|
||||
5227,
|
||||
5219,
|
||||
5244
|
||||
5244,
|
||||
5227
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -2021,7 +2023,7 @@
|
||||
"Node name for S&R": "RandomNoise"
|
||||
},
|
||||
"widgets_values": [
|
||||
42,
|
||||
45,
|
||||
"fixed"
|
||||
]
|
||||
},
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"id": "394ed254-7306-42a2-9ae6-aa880ce4456d",
|
||||
"revision": 0,
|
||||
"last_node_id": 4903,
|
||||
"last_link_id": 13572,
|
||||
"last_node_id": 4906,
|
||||
"last_link_id": 13574,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 4880,
|
||||
@@ -112,41 +112,6 @@
|
||||
"0"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4637,
|
||||
"type": "KSamplerSelect",
|
||||
"pos": [
|
||||
3724.5096416203287,
|
||||
5144.324911502549
|
||||
],
|
||||
"size": [
|
||||
210,
|
||||
58
|
||||
],
|
||||
"flags": {
|
||||
"collapsed": true
|
||||
},
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "SAMPLER",
|
||||
"type": "SAMPLER",
|
||||
"links": [
|
||||
13535
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"cnr_id": "comfy-core",
|
||||
"ver": "0.3.71",
|
||||
"Node name for S&R": "KSamplerSelect"
|
||||
},
|
||||
"widgets_values": [
|
||||
"euler"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4896,
|
||||
"type": "ManualSigmas",
|
||||
@@ -161,7 +126,7 @@
|
||||
"flags": {
|
||||
"collapsed": true
|
||||
},
|
||||
"order": 1,
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
@@ -174,9 +139,9 @@
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "ManualSigmas",
|
||||
"cnr_id": "comfy-core",
|
||||
"ver": "0.5.1",
|
||||
"Node name for S&R": "ManualSigmas"
|
||||
"ver": "0.5.1"
|
||||
},
|
||||
"widgets_values": [
|
||||
"0.909375, 0.725, 0.421875, 0.0"
|
||||
@@ -213,9 +178,9 @@
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "ImageScaleToMaxDimension",
|
||||
"cnr_id": "comfy-core",
|
||||
"ver": "0.3.75",
|
||||
"Node name for S&R": "ImageScaleToMaxDimension"
|
||||
"ver": "0.3.75"
|
||||
},
|
||||
"widgets_values": [
|
||||
"lanczos",
|
||||
@@ -263,9 +228,9 @@
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "CFGGuider",
|
||||
"cnr_id": "comfy-core",
|
||||
"ver": "0.3.75",
|
||||
"Node name for S&R": "CFGGuider"
|
||||
"ver": "0.3.75"
|
||||
},
|
||||
"widgets_values": [
|
||||
1
|
||||
@@ -285,7 +250,7 @@
|
||||
"flags": {
|
||||
"collapsed": true
|
||||
},
|
||||
"order": 8,
|
||||
"order": 9,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
@@ -307,9 +272,9 @@
|
||||
],
|
||||
"title": "CLIP Text Encode (Positive Prompt)",
|
||||
"properties": {
|
||||
"Node name for S&R": "CLIPTextEncode",
|
||||
"cnr_id": "comfy-core",
|
||||
"ver": "0.3.28",
|
||||
"Node name for S&R": "CLIPTextEncode"
|
||||
"ver": "0.3.28"
|
||||
},
|
||||
"widgets_values": [
|
||||
""
|
||||
@@ -372,9 +337,9 @@
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LTXVConditioning",
|
||||
"cnr_id": "comfy-core",
|
||||
"ver": "0.3.28",
|
||||
"Node name for S&R": "LTXVConditioning"
|
||||
"ver": "0.3.28"
|
||||
},
|
||||
"widgets_values": [
|
||||
24
|
||||
@@ -392,13 +357,13 @@
|
||||
66
|
||||
],
|
||||
"flags": {},
|
||||
"order": 9,
|
||||
"order": 8,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "video",
|
||||
"type": "VIDEO",
|
||||
"link": 13563
|
||||
"link": 13574
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
@@ -426,9 +391,9 @@
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "GetVideoComponents",
|
||||
"cnr_id": "comfy-core",
|
||||
"ver": "0.5.1",
|
||||
"Node name for S&R": "GetVideoComponents"
|
||||
"ver": "0.5.1"
|
||||
},
|
||||
"widgets_values": []
|
||||
},
|
||||
@@ -444,7 +409,7 @@
|
||||
114
|
||||
],
|
||||
"flags": {},
|
||||
"order": 2,
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
@@ -471,47 +436,14 @@
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "CheckpointLoaderSimple",
|
||||
"cnr_id": "comfy-core",
|
||||
"ver": "0.3.56",
|
||||
"Node name for S&R": "CheckpointLoaderSimple"
|
||||
"ver": "0.3.56"
|
||||
},
|
||||
"widgets_values": [
|
||||
"ltx-2-19b-dev.safetensors"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4879,
|
||||
"type": "LTXVGemmaCLIPModelLoader",
|
||||
"pos": [
|
||||
2645.621069544761,
|
||||
5316.232101812384
|
||||
],
|
||||
"size": [
|
||||
379.3903759440785,
|
||||
106
|
||||
],
|
||||
"flags": {},
|
||||
"order": 3,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "clip",
|
||||
"type": "CLIP",
|
||||
"links": [
|
||||
13531
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LTXVGemmaCLIPModelLoader"
|
||||
},
|
||||
"widgets_values": [
|
||||
"gemma-3-12b-it-qat-q4_0-unquantized/model-00001-of-00005.safetensors",
|
||||
"ltx-2-19b-dev.safetensors",
|
||||
1024
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4638,
|
||||
"type": "RandomNoise",
|
||||
@@ -526,7 +458,7 @@
|
||||
"flags": {
|
||||
"collapsed": true
|
||||
},
|
||||
"order": 4,
|
||||
"order": 2,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
@@ -539,9 +471,9 @@
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "RandomNoise",
|
||||
"cnr_id": "comfy-core",
|
||||
"ver": "0.3.71",
|
||||
"Node name for S&R": "RandomNoise"
|
||||
"ver": "0.3.71"
|
||||
},
|
||||
"widgets_values": [
|
||||
42,
|
||||
@@ -585,9 +517,9 @@
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "VAEEncodeTiled",
|
||||
"cnr_id": "comfy-core",
|
||||
"ver": "0.3.75",
|
||||
"Node name for S&R": "VAEEncodeTiled"
|
||||
"ver": "0.3.75"
|
||||
},
|
||||
"widgets_values": [
|
||||
512,
|
||||
@@ -641,9 +573,9 @@
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "CreateVideo",
|
||||
"cnr_id": "comfy-core",
|
||||
"ver": "0.5.1",
|
||||
"Node name for S&R": "CreateVideo"
|
||||
"ver": "0.5.1"
|
||||
},
|
||||
"widgets_values": [
|
||||
30
|
||||
@@ -697,32 +629,6 @@
|
||||
"auto"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4903,
|
||||
"type": "MarkdownNote",
|
||||
"pos": [
|
||||
1855.7109853800114,
|
||||
4980.619423097231
|
||||
],
|
||||
"size": [
|
||||
437.0159573854221,
|
||||
449.95073620797575
|
||||
],
|
||||
"flags": {
|
||||
"collapsed": false
|
||||
},
|
||||
"order": 5,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
"title": "Model Links",
|
||||
"properties": {},
|
||||
"widgets_values": [
|
||||
"## Model Links\n\n**LTX-2 Model Weights**\n\n- [ltx-2-19b-distilled.safetensors](https://huggingface.co/Lightricks/LTX-2/resolve/main/ltx-2-19b-distilled.safetensors)\n- [ltx-2-19b.safetensors](https://huggingface.co/Lightricks/LTX-2/resolve/main/ltx-2-19b.safetensors)\n- [Detailer](https://huggingface.co/Lightricks/LTX-2-19b-IC-LoRA-Detailer/tree/main/ltx-2-19b-ic-lora-detailer.safetensors)\n\n**Text Encoder**\n- [Google Gemma 3](https://huggingface.co/google/gemma-3-12b-pt)\n\n**Model Storage Location**\n\n```\n📂 ComfyUI/\n├── 📂 models/\n│ ├── 📂 text_encoders/\n│ │ ├── comfy_gemma_3_12B_it.safetensors\n│ ├── 📂 checkpoints/\n│ │ └── ltx-2-19b-distilled.safetensors\n│ │ └── ltx-2-19b.safetensors\n│ ├── 📂 loras/\n│ │ ├── ltx-2-19b-ic-lora-detailer.safetensors\n\n```\n\n## Report Issues\nTo report any issues when running this workflow, [go to GitHub](https://github.com/Lightricks/ComfyUI-LTXVideo/issues)\n"
|
||||
],
|
||||
"color": "#432",
|
||||
"bgcolor": "#653"
|
||||
},
|
||||
{
|
||||
"id": 4712,
|
||||
"type": "LoraLoaderModelOnly",
|
||||
@@ -755,12 +661,12 @@
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LoraLoaderModelOnly",
|
||||
"cnr_id": "comfy-core",
|
||||
"ver": "0.3.71",
|
||||
"Node name for S&R": "LoraLoaderModelOnly"
|
||||
"ver": "0.3.71"
|
||||
},
|
||||
"widgets_values": [
|
||||
"ltxv/ltx2/ltx-2-19b-ic-lora-detailer.safetensors",
|
||||
"ltx-2-19b-ic-lora-detailer.safetensors",
|
||||
1
|
||||
]
|
||||
},
|
||||
@@ -797,7 +703,7 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4897,
|
||||
"id": 4905,
|
||||
"type": "LoadVideo",
|
||||
"pos": [
|
||||
2319.858539378856,
|
||||
@@ -805,10 +711,10 @@
|
||||
],
|
||||
"size": [
|
||||
282.798828125,
|
||||
234.25266927083334
|
||||
233.0743408203125
|
||||
],
|
||||
"flags": {},
|
||||
"order": 6,
|
||||
"order": 3,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
@@ -816,19 +722,113 @@
|
||||
"name": "VIDEO",
|
||||
"type": "VIDEO",
|
||||
"links": [
|
||||
13563
|
||||
13574
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LoadVideo",
|
||||
"cnr_id": "comfy-core",
|
||||
"ver": "0.5.1",
|
||||
"Node name for S&R": "LoadVideo"
|
||||
"ver": "0.7.0"
|
||||
},
|
||||
"widgets_values": [
|
||||
"buildings.mp4",
|
||||
"video.mp4",
|
||||
"image"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4637,
|
||||
"type": "KSamplerSelect",
|
||||
"pos": [
|
||||
3724.5096416203287,
|
||||
5144.324911502549
|
||||
],
|
||||
"size": [
|
||||
210,
|
||||
58
|
||||
],
|
||||
"flags": {
|
||||
"collapsed": true
|
||||
},
|
||||
"order": 4,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "SAMPLER",
|
||||
"type": "SAMPLER",
|
||||
"links": [
|
||||
13535
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "KSamplerSelect",
|
||||
"cnr_id": "comfy-core",
|
||||
"ver": "0.3.71"
|
||||
},
|
||||
"widgets_values": [
|
||||
"euler"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4879,
|
||||
"type": "LTXVGemmaCLIPModelLoader",
|
||||
"pos": [
|
||||
2645.621069544761,
|
||||
5316.232101812384
|
||||
],
|
||||
"size": [
|
||||
379.3903759440785,
|
||||
106
|
||||
],
|
||||
"flags": {},
|
||||
"order": 5,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "clip",
|
||||
"type": "CLIP",
|
||||
"links": [
|
||||
13531
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LTXVGemmaCLIPModelLoader"
|
||||
},
|
||||
"widgets_values": [
|
||||
"gemma-3-12b-it-qat-q4_0-unquantized/model-00001-of-00005.safetensors",
|
||||
"ltx-2-19b-dev.safetensors",
|
||||
1024
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4903,
|
||||
"type": "MarkdownNote",
|
||||
"pos": [
|
||||
1856.581308246473,
|
||||
4973.559765691965
|
||||
],
|
||||
"size": [
|
||||
443.7870356486201,
|
||||
476.6587823511936
|
||||
],
|
||||
"flags": {
|
||||
"collapsed": false
|
||||
},
|
||||
"order": 6,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
"title": "Model Links",
|
||||
"properties": {},
|
||||
"widgets_values": [
|
||||
"## Model Links\n\n**LTX-2 Model Weights**\n\n- [ltx-2-19b-dev.safetensors](https://huggingface.co/Lightricks/LTX-2/resolve/main/ltx-2-19b-dev.safetensors)\n- [ltx-2-19b-ic-lora-detailer.safetensors](https://huggingface.co/Lightricks/LTX-2-19b-IC-LoRA-Detailer/resolve/main/ltx-2-19b-ic-lora-detailer.safetensors)\n\n**Text Encoder**\n- [Google Gemma 3](https://huggingface.co/google/gemma-3-12b-it-qat-q4_0-unquantized)\n\nPlease download the entire folder - \n\n- Run: \n 1. cd models/text_encoders\n 2. git clone https://huggingface.co/google/gemma-3-12b-it-qat-q4_0-unquantized\n\n*Full documentation can be found [here](https://docs.ltx.video/open-source-model/integration-tools/comfy-ui#text-encoder)\n\n**Model Storage Location**\n\n```\n📂 ComfyUI/\n├── 📂 models/\n│ ├── 📂 text_encoders/\n│ │ ├── gemma-3-12b-it-qat-q4_0-unquantized/model-00001-of-00005.safetensors\n│ ├── 📂 checkpoints/\n│ │ └── ltx-2-19b-dev.safetensors\n│ ├── 📂 loras/\n│ │ ├── ltx-2-19b-ic-lora-detailer.safetensors\n\n```\n\n## Report Issues\nTo report any issues when running this workflow, [go to GitHub](https://github.com/Lightricks/ComfyUI-LTXVideo/issues)\n"
|
||||
],
|
||||
"color": "#432",
|
||||
"bgcolor": "#653"
|
||||
}
|
||||
],
|
||||
"links": [
|
||||
@@ -976,14 +976,6 @@
|
||||
4,
|
||||
"SIGMAS"
|
||||
],
|
||||
[
|
||||
13563,
|
||||
4897,
|
||||
0,
|
||||
4898,
|
||||
0,
|
||||
"VIDEO"
|
||||
],
|
||||
[
|
||||
13564,
|
||||
4898,
|
||||
@@ -1039,6 +1031,14 @@
|
||||
4899,
|
||||
0,
|
||||
"VAE"
|
||||
],
|
||||
[
|
||||
13574,
|
||||
4905,
|
||||
0,
|
||||
4898,
|
||||
0,
|
||||
"VIDEO"
|
||||
]
|
||||
],
|
||||
"groups": [
|
||||
@@ -1053,12 +1053,7 @@
|
||||
],
|
||||
"color": "#3f789e",
|
||||
"font_size": 24,
|
||||
"flags": {},
|
||||
"nodes": [
|
||||
4521,
|
||||
4879,
|
||||
4712
|
||||
]
|
||||
"flags": {}
|
||||
},
|
||||
{
|
||||
"id": 128,
|
||||
@@ -1071,10 +1066,7 @@
|
||||
],
|
||||
"color": "#3f789e",
|
||||
"font_size": 24,
|
||||
"flags": {},
|
||||
"nodes": [
|
||||
4897
|
||||
]
|
||||
"flags": {}
|
||||
},
|
||||
{
|
||||
"id": 129,
|
||||
@@ -1087,19 +1079,7 @@
|
||||
],
|
||||
"color": "#3f789e",
|
||||
"font_size": 24,
|
||||
"flags": {},
|
||||
"nodes": [
|
||||
4880,
|
||||
4637,
|
||||
4896,
|
||||
4784,
|
||||
4878,
|
||||
2483,
|
||||
1241,
|
||||
4638,
|
||||
4866,
|
||||
4899
|
||||
]
|
||||
"flags": {}
|
||||
}
|
||||
],
|
||||
"config": {},
|
||||
|
||||
@@ -1,297 +0,0 @@
|
||||
import json
|
||||
import logging
|
||||
import weakref
|
||||
from pathlib import Path
|
||||
from types import MethodType
|
||||
|
||||
import accelerate
|
||||
import comfy.sd
|
||||
import comfy.supported_models_base
|
||||
import folder_paths
|
||||
import safetensors
|
||||
import torch
|
||||
from comfy.ldm.lightricks.model import LTXFrequenciesPrecision, LTXRopeType
|
||||
from comfy.utils import load_torch_file
|
||||
from transformers import (
|
||||
AutoImageProcessor,
|
||||
Gemma3ForConditionalGeneration,
|
||||
Gemma3Processor,
|
||||
)
|
||||
|
||||
from .embeddings_connector import Embeddings1DConnector
|
||||
from .gemma_encoder import (
|
||||
GemmaFeaturesExtractorProjLinear,
|
||||
LTXVGemmaTextEncoderModel,
|
||||
find_matching_dir,
|
||||
ltxv_gemma_tokenizer,
|
||||
)
|
||||
from .nodes_registry import comfy_node
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PREFIX_BASE = "model.diffusion_model."
|
||||
|
||||
|
||||
def _is_device_like(x):
|
||||
return isinstance(x, (str, torch.device, int)) or isinstance(
|
||||
x, torch.Tensor
|
||||
) # to(tensor) → matches tensor's device/dtype
|
||||
|
||||
|
||||
def _is_dtype_like(x):
|
||||
# torch.float16 / torch.bfloat16 / torch.float32, etc.
|
||||
return isinstance(x, torch.dtype)
|
||||
|
||||
|
||||
def protect_module_transfers(module: torch.nn.Module, warn=False):
|
||||
"""
|
||||
Make .to/.cuda/.cpu NO-OPs for device moves across an entire module tree,
|
||||
while still allowing dtype/memory_format conversions.
|
||||
"""
|
||||
|
||||
def wrap_one(m: torch.nn.Module):
|
||||
if hasattr(m, "__to_patched"): # idempotent
|
||||
return
|
||||
|
||||
orig_to = m.__class__.to
|
||||
proxy = weakref.proxy(m)
|
||||
|
||||
def safe_to(self, *args, **kwargs):
|
||||
# Block explicit/implicit device moves
|
||||
device_in_kwargs = "device" in kwargs
|
||||
device_like_positional = len(args) > 0 and _is_device_like(args[0])
|
||||
|
||||
# Allow pure dtype/memory_format conversions:
|
||||
# .to(torch.float16)
|
||||
# .to(dtype=torch.float16)
|
||||
# .to(memory_format=...)
|
||||
|
||||
if device_in_kwargs or device_like_positional:
|
||||
if warn:
|
||||
print("[protect_module_transfers] Ignoring device move .to(...).")
|
||||
return self # NO-OP for device moves
|
||||
|
||||
# Otherwise (dtype / memory_format / non_blocking without device) → delegate
|
||||
return orig_to(self, *args, **kwargs)
|
||||
|
||||
# Bind methods
|
||||
m.to = MethodType(safe_to, proxy)
|
||||
m.cuda = MethodType(lambda self, *a, **k: self, proxy) # NO-OP
|
||||
m.cpu = MethodType(lambda self, *a, **k: self, proxy) # NO-OP
|
||||
m.__to_patched = True
|
||||
|
||||
# Recurse over the whole tree
|
||||
for sub in module.modules(): # includes module itself and children
|
||||
wrap_one(sub)
|
||||
|
||||
return module
|
||||
|
||||
|
||||
def load_proj_matrix_from_ltxv(ltxv_path: Path, prefix=""):
|
||||
with safetensors.safe_open(ltxv_path, framework="pt", device="cpu") as f:
|
||||
keys = filter(lambda key: key.startswith(prefix), f.keys())
|
||||
sd = {k.removeprefix(prefix): f.get_tensor(k) for k in keys if k in f.keys()}
|
||||
|
||||
if not sd:
|
||||
return None
|
||||
|
||||
with torch.device("meta"):
|
||||
model = GemmaFeaturesExtractorProjLinear()
|
||||
|
||||
for name, tensor in sd.items():
|
||||
accelerate.utils.set_module_tensor_to_device(model, name, "cpu", value=tensor)
|
||||
|
||||
device_map = accelerate.infer_auto_device_map(
|
||||
model,
|
||||
max_memory=None,
|
||||
)
|
||||
|
||||
model = accelerate.dispatch_model(
|
||||
model,
|
||||
device_map=device_map,
|
||||
)
|
||||
|
||||
return model
|
||||
|
||||
|
||||
def load_proj_matrix_from_checkpoint(checkpoint_path: Path):
|
||||
"""
|
||||
Load model weights from a checkpoint file.
|
||||
:param checkpoint_path: Path to the checkpoint file.
|
||||
"""
|
||||
with torch.device("meta"):
|
||||
model = GemmaFeaturesExtractorProjLinear()
|
||||
model = accelerate.load_checkpoint_and_dispatch(
|
||||
model, str(checkpoint_path.resolve()), device_map="auto"
|
||||
)
|
||||
return model
|
||||
|
||||
|
||||
def ltxv_gemma_clip(encoder_path, ltxv_path, processor=None, dtype=None):
|
||||
class _LTXVGemmaTextEncoderModel(LTXVGemmaTextEncoderModel):
|
||||
def __init__(self, device="cpu", dtype=dtype, model_options={}):
|
||||
dtype = torch.bfloat16 # TODO: make this configurable
|
||||
gemma_model = Gemma3ForConditionalGeneration.from_pretrained(
|
||||
encoder_path,
|
||||
local_files_only=True,
|
||||
torch_dtype=dtype,
|
||||
device_map="auto",
|
||||
)
|
||||
|
||||
feature_extractor_linear = load_proj_matrix_from_ltxv(
|
||||
ltxv_path, "text_embedding_projection."
|
||||
)
|
||||
if feature_extractor_linear is None:
|
||||
feature_extractor_linear = load_proj_matrix_from_checkpoint(
|
||||
encoder_path / "proj_linear.safetensors"
|
||||
)
|
||||
|
||||
embeddings_connector = load_video_embeddings_connector(ltxv_path)
|
||||
audio_embeddings_connector = load_audio_embeddings_connector(ltxv_path)
|
||||
super().__init__(
|
||||
model=gemma_model,
|
||||
feature_extractor_linear=feature_extractor_linear,
|
||||
embeddings_connector=embeddings_connector,
|
||||
audio_embeddings_connector=audio_embeddings_connector,
|
||||
processor=processor,
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
)
|
||||
|
||||
# Due to memory management issue on mgpu we need to disable the offloading of sharded gemma model.
|
||||
protect_module_transfers(self)
|
||||
|
||||
return _LTXVGemmaTextEncoderModel
|
||||
|
||||
|
||||
def load_video_embeddings_connector(ltxv_path, dtype=torch.bfloat16):
|
||||
sd, metadata = load_torch_file(str(ltxv_path), return_metadata=True)
|
||||
config = json.loads(metadata.get("config", "{}"))
|
||||
transformer_config = config.get("transformer", {})
|
||||
rope_type = LTXRopeType.from_dict(transformer_config)
|
||||
frequencies_precision = LTXFrequenciesPrecision.from_dict(transformer_config)
|
||||
pe_max_pos = transformer_config.get("connector_positional_embedding_max_pos", [1])
|
||||
sd_keys = list(sd.keys())
|
||||
|
||||
video_only_connector_prefix = f"{PREFIX_BASE}embeddings_connector."
|
||||
av_connector_prefix = f"{PREFIX_BASE}video_embeddings_connector."
|
||||
prefix = (
|
||||
av_connector_prefix
|
||||
if f"{PREFIX_BASE}audio_adaln_single.linear.weight" in sd_keys
|
||||
else video_only_connector_prefix
|
||||
)
|
||||
return load_embeddings_connector(
|
||||
sd, prefix, dtype, rope_type, frequencies_precision, pe_max_pos
|
||||
)
|
||||
|
||||
|
||||
def load_audio_embeddings_connector(ltxv_path, dtype=torch.bfloat16):
|
||||
sd, metadata = load_torch_file(str(ltxv_path), return_metadata=True)
|
||||
config = json.loads(metadata.get("config", "{}"))
|
||||
transformer_config = config.get("transformer", {})
|
||||
rope_type = LTXRopeType.from_dict(transformer_config)
|
||||
frequencies_precision = LTXFrequenciesPrecision.from_dict(transformer_config)
|
||||
pe_max_pos = transformer_config.get("connector_positional_embedding_max_pos", [1])
|
||||
return load_embeddings_connector(
|
||||
sd,
|
||||
f"{PREFIX_BASE}audio_embeddings_connector.",
|
||||
dtype,
|
||||
rope_type,
|
||||
frequencies_precision,
|
||||
pe_max_pos,
|
||||
)
|
||||
|
||||
|
||||
def load_embeddings_connector(
|
||||
sd,
|
||||
connector_prefix,
|
||||
dtype=torch.bfloat16,
|
||||
rope_type=LTXRopeType.INTERLEAVED,
|
||||
frequencies_precision=LTXFrequenciesPrecision.FLOAT32,
|
||||
pe_max_pos=None,
|
||||
):
|
||||
sd_connector = {
|
||||
k[len(connector_prefix) :]: v
|
||||
for k, v in sd.items()
|
||||
if k.startswith(connector_prefix)
|
||||
}
|
||||
|
||||
if len(sd_connector) == 0:
|
||||
return None
|
||||
|
||||
operations = comfy.ops.pick_operations(dtype, dtype, disable_fast_fp8=True)
|
||||
connector = Embeddings1DConnector(
|
||||
dtype=dtype,
|
||||
operations=operations,
|
||||
positional_embedding_max_pos=pe_max_pos if pe_max_pos is not None else [1],
|
||||
split_rope=rope_type == LTXRopeType.SPLIT,
|
||||
double_precision_rope=frequencies_precision == LTXFrequenciesPrecision.FLOAT64,
|
||||
)
|
||||
connector.load_state_dict(sd_connector)
|
||||
device_map = accelerate.infer_auto_device_map(connector)
|
||||
connector = accelerate.dispatch_model(connector, device_map=device_map)
|
||||
return connector
|
||||
|
||||
|
||||
@comfy_node(
|
||||
name="LTXVGemmaCLIPModelLoaderMGPU", description="Gemma 3 Model Loader on MGPU"
|
||||
)
|
||||
class LTXVGemmaCLIPModelLoaderMGPU:
|
||||
@classmethod
|
||||
def INPUT_TYPES(s):
|
||||
return {
|
||||
"required": {
|
||||
"gemma_path": (
|
||||
folder_paths.get_filename_list("text_encoders"),
|
||||
{"tooltip": "The name of the text encoder model to load."},
|
||||
),
|
||||
"ltxv_path": (
|
||||
folder_paths.get_filename_list("checkpoints"),
|
||||
{"tooltip": "The name of the ltxv model to load."},
|
||||
),
|
||||
"max_length": (
|
||||
"INT",
|
||||
{"default": 1024, "min": 16, "max": 131072, "step": 8},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("CLIP",)
|
||||
RETURN_NAMES = ("clip",)
|
||||
FUNCTION = "load_model"
|
||||
CATEGORY = "lightricks/LTXV"
|
||||
TITLE = "LTXV Gemma CLIP Loader"
|
||||
OUTPUT_NODE = False
|
||||
|
||||
def load_model(self, gemma_path: str, ltxv_path: str, max_length: int):
|
||||
path = Path(folder_paths.get_full_path("text_encoders", gemma_path))
|
||||
model_root = path.parents[1]
|
||||
tokenizer_path = Path(find_matching_dir(model_root, "tokenizer.model"))
|
||||
gemma_model_path = Path(find_matching_dir(model_root, "model*.safetensors"))
|
||||
|
||||
tokenizer_class = ltxv_gemma_tokenizer(tokenizer_path, max_length=max_length)
|
||||
|
||||
processor = None
|
||||
try:
|
||||
image_processor = AutoImageProcessor.from_pretrained(
|
||||
str(model_root),
|
||||
local_files_only=True,
|
||||
)
|
||||
processor = Gemma3Processor(
|
||||
image_processor=image_processor,
|
||||
tokenizer=tokenizer_class().tokenizer,
|
||||
)
|
||||
logger.info(f"Loaded processor from {model_root} - enhancement enabled")
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not load processor from {model_root}: {e}")
|
||||
|
||||
clip_dtype = torch.bfloat16
|
||||
ltxv_full_path = folder_paths.get_full_path("checkpoints", ltxv_path)
|
||||
clip_target = comfy.supported_models_base.ClipTarget(
|
||||
tokenizer=tokenizer_class,
|
||||
clip=ltxv_gemma_clip(
|
||||
gemma_model_path, ltxv_full_path, processor=processor, dtype=clip_dtype
|
||||
),
|
||||
)
|
||||
|
||||
return (comfy.sd.CLIP(clip_target),)
|
||||
@@ -1,4 +1,3 @@
|
||||
accelerate
|
||||
diffusers
|
||||
einops
|
||||
huggingface_hub>=0.25.2
|
||||
|
||||
Reference in New Issue
Block a user