Compare commits
88
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
78897cfe2c | ||
|
|
ef7d6df232 | ||
|
|
f061716fc2 | ||
|
|
eba5309a76 | ||
|
|
7709568b7f | ||
|
|
787185499a | ||
|
|
8577276ea6 | ||
|
|
3b1434d09f | ||
|
|
1da61ed075 | ||
|
|
1c77f6b5e2 | ||
|
|
67429edd85 | ||
|
|
450316699e | ||
|
|
6713097eb8 | ||
|
|
fc1495f8a0 | ||
|
|
4958316d52 | ||
|
|
a43d47be38 | ||
|
|
c640602dbd | ||
|
|
11f20482f3 | ||
|
|
a7e73b968c | ||
|
|
90f500d9d1 | ||
|
|
f16afae95d | ||
|
|
f2003c88ab | ||
|
|
947f5dee1e | ||
|
|
3dff98a341 | ||
|
|
b2cba8c320 | ||
|
|
ac23ad2e8b | ||
|
|
56a89b18c6 | ||
|
|
12889efc55 | ||
|
|
39ea1a8128 | ||
|
|
04f7fd09fd | ||
|
|
51af4749a5 | ||
|
|
e6ed13773b | ||
|
|
50f037a5b4 | ||
|
|
e57e93a91d | ||
|
|
9ad671b1c3 | ||
|
|
6b72f43cc9 | ||
|
|
b19f88a3ea | ||
|
|
e929d0ef8d | ||
|
|
91d0c06847 | ||
|
|
d89597c2c1 | ||
|
|
d8c8d3afce | ||
|
|
3962fb809a | ||
|
|
87e85bfa3e | ||
|
|
8a575afae6 | ||
|
|
3bc7986638 | ||
|
|
feec05e608 | ||
|
|
bc9b43c89c | ||
|
|
50844d8dfe | ||
|
|
b450902437 | ||
|
|
e234a2b167 | ||
|
|
e68b02da93 | ||
|
|
4abda616f4 | ||
|
|
2820626a49 | ||
|
|
c040b83cbc | ||
|
|
6fff546770 | ||
|
|
436a83340d | ||
|
|
b4b4068873 | ||
|
|
098a38a5e5 | ||
|
|
06e2791795 | ||
|
|
b311e23c0e | ||
|
|
7f6d4a63af | ||
|
|
bde2dda60e | ||
|
|
698030281c | ||
|
|
87b7e429c4 | ||
|
|
2498963ea0 | ||
|
|
71005b6ef7 | ||
|
|
0908604ec5 | ||
|
|
dff132868e | ||
|
|
57c12ecb91 | ||
|
|
34da467904 | ||
|
|
4e75a5dc0c | ||
|
|
40bbc722e7 | ||
|
|
413ab4c98e | ||
|
|
3500884a17 | ||
|
|
a2c9120eb1 | ||
|
|
bc6cfc6f48 | ||
|
|
d836d83f69 | ||
|
|
651481e42c | ||
|
|
4fce7e2cfd | ||
|
|
d8873ae85a | ||
|
|
31c51c9907 | ||
|
|
022ddfd619 | ||
|
|
b11310970c | ||
|
|
3d12c323fb | ||
|
|
140a541a93 | ||
|
|
560b6f597e | ||
|
|
57f2bb3dc2 | ||
|
|
925dd4f75c |
+40
@@ -305,5 +305,45 @@ pyrightconfig.json
|
||||
.history
|
||||
.ionide
|
||||
|
||||
|
||||
### macOS ###
|
||||
# General
|
||||
.DS_Store
|
||||
.AppleDouble
|
||||
.LSOverride
|
||||
|
||||
# Icon must end with two \r
|
||||
Icon
|
||||
|
||||
|
||||
# Thumbnails
|
||||
._*
|
||||
|
||||
# Files that might appear in the root of a volume
|
||||
.DocumentRevisions-V100
|
||||
.fseventsd
|
||||
.Spotlight-V100
|
||||
.TemporaryItems
|
||||
.Trashes
|
||||
.VolumeIcon.icns
|
||||
.com.apple.timemachine.donotpresent
|
||||
|
||||
# Directories potentially created on remote AFP share
|
||||
.AppleDB
|
||||
.AppleDesktop
|
||||
Network Trash Folder
|
||||
Temporary Items
|
||||
.apdisk
|
||||
|
||||
### macOS Patch ###
|
||||
# iCloud generated files
|
||||
*.icloud
|
||||
|
||||
# End of https://www.toptal.com/developers/gitignore/api/python,visualstudiocode,pycharm
|
||||
|
||||
.dev.md
|
||||
.dev
|
||||
CLAUDE.md
|
||||
AGENTS.md
|
||||
.gitnexus/
|
||||
.claude/
|
||||
+1
-1
@@ -1,3 +1,3 @@
|
||||
[submodule "lightx2v"]
|
||||
path = lightx2v
|
||||
url = https://github.com/ModelTC/lightx2v.git
|
||||
url = https://github.com/ModelTC/LightX2V.git
|
||||
|
||||
@@ -1,4 +0,0 @@
|
||||
line-length = 150
|
||||
indent-width = 4
|
||||
|
||||
extend-select = ["I"]
|
||||
+40
-3
@@ -1,10 +1,47 @@
|
||||
"""ComfyUI-Lightx2vWrapper entrypoint.
|
||||
|
||||
ComfyUI discovers custom nodes by importing this package and reading
|
||||
``NODE_CLASS_MAPPINGS`` / ``NODE_DISPLAY_NAME_MAPPINGS``. The actual node
|
||||
classes live under the ``nodes/`` subpackage.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
current_path = Path(__file__).parent.absolute()
|
||||
print("Current path set to:", current_path)
|
||||
sys.path.insert(0, os.path.join(current_path, "lightx2v")) # Adjust the path as needed
|
||||
|
||||
def _setup_env() -> None:
|
||||
"""Set environment variables consumed by the bundled lightx2v engine.
|
||||
|
||||
Done in a function (instead of bare module-level statements) so the
|
||||
side effects are explicit and easy to audit. ComfyUI imports this
|
||||
module exactly once at startup, which is when these need to be set.
|
||||
"""
|
||||
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
|
||||
os.environ.setdefault("PROFILING_DEBUG_LEVEL", "2")
|
||||
os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
|
||||
os.environ.setdefault("ENABLE_GRAPH_MODE", "false")
|
||||
os.environ.setdefault("ENABLE_PROFILING_DEBUG", "true")
|
||||
os.environ.setdefault("DTYPE", "BF16")
|
||||
|
||||
|
||||
def _register_lightx2v_submodule() -> None:
|
||||
"""Expose the bundled ``lightx2v/`` git submodule on ``sys.path``.
|
||||
|
||||
The submodule ships its own top-level package also named ``lightx2v``;
|
||||
putting the outer directory on ``sys.path`` lets internal modules import
|
||||
``lightx2v.xxx`` directly (as they do, e.g. ``lightx2v.common.ops``).
|
||||
Our own nodes import via the relative path ``..lightx2v.lightx2v.xxx``
|
||||
and do not depend on this entry, but third-party / lightx2v-internal
|
||||
code does.
|
||||
"""
|
||||
submodule_root = Path(__file__).parent.absolute() / "lightx2v"
|
||||
if str(submodule_root) not in sys.path:
|
||||
sys.path.insert(0, str(submodule_root))
|
||||
|
||||
|
||||
_setup_env()
|
||||
_register_lightx2v_submodule()
|
||||
|
||||
from .nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS # noqa: E402
|
||||
|
||||
|
||||
@@ -1,510 +0,0 @@
|
||||
"""Modular configuration system for LightX2V ComfyUI integration."""
|
||||
|
||||
import copy
|
||||
import importlib.util
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
import torch
|
||||
from easydict import EasyDict
|
||||
|
||||
|
||||
def is_fp8_supported_gpu():
|
||||
if not torch.cuda.is_available():
|
||||
return False
|
||||
compute_capability = torch.cuda.get_device_capability(0)
|
||||
major, minor = compute_capability
|
||||
return (major == 8 and minor == 9) or (major >= 9)
|
||||
|
||||
|
||||
def is_module_installed(module_name):
|
||||
try:
|
||||
spec = importlib.util.find_spec(module_name)
|
||||
return spec is not None
|
||||
except ModuleNotFoundError:
|
||||
return False
|
||||
|
||||
|
||||
def get_available_quant_ops():
|
||||
available_ops = []
|
||||
|
||||
vllm_installed = is_module_installed("vllm")
|
||||
if vllm_installed:
|
||||
available_ops.append(("vllm", True))
|
||||
else:
|
||||
available_ops.append(("vllm", False))
|
||||
|
||||
sgl_installed = is_module_installed("sgl_kernel")
|
||||
if sgl_installed:
|
||||
available_ops.append(("sgl", True))
|
||||
else:
|
||||
available_ops.append(("sgl", False))
|
||||
|
||||
q8f_installed = is_module_installed("q8_kernels")
|
||||
if q8f_installed:
|
||||
available_ops.append(("q8f", True))
|
||||
else:
|
||||
available_ops.append(("q8f", False))
|
||||
|
||||
return available_ops
|
||||
|
||||
|
||||
def get_available_attn_ops():
|
||||
available_ops = []
|
||||
|
||||
vllm_installed = is_module_installed("flash_attn")
|
||||
if vllm_installed:
|
||||
available_ops.append(("flash_attn2", True))
|
||||
else:
|
||||
available_ops.append(("flash_attn2", False))
|
||||
|
||||
sgl_installed = is_module_installed("flash_attn_interface")
|
||||
if sgl_installed:
|
||||
available_ops.append(("flash_attn3", True))
|
||||
else:
|
||||
available_ops.append(("flash_attn3", False))
|
||||
|
||||
q8f_installed = is_module_installed("sageattention")
|
||||
if q8f_installed:
|
||||
available_ops.append(("sage_attn2", True))
|
||||
else:
|
||||
available_ops.append(("sage_attn2", False))
|
||||
|
||||
torch_installed = is_module_installed("torch")
|
||||
if torch_installed:
|
||||
available_ops.append(("torch_sdpa", True))
|
||||
else:
|
||||
available_ops.append(("torch_sdpa", False))
|
||||
|
||||
return available_ops
|
||||
|
||||
|
||||
class LightX2VDefaultConfig:
|
||||
"""Central default configuration for LightX2V."""
|
||||
|
||||
DEFAULT_CONFIG = {
|
||||
# ========== Model Configuration ==========
|
||||
"model_cls": "wan2.1",
|
||||
"model_path": "",
|
||||
"task": "t2v",
|
||||
"mode": "infer",
|
||||
# ========== Inference Parameters ==========
|
||||
"infer_steps": 40,
|
||||
"seed": 42,
|
||||
"sample_guide_scale": 5.0,
|
||||
"sample_shift": 5,
|
||||
"enable_cfg": True,
|
||||
"prompt": "",
|
||||
"negative_prompt": "",
|
||||
# ========== Video Parameters ==========
|
||||
"target_height": 480,
|
||||
"target_width": 832,
|
||||
"target_video_length": 81,
|
||||
"fps": 16,
|
||||
"vae_stride": [4, 8, 8],
|
||||
"patch_size": [1, 2, 2],
|
||||
# ========== Feature Caching (TeaCache) ==========
|
||||
"feature_caching": "NoCaching",
|
||||
"teacache_thresh": 0.26,
|
||||
"coefficients": None, # Auto-calculated
|
||||
"use_ret_steps": False,
|
||||
# ========== Quantization ==========
|
||||
"dit_quant_scheme": "bf16",
|
||||
"t5_quant_scheme": "bf16",
|
||||
"clip_quant_scheme": "fp16",
|
||||
"quant_op": "vllm",
|
||||
"precision_mode": "fp32",
|
||||
"dit_quantized_ckpt": None,
|
||||
"t5_quantized_ckpt": None,
|
||||
"clip_quantized_ckpt": None,
|
||||
"mm_config": {"mm_type": "Default"},
|
||||
# ========== GPU Memory Optimization ==========
|
||||
"rotary_chunk": False,
|
||||
"rotary_chunk_size": 100,
|
||||
"clean_cuda_cache": False,
|
||||
"torch_compile": False,
|
||||
"attention_type": "flash_attn3",
|
||||
"self_attn_1_type": "flash_attn3",
|
||||
"cross_attn_1_type": "flash_attn3",
|
||||
"cross_attn_2_type": "flash_attn3",
|
||||
# ========== Async Offloading ==========
|
||||
"cpu_offload": False,
|
||||
"offload_granularity": "phase",
|
||||
"offload_ratio": 1.0,
|
||||
"t5_cpu_offload": False,
|
||||
"t5_offload_granularity": "model",
|
||||
"lazy_load": False,
|
||||
"unload_modules": False,
|
||||
# ========== Lightweight VAE ==========
|
||||
"use_tiny_vae": False,
|
||||
"tiny_vae": False,
|
||||
"tiny_vae_path": None,
|
||||
"use_tiling_vae": False,
|
||||
# ========== Other Settings ==========
|
||||
"lora_path": None,
|
||||
"strength_model": 1.0,
|
||||
"do_mm_calib": False,
|
||||
"parallel_attn_type": None,
|
||||
"parallel_vae": False,
|
||||
"max_area": False,
|
||||
"use_prompt_enhancer": False,
|
||||
"text_len": 512,
|
||||
}
|
||||
|
||||
|
||||
class CoefficientCalculator:
|
||||
"""Calculate TeaCache coefficients based on model and resolution."""
|
||||
|
||||
COEFFICIENTS = {
|
||||
"t2v": {
|
||||
"1.3b": {
|
||||
"default": [
|
||||
[
|
||||
-5.21862437e04,
|
||||
9.23041404e03,
|
||||
-5.28275948e02,
|
||||
1.36987616e01,
|
||||
-4.99875664e-02,
|
||||
],
|
||||
[
|
||||
2.39676752e03,
|
||||
-1.31110545e03,
|
||||
2.01331979e02,
|
||||
-8.29855975e00,
|
||||
1.37887774e-01,
|
||||
],
|
||||
]
|
||||
},
|
||||
"14b": {
|
||||
"default": [
|
||||
[
|
||||
-3.03318725e05,
|
||||
4.90537029e04,
|
||||
-2.65530556e03,
|
||||
5.87365115e01,
|
||||
-3.15583525e-01,
|
||||
],
|
||||
[
|
||||
-5784.54975374,
|
||||
5449.50911966,
|
||||
-1811.16591783,
|
||||
256.27178429,
|
||||
-13.02252404,
|
||||
],
|
||||
]
|
||||
},
|
||||
},
|
||||
"i2v": {
|
||||
"720p": [
|
||||
[
|
||||
8.10705460e03,
|
||||
2.13393892e03,
|
||||
-3.72934672e02,
|
||||
1.66203073e01,
|
||||
-4.17769401e-02,
|
||||
],
|
||||
[-114.36346466, 65.26524496, -18.82220707, 4.91518089, -0.23412683],
|
||||
],
|
||||
"480p": [
|
||||
[
|
||||
2.57151496e05,
|
||||
-3.54229917e04,
|
||||
1.40286849e03,
|
||||
-1.35890334e01,
|
||||
1.32517977e-01,
|
||||
],
|
||||
[
|
||||
-3.02331670e02,
|
||||
2.23948934e02,
|
||||
-5.25463970e01,
|
||||
5.87348440e00,
|
||||
-2.01973289e-01,
|
||||
],
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def get_coefficients(
|
||||
cls,
|
||||
task: str,
|
||||
model_size: str,
|
||||
resolution: Tuple[int, int],
|
||||
use_ret_steps: bool,
|
||||
) -> List[List[float]]:
|
||||
"""Get appropriate coefficients for TeaCache."""
|
||||
if task == "t2v":
|
||||
coeffs = cls.COEFFICIENTS["t2v"].get(model_size, {}).get("default", None)
|
||||
else: # i2v
|
||||
width, height = resolution
|
||||
if height >= 720 or width >= 720:
|
||||
coeffs = cls.COEFFICIENTS["i2v"]["720p"]
|
||||
else:
|
||||
coeffs = cls.COEFFICIENTS["i2v"]["480p"]
|
||||
|
||||
if coeffs:
|
||||
return coeffs[0] if use_ret_steps else coeffs[1]
|
||||
raise ValueError(
|
||||
f"No coefficients found for task: {task}, model_size: {model_size}, resolution: {resolution}, use_ret_steps: {use_ret_steps}"
|
||||
)
|
||||
|
||||
|
||||
class ModularConfigManager:
|
||||
"""Manages modular configuration without presets."""
|
||||
|
||||
def __init__(self):
|
||||
self.base_config = copy.deepcopy(LightX2VDefaultConfig.DEFAULT_CONFIG)
|
||||
self._available_attn_ops = None
|
||||
self._available_quant_ops = None
|
||||
|
||||
@property
|
||||
def available_attention_types(self) -> List[str]:
|
||||
"""Get available attention types."""
|
||||
if self._available_attn_ops is None:
|
||||
self._available_attn_ops = get_available_attn_ops()
|
||||
|
||||
available = []
|
||||
for op_name, is_available in self._available_attn_ops:
|
||||
if is_available:
|
||||
available.append(op_name)
|
||||
|
||||
if "torch_sdpa" not in available:
|
||||
available.append("torch_sdpa")
|
||||
|
||||
return available
|
||||
|
||||
@property
|
||||
def available_quant_schemes(self) -> List[str]:
|
||||
"""Get available quantization schemes."""
|
||||
if self._available_quant_ops is None:
|
||||
self._available_quant_ops = get_available_quant_ops()
|
||||
|
||||
available = []
|
||||
for op_name, is_available in self._available_quant_ops:
|
||||
if is_available:
|
||||
available.append(op_name)
|
||||
|
||||
return available
|
||||
|
||||
def apply_inference_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Apply basic inference configuration."""
|
||||
updates = {}
|
||||
|
||||
if "model_cls" in config:
|
||||
updates["model_cls"] = config["model_cls"]
|
||||
if "model_path" in config:
|
||||
updates["model_path"] = config["model_path"]
|
||||
if "task" in config:
|
||||
updates["task"] = config["task"]
|
||||
|
||||
if "infer_steps" in config:
|
||||
updates["infer_steps"] = config["infer_steps"]
|
||||
if "seed" in config and config["seed"] != -1:
|
||||
updates["seed"] = config["seed"]
|
||||
if "cfg_scale" in config:
|
||||
updates["sample_guide_scale"] = config["cfg_scale"]
|
||||
updates["enable_cfg"] = config["cfg_scale"] != 1.0
|
||||
if "sample_shift" in config:
|
||||
updates["sample_shift"] = config["sample_shift"]
|
||||
|
||||
if "height" in config:
|
||||
updates["target_height"] = config["height"]
|
||||
if "width" in config:
|
||||
updates["target_width"] = config["width"]
|
||||
if "video_length" in config:
|
||||
updates["target_video_length"] = config["video_length"]
|
||||
if "fps" in config:
|
||||
updates["fps"] = config["fps"]
|
||||
|
||||
if "denoising_step_list" in config:
|
||||
updates["denoising_step_list"] = config["denoising_step_list"]
|
||||
|
||||
return updates
|
||||
|
||||
def apply_teacache_config(
|
||||
self, config: Dict[str, Any], model_info: Dict[str, Any]
|
||||
) -> Dict[str, Any]:
|
||||
"""Apply TeaCache configuration."""
|
||||
updates = {}
|
||||
|
||||
if config.get("enable", False):
|
||||
updates["feature_caching"] = "Tea"
|
||||
updates["teacache_thresh"] = config.get("threshold", 0.26)
|
||||
updates["use_ret_steps"] = config.get("use_ret_steps", False)
|
||||
|
||||
task = model_info.get("task", "t2v")
|
||||
model_size = "14b" if "14b" in model_info.get("model_cls", "") else "1.3b"
|
||||
resolution = (
|
||||
model_info.get("target_width", 832),
|
||||
model_info.get("target_height", 480),
|
||||
)
|
||||
|
||||
coeffs = CoefficientCalculator.get_coefficients(
|
||||
task, model_size, resolution, updates["use_ret_steps"]
|
||||
)
|
||||
updates["coefficients"] = coeffs
|
||||
else:
|
||||
updates["feature_caching"] = "NoCaching"
|
||||
|
||||
return updates
|
||||
|
||||
def apply_quantization_config(
|
||||
self, config: Dict[str, Any], model_path: str
|
||||
) -> Dict[str, Any]:
|
||||
"""Apply quantization configuration."""
|
||||
updates = {}
|
||||
|
||||
dit_scheme = config.get("dit_precision", "bf16")
|
||||
updates["dit_quant_scheme"] = dit_scheme
|
||||
if dit_scheme != "bf16":
|
||||
updates["dit_quantized_ckpt"] = os.path.join(model_path, dit_scheme)
|
||||
|
||||
t5_scheme = config.get("t5_precision", "bf16")
|
||||
updates["t5_quant_scheme"] = t5_scheme
|
||||
updates["t5_quantized"] = t5_scheme != "bf16"
|
||||
if t5_scheme != "bf16":
|
||||
t5_path = os.path.join(model_path, t5_scheme)
|
||||
updates["t5_quantized_ckpt"] = os.path.join(
|
||||
t5_path, f"models_t5_umt5-xxl-enc-{t5_scheme}.pth"
|
||||
)
|
||||
|
||||
clip_scheme = config.get("clip_precision", "fp16")
|
||||
updates["clip_quant_scheme"] = clip_scheme
|
||||
updates["clip_quantized"] = clip_scheme != "fp16"
|
||||
if clip_scheme != "fp16":
|
||||
clip_path = os.path.join(model_path, clip_scheme)
|
||||
updates["clip_quantized_ckpt"] = os.path.join(
|
||||
clip_path, f"clip-{clip_scheme}.pth"
|
||||
)
|
||||
|
||||
quant_backend = config.get("quant_backend", "vllm")
|
||||
updates["quant_op"] = quant_backend
|
||||
|
||||
if dit_scheme != "bf16":
|
||||
if quant_backend == "vllm":
|
||||
mm_type = f"W-{dit_scheme}-channel-sym-A-{dit_scheme}-channel-sym-dynamic-Vllm"
|
||||
elif quant_backend == "sgl":
|
||||
if dit_scheme == "int8":
|
||||
mm_type = f"W-{dit_scheme}-channel-sym-A-{dit_scheme}-channel-sym-dynamic-Sgl-ActVllm"
|
||||
else:
|
||||
mm_type = f"W-{dit_scheme}-channel-sym-A-{dit_scheme}-channel-sym-dynamic-Sgl"
|
||||
elif quant_backend == "q8f":
|
||||
mm_type = (
|
||||
f"W-{dit_scheme}-channel-sym-A-{dit_scheme}-channel-sym-dynamic-Q8F"
|
||||
)
|
||||
else:
|
||||
mm_type = "Default"
|
||||
|
||||
updates["mm_config"] = {"mm_type": mm_type}
|
||||
else:
|
||||
updates["mm_config"] = {"mm_type": "Default"}
|
||||
|
||||
updates["precision_mode"] = config.get("sensitive_layers_precision", "fp32")
|
||||
|
||||
return updates
|
||||
|
||||
def apply_memory_optimization(self, config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Apply memory optimization settings."""
|
||||
updates = {}
|
||||
|
||||
level = config.get("optimization_level", "none")
|
||||
|
||||
# GPU optimization
|
||||
if config.get("enable_rotary_chunk", False) or level in ["high", "extreme"]:
|
||||
updates["rotary_chunk"] = True
|
||||
updates["rotary_chunk_size"] = config.get("rotary_chunk_size", 100)
|
||||
|
||||
if config.get("clean_cuda_cache", False) or level == "extreme":
|
||||
updates["clean_cuda_cache"] = True
|
||||
|
||||
# CPU offloading
|
||||
if config.get("enable_cpu_offload", False) or level in [
|
||||
"medium",
|
||||
"high",
|
||||
"extreme",
|
||||
]:
|
||||
updates["cpu_offload"] = True
|
||||
updates["offload_granularity"] = config.get("offload_granularity", "phase")
|
||||
updates["offload_ratio"] = config.get("offload_ratio", 1.0)
|
||||
|
||||
# T5 offloading
|
||||
if level in ["high", "extreme"]:
|
||||
updates["t5_cpu_offload"] = True
|
||||
updates["t5_offload_granularity"] = (
|
||||
"block" if level == "extreme" else "model"
|
||||
)
|
||||
|
||||
# Module management
|
||||
if config.get("lazy_load", False) or level == "extreme":
|
||||
updates["lazy_load"] = True
|
||||
|
||||
if config.get("unload_after_inference", False) or level == "extreme":
|
||||
updates["unload_modules"] = True
|
||||
|
||||
# Attention type
|
||||
attention_type = config.get("attention_type", "flash_attn3")
|
||||
updates["attention_type"] = attention_type
|
||||
updates["self_attn_1_type"] = attention_type
|
||||
updates["cross_attn_1_type"] = attention_type
|
||||
updates["cross_attn_2_type"] = attention_type
|
||||
|
||||
return updates
|
||||
|
||||
def apply_vae_config(
|
||||
self, config: Dict[str, Any], model_path: str
|
||||
) -> Dict[str, Any]:
|
||||
"""Apply VAE configuration."""
|
||||
updates = {}
|
||||
|
||||
if config.get("use_tiny_vae", False):
|
||||
updates["use_tiny_vae"] = True
|
||||
updates["tiny_vae"] = True
|
||||
updates["tiny_vae_path"] = os.path.join(model_path, "taew2_1.pth")
|
||||
|
||||
if config.get("use_tiling_vae", False):
|
||||
updates["use_tiling_vae"] = True
|
||||
|
||||
return updates
|
||||
|
||||
def build_final_config(self, configs: Dict[str, Dict[str, Any]]) -> EasyDict:
|
||||
"""Build final configuration from module configs."""
|
||||
final_config = copy.deepcopy(self.base_config)
|
||||
|
||||
if "inference" in configs:
|
||||
final_config.update(self.apply_inference_config(configs["inference"]))
|
||||
|
||||
if "teacache" in configs:
|
||||
teacache_updates = self.apply_teacache_config(
|
||||
configs["teacache"],
|
||||
final_config,
|
||||
)
|
||||
final_config.update(teacache_updates)
|
||||
|
||||
if "quantization" in configs:
|
||||
model_path = final_config.get("model_path", "")
|
||||
quant_updates = self.apply_quantization_config(
|
||||
configs["quantization"], model_path
|
||||
)
|
||||
final_config.update(quant_updates)
|
||||
|
||||
if "memory" in configs:
|
||||
final_config.update(self.apply_memory_optimization(configs["memory"]))
|
||||
|
||||
if "vae" in configs:
|
||||
model_path = final_config.get("model_path", "")
|
||||
final_config.update(self.apply_vae_config(configs["vae"], model_path))
|
||||
|
||||
model_config_path = os.path.join(final_config["model_path"], "config.json")
|
||||
if os.path.exists(model_config_path):
|
||||
try:
|
||||
with open(model_config_path, "r") as f:
|
||||
model_config = json.load(f)
|
||||
for key, value in model_config.items():
|
||||
if key not in final_config or final_config[key] is None:
|
||||
final_config[key] = value
|
||||
except Exception as e:
|
||||
logging.warning(f"Failed to load model config: {e}")
|
||||
|
||||
return EasyDict(final_config)
|
||||
@@ -0,0 +1,41 @@
|
||||
"""Bridge between ComfyUI widget values and lightx2v's internal config schema.
|
||||
|
||||
Submodules:
|
||||
- ``capability`` GPU + backend-op detection (pure functions)
|
||||
- ``defaults`` ``LightX2VDefaultConfig`` — wrapper-side starting values
|
||||
- ``teacache_coeffs`` ``CoefficientCalculator`` — polynomial constants
|
||||
- ``translator/`` per-feature wrapper-key -> lightx2v-key translators,
|
||||
plus ``ModularConfigManager`` that orchestrates them
|
||||
|
||||
Public surface (re-exported here for backward compat with existing imports
|
||||
``from .bridge import …``):
|
||||
"""
|
||||
|
||||
from .capability import (
|
||||
get_available_attn_ops,
|
||||
get_available_ops,
|
||||
get_available_quant_ops,
|
||||
get_gpu_capability,
|
||||
is_ada_architecture_gpu,
|
||||
is_fp8_supported_gpu,
|
||||
is_module_installed,
|
||||
)
|
||||
from .defaults import LightX2VDefaultConfig
|
||||
from .teacache_coeffs import CoefficientCalculator
|
||||
from .translator import ModularConfigManager
|
||||
|
||||
__all__ = [
|
||||
# capability
|
||||
"get_gpu_capability",
|
||||
"is_fp8_supported_gpu",
|
||||
"is_ada_architecture_gpu",
|
||||
"is_module_installed",
|
||||
"get_available_ops",
|
||||
"get_available_quant_ops",
|
||||
"get_available_attn_ops",
|
||||
# defaults / coeffs
|
||||
"LightX2VDefaultConfig",
|
||||
"CoefficientCalculator",
|
||||
# translator orchestrator
|
||||
"ModularConfigManager",
|
||||
]
|
||||
@@ -0,0 +1,80 @@
|
||||
"""GPU and backend-op capability detection.
|
||||
|
||||
Pure functions — no state, no module-level side effects. Cheap to call
|
||||
(the underlying ``torch.cuda`` / ``importlib`` probes are fast).
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import logging
|
||||
from typing import List, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
def get_gpu_capability():
|
||||
if not torch.cuda.is_available():
|
||||
return None, None
|
||||
try:
|
||||
return torch.cuda.get_device_capability(0)
|
||||
except Exception as e:
|
||||
logging.warning(f"Failed to get GPU capability: {e}")
|
||||
return None, None
|
||||
|
||||
|
||||
def is_fp8_supported_gpu() -> bool:
|
||||
major, minor = get_gpu_capability()
|
||||
if major is None:
|
||||
return False
|
||||
return (major == 8 and minor == 9) or (major >= 9)
|
||||
|
||||
|
||||
def is_ada_architecture_gpu() -> bool:
|
||||
major, minor = get_gpu_capability()
|
||||
if major is None:
|
||||
return False
|
||||
return major == 8 and minor == 9
|
||||
|
||||
|
||||
def is_module_installed(module_name: str) -> bool:
|
||||
try:
|
||||
spec = importlib.util.find_spec(module_name)
|
||||
return spec is not None
|
||||
except ModuleNotFoundError:
|
||||
return False
|
||||
|
||||
|
||||
def get_available_ops(op_mapping: dict) -> List[Tuple[str, bool]]:
|
||||
return [(op_name, is_module_installed(module_name)) for op_name, module_name in op_mapping.items()]
|
||||
|
||||
|
||||
_QUANT_OP_MAPPING = {
|
||||
"sgl": "sgl_kernel",
|
||||
"vllm": "vllm",
|
||||
"q8f": "q8_kernels",
|
||||
"torchao": "torchao",
|
||||
}
|
||||
|
||||
_ATTN_OP_MAPPING = {
|
||||
"sage_attn2": "sageattention",
|
||||
"sage_attn3": "sageattn3",
|
||||
"flash_attn3": "flash_attn_interface",
|
||||
"flash_attn2": "flash_attn",
|
||||
"torch_sdpa": "torch",
|
||||
}
|
||||
|
||||
|
||||
def get_available_quant_ops() -> List[Tuple[str, bool]]:
|
||||
available_ops = get_available_ops(_QUANT_OP_MAPPING)
|
||||
|
||||
# Prefer q8f on Ada (sm_8.9) GPUs — best perf/precision tradeoff there.
|
||||
if is_ada_architecture_gpu():
|
||||
q8f_available = next((op for op in available_ops if op[0] == "q8f" and op[1]), None)
|
||||
if q8f_available:
|
||||
available_ops.remove(q8f_available)
|
||||
available_ops.insert(0, q8f_available)
|
||||
|
||||
return available_ops
|
||||
|
||||
|
||||
def get_available_attn_ops() -> List[Tuple[str, bool]]:
|
||||
return get_available_ops(_ATTN_OP_MAPPING)
|
||||
@@ -0,0 +1,80 @@
|
||||
"""Default config values that the wrapper provides to lightx2v.
|
||||
|
||||
These are the wrapper's *starting point* — lightx2v's own ``set_config`` will
|
||||
further merge from ``config_json`` and the model's own ``config.json`` on disk
|
||||
(see ``lightx2v/utils/set_config.py:set_config``). Anything lightx2v sets
|
||||
internally (``vae_stride``, ``patch_size``, etc.) should NOT be duplicated here.
|
||||
"""
|
||||
|
||||
|
||||
class LightX2VDefaultConfig:
|
||||
"""Central default configuration for LightX2V."""
|
||||
|
||||
DEFAULT_ATTENTION_TYPE = "flash_attn3"
|
||||
DEFAULT_QUANTIZATION_SCHEMES = {
|
||||
"dit": "Default",
|
||||
"t5": "Default",
|
||||
"clip": "Default",
|
||||
"adapter": "Default",
|
||||
}
|
||||
|
||||
DEFAULT_CONFIG = {
|
||||
# Model
|
||||
"model_cls": "wan2.1",
|
||||
"model_path": "",
|
||||
"task": "t2v",
|
||||
# Inference
|
||||
"infer_steps": 40,
|
||||
"seed": 42,
|
||||
"sample_guide_scale": 5.0,
|
||||
"sample_shift": 5,
|
||||
"enable_cfg": True,
|
||||
"prompt": "",
|
||||
"negative_prompt": "",
|
||||
# Video / Image output (lightx2v field names — see translator/inference.py)
|
||||
"target_height": 480,
|
||||
"target_width": 832,
|
||||
"target_video_length": 81,
|
||||
"fps": 16,
|
||||
# TeaCache
|
||||
"feature_caching": "NoCaching",
|
||||
"teacache_thresh": 0.26,
|
||||
"coefficients": None,
|
||||
"use_ret_steps": False,
|
||||
# Quantization
|
||||
"dit_quant_scheme": DEFAULT_QUANTIZATION_SCHEMES["dit"],
|
||||
"t5_quant_scheme": DEFAULT_QUANTIZATION_SCHEMES["t5"],
|
||||
"clip_quant_scheme": DEFAULT_QUANTIZATION_SCHEMES["clip"],
|
||||
"adapter_quant_scheme": DEFAULT_QUANTIZATION_SCHEMES["adapter"],
|
||||
# Attention
|
||||
"self_attn_1_type": DEFAULT_ATTENTION_TYPE,
|
||||
"cross_attn_1_type": DEFAULT_ATTENTION_TYPE,
|
||||
"cross_attn_2_type": DEFAULT_ATTENTION_TYPE,
|
||||
# Memory / offload
|
||||
"rotary_chunk": False,
|
||||
"rotary_chunk_size": 100,
|
||||
"clean_cuda_cache": False,
|
||||
"torch_compile": False,
|
||||
"cpu_offload": False,
|
||||
"offload_granularity": "block",
|
||||
"offload_ratio": 1.0,
|
||||
"t5_cpu_offload": False,
|
||||
"t5_offload_granularity": "model",
|
||||
"lazy_load": False,
|
||||
"unload_modules": False,
|
||||
# VAE
|
||||
"use_tiling_vae": False,
|
||||
# Misc
|
||||
"do_mm_calib": False,
|
||||
"max_area": False,
|
||||
"use_prompt_enhancer": False,
|
||||
"text_len": 512,
|
||||
"use_31_block": True,
|
||||
"parallel": False,
|
||||
"seq_parallel": False,
|
||||
"cfg_parallel": False,
|
||||
"audio_sr": 16000,
|
||||
"talk_objects": None,
|
||||
"boundary_step_index": 2,
|
||||
"rope_type": "torch",
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
"""TeaCache polynomial coefficients per (task, model size, resolution).
|
||||
|
||||
These constants come from the upstream TeaCache calibration runs (one set per
|
||||
task/resolution bucket). They are pure data; no logic here other than picking
|
||||
the right bucket.
|
||||
"""
|
||||
|
||||
from typing import List, Tuple
|
||||
|
||||
|
||||
class CoefficientCalculator:
|
||||
"""Pick TeaCache polynomial coefficients for a given task/model/resolution."""
|
||||
|
||||
COEFFICIENTS = {
|
||||
"t2v": {
|
||||
"1.3b": {
|
||||
"default": [
|
||||
[-5.21862437e04, 9.23041404e03, -5.28275948e02, 1.36987616e01, -4.99875664e-02],
|
||||
[2.39676752e03, -1.31110545e03, 2.01331979e02, -8.29855975e00, 1.37887774e-01],
|
||||
]
|
||||
},
|
||||
"14b": {
|
||||
"default": [
|
||||
[-3.03318725e05, 4.90537029e04, -2.65530556e03, 5.87365115e01, -3.15583525e-01],
|
||||
[-5784.54975374, 5449.50911966, -1811.16591783, 256.27178429, -13.02252404],
|
||||
]
|
||||
},
|
||||
},
|
||||
"i2v": {
|
||||
"720p": [
|
||||
[8.10705460e03, 2.13393892e03, -3.72934672e02, 1.66203073e01, -4.17769401e-02],
|
||||
[-114.36346466, 65.26524496, -18.82220707, 4.91518089, -0.23412683],
|
||||
],
|
||||
"480p": [
|
||||
[2.57151496e05, -3.54229917e04, 1.40286849e03, -1.35890334e01, 1.32517977e-01],
|
||||
[-3.02331670e02, 2.23948934e02, -5.25463970e01, 5.87348440e00, -2.01973289e-01],
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def get_coefficients(
|
||||
cls,
|
||||
task: str,
|
||||
model_size: str,
|
||||
resolution: Tuple[int, int],
|
||||
use_ret_steps: bool,
|
||||
) -> List[float]:
|
||||
"""Pick the right coefficient row for this (task, model_size, resolution).
|
||||
|
||||
``use_ret_steps`` selects between the two calibration runs (cache key
|
||||
steps only vs. cache all steps).
|
||||
"""
|
||||
if task == "t2v":
|
||||
coeffs = cls.COEFFICIENTS["t2v"].get(model_size, {}).get("default", None)
|
||||
else: # i2v
|
||||
width, height = resolution
|
||||
coeffs = cls.COEFFICIENTS["i2v"]["720p"] if height >= 720 or width >= 720 else cls.COEFFICIENTS["i2v"]["480p"]
|
||||
|
||||
if coeffs:
|
||||
return coeffs[0] if use_ret_steps else coeffs[1]
|
||||
raise ValueError(
|
||||
f"No coefficients found for task: {task}, model_size: {model_size}, resolution: {resolution}, use_ret_steps: {use_ret_steps}"
|
||||
)
|
||||
@@ -0,0 +1,26 @@
|
||||
"""Field translators: ComfyUI widget value -> lightx2v config dict.
|
||||
|
||||
Each module here mirrors one ComfyUI Config node in ``nodes/config.py``:
|
||||
|
||||
LightX2VInferenceConfig <-> translator/inference.py
|
||||
LightX2VTeaCache <-> translator/teacache.py
|
||||
LightX2VQuantization <-> translator/quant.py
|
||||
LightX2VMemoryOptimization <-> translator/memory.py
|
||||
|
||||
``translator/pipeline.py`` orchestrates them and adds the model's own
|
||||
``config.json`` (read from disk by lightx2v's ``set_config``).
|
||||
"""
|
||||
|
||||
from .inference import apply_inference_config
|
||||
from .memory import apply_memory_optimization
|
||||
from .pipeline import ModularConfigManager
|
||||
from .quant import apply_quantization_config
|
||||
from .teacache import apply_teacache_config
|
||||
|
||||
__all__ = [
|
||||
"apply_inference_config",
|
||||
"apply_teacache_config",
|
||||
"apply_quantization_config",
|
||||
"apply_memory_optimization",
|
||||
"ModularConfigManager",
|
||||
]
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Translate ``LightX2VInferenceConfig`` widget values into lightx2v config keys.
|
||||
|
||||
The wrapper's widget naming follows ComfyUI conventions (``height``, ``width``,
|
||||
``video_length``, ``cfg_scale`` …). lightx2v's internal naming is different
|
||||
(``target_height``, ``target_width``, ``target_video_length``,
|
||||
``sample_guide_scale`` …). The single source of truth for that translation
|
||||
is the ``WRAPPER_TO_LIGHTX2V_FIELDS`` table below — when adding a new field,
|
||||
add a row there rather than burying the rename inside the function body.
|
||||
"""
|
||||
|
||||
import os
|
||||
from typing import Any, Dict
|
||||
|
||||
from ..defaults import LightX2VDefaultConfig
|
||||
|
||||
# Direct rename map: wrapper-side key -> lightx2v-side key.
|
||||
# A row of ("foo", "foo") means the name matches but we still want to forward
|
||||
# the value explicitly (rather than relying on the default config).
|
||||
WRAPPER_TO_LIGHTX2V_FIELDS: Dict[str, str] = {
|
||||
# Model selection — names match.
|
||||
"model_cls": "model_cls",
|
||||
"model_path": "model_path",
|
||||
"task": "task",
|
||||
# Inference loop.
|
||||
"infer_steps": "infer_steps",
|
||||
"seed": "seed",
|
||||
"sample_shift": "sample_shift",
|
||||
# Output shape — wrapper uses bare names, lightx2v prefixes with target_.
|
||||
"height": "target_height",
|
||||
"width": "target_width",
|
||||
"video_length": "target_video_length",
|
||||
"fps": "target_fps",
|
||||
"video_duration": "video_duration",
|
||||
# Image preprocessing.
|
||||
"resize_mode": "resize_mode",
|
||||
"fixed_area": "fixed_area",
|
||||
# Sekotalk-specific.
|
||||
"prev_frame_length": "prev_frame_length",
|
||||
# Distillation.
|
||||
"denoising_step_list": "denoising_step_list",
|
||||
"use_31_block": "use_31_block",
|
||||
}
|
||||
|
||||
# Attention type fans out to three internal slots in lightx2v.
|
||||
_ATTN_TYPE_SLOTS = ("self_attn_1_type", "cross_attn_1_type", "cross_attn_2_type")
|
||||
|
||||
|
||||
def apply_inference_config(config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Translate inference widget values to a partial lightx2v config dict."""
|
||||
updates: Dict[str, Any] = {}
|
||||
|
||||
# Bulk rename via the table.
|
||||
for wrapper_key, lightx2v_key in WRAPPER_TO_LIGHTX2V_FIELDS.items():
|
||||
if wrapper_key not in config:
|
||||
continue
|
||||
# seed=-1 means "use lightx2v's default / random"; leave it out.
|
||||
if wrapper_key == "seed" and config[wrapper_key] == -1:
|
||||
continue
|
||||
updates[lightx2v_key] = config[wrapper_key]
|
||||
|
||||
# cfg_scale -> sample_guide_scale (and toggle enable_cfg).
|
||||
if "cfg_scale" in config:
|
||||
updates["sample_guide_scale"] = config["cfg_scale"]
|
||||
updates["enable_cfg"] = config["cfg_scale"] != 1.0
|
||||
|
||||
# Wan2.2 MoE has two CFG scales (high/low noise) and a boundary param.
|
||||
model_cls = config.get("model_cls", "")
|
||||
if "wan2.2_moe" in model_cls:
|
||||
updates["boundary"] = 0.9
|
||||
updates["sample_guide_scale"] = [config.get("cfg_scale"), config.get("cfg_scale2")]
|
||||
if "wan2.2" in model_cls:
|
||||
updates["use_image_encoder"] = False
|
||||
|
||||
# One widget value drives three lightx2v attention slots.
|
||||
attention_type = config.get("attention_type", LightX2VDefaultConfig.DEFAULT_ATTENTION_TYPE)
|
||||
for slot in _ATTN_TYPE_SLOTS:
|
||||
updates[slot] = attention_type
|
||||
|
||||
# TAEW2.1 lightweight VAE lives next to the model.
|
||||
if config.get("use_tiny_vae", False):
|
||||
updates["use_tiny_vae"] = True
|
||||
updates["tiny_vae"] = True
|
||||
updates["tiny_vae_path"] = os.path.join(config["model_path"], "taew2_1.pth")
|
||||
|
||||
return updates
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Translate ``LightX2VMemoryOptimization`` widget values into lightx2v config keys.
|
||||
|
||||
Several toggles only matter when their parent is enabled (e.g. ``offload_granularity``
|
||||
only when ``cpu_offload=True``). Those nested keys are written only on the
|
||||
true-branch to keep the resulting config dict minimal.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict
|
||||
|
||||
# Direct rename: wrapper key -> lightx2v key.
|
||||
WRAPPER_TO_LIGHTX2V_FIELDS: Dict[str, str] = {
|
||||
"enable_rotary_chunk": "rotary_chunk",
|
||||
"clean_cuda_cache": "clean_cuda_cache",
|
||||
"cpu_offload": "cpu_offload",
|
||||
"t5_cpu_offload": "t5_cpu_offload",
|
||||
"vae_cpu_offload": "vae_cpu_offload",
|
||||
"audio_encoder_cpu_offload": "audio_encoder_cpu_offload",
|
||||
"audio_adapter_cpu_offload": "audio_adapter_cpu_offload",
|
||||
"lazy_load": "lazy_load",
|
||||
"unload_after_inference": "unload_modules",
|
||||
"use_tiling_vae": "use_tiling_vae",
|
||||
}
|
||||
|
||||
|
||||
def apply_memory_optimization(config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Translate memory-optimization widget values."""
|
||||
updates: Dict[str, Any] = {}
|
||||
|
||||
# NOTE: legacy behavior — when a specific offload key is missing, fall back
|
||||
# to the global ``cpu_offload`` flag. This means if the user only sets
|
||||
# ``cpu_offload=True``, every sub-offload (T5/VAE/audio…) silently follows.
|
||||
# Preserved as-is for backward compat; revisit when audio_* offloads
|
||||
# become widget-exposed everywhere.
|
||||
global_cpu_offload = config.get("cpu_offload", False)
|
||||
for wrapper_key, lightx2v_key in WRAPPER_TO_LIGHTX2V_FIELDS.items():
|
||||
updates[lightx2v_key] = config.get(wrapper_key, global_cpu_offload)
|
||||
|
||||
if updates.get("rotary_chunk"):
|
||||
updates["rotary_chunk_size"] = config.get("rotary_chunk_size", 100)
|
||||
|
||||
if updates.get("cpu_offload"):
|
||||
updates["offload_granularity"] = config.get("offload_granularity", "phase")
|
||||
updates["offload_ratio"] = config.get("offload_ratio", 1.0)
|
||||
|
||||
if updates.get("t5_cpu_offload"):
|
||||
updates["t5_offload_granularity"] = config.get("t5_offload_granularity", "model")
|
||||
|
||||
return updates
|
||||
@@ -0,0 +1,111 @@
|
||||
"""Orchestrate per-feature translators into a final lightx2v config dict.
|
||||
|
||||
Flow:
|
||||
1. start from ``LightX2VDefaultConfig.DEFAULT_CONFIG``
|
||||
2. apply inference / memory / teacache / quantization translators in order
|
||||
(teacache runs after inference so it can see the resolved task/resolution)
|
||||
3. attach LoRA chain and talk_objects (no rename needed)
|
||||
4. shallow-merge the model's own ``config.json`` for keys still unset
|
||||
(lightx2v's ``set_config`` will further read its own model config later)
|
||||
5. wrap as ``EasyDict`` so consumers can use attribute access
|
||||
|
||||
NOTE: ``input_info`` (the per-call dataclass in ``lightx2v.utils.input_info``)
|
||||
is NOT built here. The inference node constructs it dynamically from this
|
||||
config plus the runtime image/audio paths, because lightx2v itself distinguishes
|
||||
"persistent config" from "per-call input_info".
|
||||
"""
|
||||
|
||||
import copy
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Dict
|
||||
|
||||
from easydict import EasyDict
|
||||
|
||||
from ..capability import get_available_attn_ops, get_available_quant_ops
|
||||
from ..defaults import LightX2VDefaultConfig
|
||||
from .inference import apply_inference_config
|
||||
from .memory import apply_memory_optimization
|
||||
from .quant import apply_quantization_config
|
||||
from .teacache import apply_teacache_config
|
||||
|
||||
|
||||
class ModularConfigManager:
|
||||
"""Compose translators into a final lightx2v config."""
|
||||
|
||||
def __init__(self):
|
||||
self.base_config = copy.deepcopy(LightX2VDefaultConfig.DEFAULT_CONFIG)
|
||||
self._available_attn_ops = None
|
||||
self._available_quant_ops = None
|
||||
|
||||
@staticmethod
|
||||
def _filter_available(ops_list, fallback=None):
|
||||
available = [name for name, ok in ops_list if ok]
|
||||
if fallback and fallback not in available:
|
||||
available.append(fallback)
|
||||
return available
|
||||
|
||||
@property
|
||||
def available_attention_types(self):
|
||||
if self._available_attn_ops is None:
|
||||
self._available_attn_ops = get_available_attn_ops()
|
||||
return self._filter_available(self._available_attn_ops, "torch_sdpa")
|
||||
|
||||
@property
|
||||
def available_quant_schemes(self):
|
||||
if self._available_quant_ops is None:
|
||||
self._available_quant_ops = get_available_quant_ops()
|
||||
return self._filter_available(self._available_quant_ops)
|
||||
|
||||
# Exposed for tests/debugging; the public entrypoint is build_final_config_from_combined.
|
||||
apply_inference_config = staticmethod(apply_inference_config)
|
||||
apply_teacache_config = staticmethod(apply_teacache_config)
|
||||
apply_quantization_config = staticmethod(apply_quantization_config)
|
||||
apply_memory_optimization = staticmethod(apply_memory_optimization)
|
||||
|
||||
@staticmethod
|
||||
def _load_model_config(model_path: str) -> Dict[str, Any]:
|
||||
config_path = os.path.join(model_path, "config.json")
|
||||
if not os.path.exists(config_path):
|
||||
return {}
|
||||
try:
|
||||
with open(config_path, "r") as f:
|
||||
return json.load(f)
|
||||
except Exception as e:
|
||||
logging.warning(f"Failed to load model config: {e}")
|
||||
return {}
|
||||
|
||||
def build_final_config_from_combined(self, combined_config) -> EasyDict:
|
||||
"""Build the final lightx2v config from a CombinedConfig dataclass."""
|
||||
final_config = copy.deepcopy(self.base_config)
|
||||
|
||||
if combined_config.inference:
|
||||
final_config.update(apply_inference_config(combined_config.inference.to_dict()))
|
||||
|
||||
if combined_config.memory:
|
||||
final_config.update(apply_memory_optimization(combined_config.memory.to_dict()))
|
||||
|
||||
# teacache reads the (already-resolved) task and resolution off final_config.
|
||||
if combined_config.teacache:
|
||||
final_config.update(apply_teacache_config(combined_config.teacache.to_dict(), final_config))
|
||||
|
||||
if combined_config.quantization:
|
||||
final_config.update(apply_quantization_config(combined_config.quantization.to_dict()))
|
||||
|
||||
if combined_config.lora_configs:
|
||||
final_config["lora_configs"] = [lora.to_dict() for lora in combined_config.lora_configs]
|
||||
|
||||
if combined_config.talk_objects:
|
||||
final_config.update(combined_config.talk_objects.to_dict())
|
||||
|
||||
# Shallow-merge the model's own config.json for keys still unset.
|
||||
# lightx2v's own set_config.auto_calc_config will do its own deeper
|
||||
# merge of model_path/config.json — this just gives translators a
|
||||
# chance to see model-side hints (e.g. text_len) when they run.
|
||||
model_config = self._load_model_config(final_config.get("model_path", ""))
|
||||
for key, value in model_config.items():
|
||||
if key not in final_config or final_config[key] is None:
|
||||
final_config[key] = value
|
||||
|
||||
return EasyDict(final_config)
|
||||
@@ -0,0 +1,25 @@
|
||||
"""Translate ``LightX2VQuantization`` widget values into lightx2v config keys.
|
||||
|
||||
Each of dit/t5/clip/adapter contributes two keys to lightx2v:
|
||||
``{component}_quantized`` (bool) and ``{component}_quant_scheme`` (str).
|
||||
``"Default"`` means "leave as-is".
|
||||
"""
|
||||
|
||||
from typing import Any, Dict
|
||||
|
||||
from ..defaults import LightX2VDefaultConfig
|
||||
|
||||
_COMPONENTS = ("dit", "t5", "clip", "adapter")
|
||||
|
||||
|
||||
def apply_quantization_config(config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Translate quantization widget values."""
|
||||
updates: Dict[str, Any] = {}
|
||||
defaults = LightX2VDefaultConfig.DEFAULT_QUANTIZATION_SCHEMES
|
||||
|
||||
for component in _COMPONENTS:
|
||||
scheme = config.get(f"{component}_quant_scheme", defaults[component])
|
||||
updates[f"{component}_quantized"] = scheme != "Default"
|
||||
updates[f"{component}_quant_scheme"] = scheme
|
||||
|
||||
return updates
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Translate ``LightX2VTeaCache`` widget values into lightx2v config keys.
|
||||
|
||||
Wrapper-side ``enable / threshold / use_ret_steps`` -> lightx2v-side
|
||||
``feature_caching / teacache_thresh / use_ret_steps`` + polynomial coefficients
|
||||
picked from the calibration table.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict
|
||||
|
||||
from ..teacache_coeffs import CoefficientCalculator
|
||||
|
||||
|
||||
def apply_teacache_config(config: Dict[str, Any], model_info: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Translate TeaCache widget values.
|
||||
|
||||
``model_info`` is the partially-built lightx2v config so we can pick
|
||||
coefficients matched to the actual task and output resolution.
|
||||
"""
|
||||
if not config.get("enable", False):
|
||||
return {"feature_caching": "NoCaching"}
|
||||
|
||||
use_ret_steps = config.get("use_ret_steps", False)
|
||||
task = model_info.get("task", "t2v")
|
||||
model_size = "14b" if "14b" in model_info.get("model_cls", "") else "1.3b"
|
||||
resolution = (
|
||||
model_info.get("target_width", 832),
|
||||
model_info.get("target_height", 480),
|
||||
)
|
||||
|
||||
return {
|
||||
"feature_caching": "Tea",
|
||||
"teacache_thresh": config.get("threshold", 0.26),
|
||||
"use_ret_steps": use_ret_steps,
|
||||
"coefficients": CoefficientCalculator.get_coefficients(task, model_size, resolution, use_ret_steps),
|
||||
}
|
||||
@@ -0,0 +1,369 @@
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import uuid
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
from easydict import EasyDict
|
||||
|
||||
from .bridge import (
|
||||
ModularConfigManager,
|
||||
)
|
||||
from .data_models import (
|
||||
CombinedConfig,
|
||||
InferenceConfig,
|
||||
LoRAConfig,
|
||||
MemoryOptimizationConfig,
|
||||
QuantizationConfig,
|
||||
TalkObject,
|
||||
TalkObjectsConfig,
|
||||
TeaCacheConfig,
|
||||
)
|
||||
from .file_handlers import (
|
||||
AudioFileHandler,
|
||||
ComfyUIFileResolver,
|
||||
MaskFileHandler,
|
||||
)
|
||||
from .model_utils import get_lora_full_path, get_model_full_path
|
||||
|
||||
|
||||
class ConfigValidator:
|
||||
"""Validator for configuration parameters."""
|
||||
|
||||
@staticmethod
|
||||
def validate_inference_config(config: InferenceConfig) -> InferenceConfig:
|
||||
"""Validate and adjust inference configuration."""
|
||||
# Ensure video length is valid
|
||||
if config.video_length < 16:
|
||||
logging.warning("Video length is too short, setting to 16")
|
||||
config.video_length = 16
|
||||
|
||||
# Adjust video length to be compatible with model requirements
|
||||
remainder = (config.video_length - 1) % 4
|
||||
if remainder != 0:
|
||||
config.video_length = config.video_length + (4 - remainder)
|
||||
|
||||
# Set FPS based on model class
|
||||
if config.model_cls == "hunyuan":
|
||||
config.fps = 24
|
||||
else:
|
||||
config.fps = 16
|
||||
|
||||
# Handle random seed
|
||||
if config.seed == -1:
|
||||
config.seed = np.random.randint(0, 2**32 - 1)
|
||||
|
||||
return config
|
||||
|
||||
@staticmethod
|
||||
def validate_dimensions(height: int, width: int) -> Tuple[int, int]:
|
||||
"""Validate and adjust video dimensions."""
|
||||
# Ensure dimensions are multiples of 8
|
||||
height = (height // 8) * 8
|
||||
width = (width // 8) * 8
|
||||
|
||||
# Ensure minimum dimensions
|
||||
height = max(64, height)
|
||||
width = max(64, width)
|
||||
|
||||
# Ensure maximum dimensions
|
||||
height = min(2048, height)
|
||||
width = min(2048, width)
|
||||
|
||||
return height, width
|
||||
|
||||
|
||||
class InferenceConfigBuilder:
|
||||
"""Builder for inference configuration."""
|
||||
|
||||
def __init__(self):
|
||||
self.validator = ConfigValidator()
|
||||
|
||||
def build(
|
||||
self,
|
||||
model_cls: str,
|
||||
model_name: str,
|
||||
task: str,
|
||||
infer_steps: int,
|
||||
seed: int,
|
||||
cfg_scale: float,
|
||||
cfg_scale2: float,
|
||||
sample_shift: int,
|
||||
height: int,
|
||||
width: int,
|
||||
duration: float,
|
||||
attention_type: str,
|
||||
**optional_params,
|
||||
) -> InferenceConfig:
|
||||
"""Build inference configuration from parameters."""
|
||||
# Get model path
|
||||
model_path = get_model_full_path(model_name)
|
||||
|
||||
# Calculate video length from duration
|
||||
fps = 24 if model_cls == "hunyuan" else 16
|
||||
video_length = int(round(duration * fps))
|
||||
|
||||
# Validate dimensions
|
||||
height, width = self.validator.validate_dimensions(height, width)
|
||||
|
||||
# Create base config
|
||||
config = InferenceConfig(
|
||||
model_cls=model_cls,
|
||||
model_path=model_path,
|
||||
task=task,
|
||||
infer_steps=infer_steps,
|
||||
seed=seed,
|
||||
cfg_scale=cfg_scale,
|
||||
cfg_scale2=cfg_scale2,
|
||||
sample_shift=sample_shift,
|
||||
height=height,
|
||||
width=width,
|
||||
video_length=video_length,
|
||||
fps=fps,
|
||||
video_duration=duration,
|
||||
attention_type=attention_type,
|
||||
)
|
||||
|
||||
# Handle optional parameters
|
||||
self._apply_optional_params(config, optional_params)
|
||||
|
||||
# Validate final config
|
||||
config = self.validator.validate_inference_config(config)
|
||||
|
||||
return config
|
||||
|
||||
def _apply_optional_params(self, config: InferenceConfig, optional_params: Dict[str, Any]):
|
||||
"""Apply optional parameters to config."""
|
||||
# Handle denoising steps
|
||||
if "denoising_steps" in optional_params:
|
||||
steps_str = optional_params["denoising_steps"]
|
||||
if steps_str and steps_str.strip():
|
||||
try:
|
||||
steps_list = [float(s.strip()) for s in steps_str.split(",")]
|
||||
config.denoising_step_list = steps_list
|
||||
config.infer_steps = len(steps_list)
|
||||
except ValueError:
|
||||
logging.warning(f"Invalid denoising steps: {steps_str}")
|
||||
|
||||
# Handle other optional params
|
||||
for param in [
|
||||
"resize_mode",
|
||||
"fixed_area",
|
||||
"segment_length",
|
||||
"prev_frame_length",
|
||||
"use_tiny_vae",
|
||||
]:
|
||||
if param in optional_params:
|
||||
setattr(config, param, optional_params[param])
|
||||
|
||||
# Special handling for seko models
|
||||
if "seko" in config.model_cls:
|
||||
config.video_length = optional_params.get("segment_length", 81)
|
||||
config.use_31_block = False
|
||||
if "2.5" in config.model_path:
|
||||
config.use_31_block = True
|
||||
if "rs2v" in config.task:
|
||||
config.use_31_block = True
|
||||
if "prev_frame_length" in optional_params:
|
||||
config.prev_frame_length = optional_params["prev_frame_length"]
|
||||
|
||||
|
||||
class TalkObjectConfigBuilder:
|
||||
"""Builder for talk object configurations."""
|
||||
|
||||
def __init__(self):
|
||||
self.audio_handler = AudioFileHandler()
|
||||
self.mask_handler = MaskFileHandler()
|
||||
self.resolver = ComfyUIFileResolver()
|
||||
|
||||
def build_from_input(
|
||||
self,
|
||||
name: str,
|
||||
audio: Optional[Any] = None,
|
||||
mask: Optional[Any] = None,
|
||||
save_to_input: bool = True,
|
||||
) -> TalkObject:
|
||||
if audio is None:
|
||||
return None
|
||||
|
||||
talk_object = TalkObject(name=name)
|
||||
|
||||
if save_to_input and audio is not None:
|
||||
audio_path = self._save_audio_to_input(name, audio)
|
||||
if audio_path:
|
||||
talk_object.audio = audio_path
|
||||
|
||||
else:
|
||||
talk_object.audio = audio
|
||||
|
||||
if mask is not None:
|
||||
if save_to_input:
|
||||
mask_path = self._save_mask_to_input(name, mask)
|
||||
if mask_path:
|
||||
talk_object.mask = mask_path
|
||||
else:
|
||||
talk_object.mask = mask
|
||||
|
||||
return talk_object
|
||||
|
||||
def build_from_json(self, json_config: str) -> Optional[TalkObjectsConfig]:
|
||||
"""Build talk objects configuration from JSON."""
|
||||
try:
|
||||
objects_data = json.loads(json_config)
|
||||
if not isinstance(objects_data, list):
|
||||
logging.error("JSON config must be a list")
|
||||
return None
|
||||
|
||||
config = TalkObjectsConfig()
|
||||
|
||||
for obj_data in objects_data:
|
||||
if not isinstance(obj_data, dict) or "audio" not in obj_data:
|
||||
continue
|
||||
|
||||
talk_obj = TalkObject(
|
||||
name=obj_data.get("name", "unknown"),
|
||||
audio=obj_data["audio"],
|
||||
mask=obj_data.get("mask"),
|
||||
)
|
||||
config.add_object(talk_obj)
|
||||
|
||||
return config if config.talk_objects else None
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
logging.error(f"Failed to parse JSON: {e}")
|
||||
|
||||
def build_from_files(self, audio_files: str, mask_files: str = "", names: str = "") -> Optional[TalkObjectsConfig]:
|
||||
"""Build talk objects configuration from file lists."""
|
||||
audio_list = [f.strip() for f in audio_files.split("\n") if f.strip()]
|
||||
if not audio_list:
|
||||
return None
|
||||
|
||||
mask_list = [f.strip() for f in mask_files.split("\n") if f.strip()] if mask_files else []
|
||||
name_list = [n.strip() for n in names.split("\n") if n.strip()] if names else []
|
||||
|
||||
config = TalkObjectsConfig()
|
||||
|
||||
for i, audio_file in enumerate(audio_list):
|
||||
talk_obj = TalkObject(
|
||||
name=name_list[i] if i < len(name_list) else f"person_{i + 1}",
|
||||
audio=audio_file,
|
||||
mask=mask_list[i] if i < len(mask_list) else None,
|
||||
)
|
||||
config.add_object(talk_obj)
|
||||
|
||||
return config
|
||||
|
||||
def _save_audio_to_input(self, name: str, audio_data: Any) -> Optional[str]:
|
||||
try:
|
||||
filename = f"{name}_audio_{uuid.uuid4().hex[:8]}.wav"
|
||||
return self.resolver.save_to_input(audio_data, filename, self.audio_handler)
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to save audio: {e}")
|
||||
return None
|
||||
|
||||
def _save_mask_to_input(self, name: str, mask_data: Any) -> Optional[str]:
|
||||
try:
|
||||
filename = f"{name}_mask_{uuid.uuid4().hex[:8]}.png"
|
||||
return self.resolver.save_to_input(mask_data, filename, self.mask_handler)
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to save mask: {e}")
|
||||
return None
|
||||
|
||||
|
||||
class ConfigBuilder:
|
||||
"""Main configuration builder that combines all configs."""
|
||||
|
||||
def __init__(self):
|
||||
self.manager = ModularConfigManager()
|
||||
|
||||
def combine_configs(
|
||||
self,
|
||||
inference_config: InferenceConfig,
|
||||
teacache_config: Optional[TeaCacheConfig] = None,
|
||||
quantization_config: Optional[QuantizationConfig] = None,
|
||||
memory_config: Optional[MemoryOptimizationConfig] = None,
|
||||
lora_chain: Optional[List[Dict[str, Any]]] = None,
|
||||
talk_objects_config: Optional[TalkObjectsConfig] = None,
|
||||
) -> EasyDict:
|
||||
# Create combined configuration
|
||||
combined = CombinedConfig(
|
||||
inference=inference_config,
|
||||
teacache=teacache_config,
|
||||
quantization=quantization_config,
|
||||
memory=memory_config,
|
||||
talk_objects=talk_objects_config,
|
||||
)
|
||||
|
||||
# Process LoRA configs if provided
|
||||
if lora_chain:
|
||||
for lora_dict in lora_chain:
|
||||
lora_config = LoRAConfig(path=lora_dict["path"], strength=lora_dict.get("strength", 1.0))
|
||||
combined.lora_configs.append(lora_config)
|
||||
|
||||
# Build final configuration from combined config
|
||||
final_config = self.manager.build_final_config_from_combined(combined)
|
||||
|
||||
return final_config
|
||||
|
||||
# Keys that affect runner construction (and therefore require a reinit when they
|
||||
# change). Per-call fields like prompt / seed / infer_steps deliberately omitted.
|
||||
# (default, ...) tuples — first element is the value used when the field is absent.
|
||||
_HASH_FIELDS = (
|
||||
("model_cls", None),
|
||||
("model_path", None),
|
||||
("task", None),
|
||||
("t5_quantized", False),
|
||||
("clip_quantized", False),
|
||||
("lora_configs", None),
|
||||
("cross_attn_1_type", None),
|
||||
("cross_attn_2_type", None),
|
||||
("self_attn_1_type", None),
|
||||
("self_attn_2_type", None),
|
||||
("cpu_offload", False),
|
||||
("offload_granularity", None),
|
||||
("offload_ratio", None),
|
||||
("t5_cpu_offload", False),
|
||||
("t5_offload_granularity", None),
|
||||
("audio_encoder_cpu_offload", False),
|
||||
("audio_adapter_cpu_offload", False),
|
||||
("vae_cpu_offload", False),
|
||||
("use_tiling_vae", False),
|
||||
("unload_after_inference", False),
|
||||
("enable_rotary_chunk", False),
|
||||
("rotary_chunk_size", None),
|
||||
("clean_cuda_cache", False),
|
||||
("torch_compile", False),
|
||||
("threshold", None),
|
||||
("use_ret_steps", False),
|
||||
("t5_quant_scheme", None),
|
||||
("clip_quant_scheme", None),
|
||||
("adapter_quant_scheme", None),
|
||||
("adapter_quantized", False),
|
||||
("feature_caching", None),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_config_hash(config) -> str:
|
||||
"""Hash the runner-construction-relevant config fields. Per-call fields are excluded."""
|
||||
relevant = {k: getattr(config, k, default) for k, default in ConfigBuilder._HASH_FIELDS}
|
||||
return hashlib.md5(json.dumps(relevant, sort_keys=True).encode()).hexdigest()
|
||||
|
||||
|
||||
class LoRAChainBuilder:
|
||||
"""Builder for LoRA chain configurations."""
|
||||
|
||||
@staticmethod
|
||||
def build_chain(lora_name: str, strength: float, existing_chain: Optional[List[Dict]] = None) -> List[Dict]:
|
||||
"""Build or extend a LoRA chain."""
|
||||
if existing_chain is None:
|
||||
chain = []
|
||||
else:
|
||||
chain = existing_chain.copy()
|
||||
|
||||
lora_path = get_lora_full_path(lora_name)
|
||||
if lora_path:
|
||||
lora_config = {"path": lora_path, "strength": strength}
|
||||
chain.append(lora_config)
|
||||
|
||||
return chain
|
||||
+193
@@ -0,0 +1,193 @@
|
||||
"""Data models for LightX2V ComfyUI wrapper."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
|
||||
@dataclass
|
||||
class TalkObject:
|
||||
"""Single talk object containing audio and optional mask."""
|
||||
|
||||
name: str
|
||||
audio: Optional[Union[str, Dict[str, Any], torch.Tensor, np.ndarray]] = None
|
||||
mask: Optional[Union[str, torch.Tensor, np.ndarray]] = None
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert to dictionary for pipeline."""
|
||||
result = {"name": self.name}
|
||||
|
||||
if isinstance(self.audio, str):
|
||||
result["audio"] = self.audio
|
||||
elif self.audio is not None:
|
||||
result["audio_data"] = self.audio
|
||||
|
||||
if isinstance(self.mask, str):
|
||||
result["mask"] = self.mask
|
||||
elif self.mask is not None:
|
||||
result["mask_data"] = self.mask
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@dataclass
|
||||
class InferenceConfig:
|
||||
"""Basic inference configuration."""
|
||||
|
||||
model_cls: str = "wan2.1"
|
||||
model_path: str = ""
|
||||
task: str = "i2v"
|
||||
infer_steps: int = 4
|
||||
seed: int = 42
|
||||
cfg_scale: float = 5.0
|
||||
cfg_scale2: float = 5.0
|
||||
sample_shift: int = 5
|
||||
height: int = 1280
|
||||
width: int = 720
|
||||
video_length: int = 81
|
||||
fps: int = 16
|
||||
video_duration: float = 5.0
|
||||
attention_type: str = "torch_sdpa"
|
||||
use_31_block: bool = True
|
||||
|
||||
# Optional parameters
|
||||
denoising_step_list: Optional[List[float]] = None
|
||||
resize_mode: str = "adaptive"
|
||||
fixed_area: str = "720p"
|
||||
segment_length: int = 81
|
||||
prev_frame_length: int = 5
|
||||
use_tiny_vae: bool = False
|
||||
|
||||
# Runtime parameters
|
||||
prompt: str = ""
|
||||
negative_prompt: str = ""
|
||||
image_path: Optional[str] = None
|
||||
audio_path: Optional[str] = None
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert to dictionary, excluding None values."""
|
||||
result = {}
|
||||
for key, value in self.__dict__.items():
|
||||
if value is not None:
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
@dataclass
|
||||
class TeaCacheConfig:
|
||||
"""TeaCache configuration."""
|
||||
|
||||
enable: bool = False
|
||||
threshold: float = 0.26
|
||||
use_ret_steps: bool = False
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"enable": self.enable,
|
||||
"threshold": self.threshold,
|
||||
"use_ret_steps": self.use_ret_steps,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class QuantizationConfig:
|
||||
"""Quantization configuration."""
|
||||
|
||||
dit_quant_scheme: str = "Default"
|
||||
t5_quant_scheme: str = "Default"
|
||||
clip_quant_scheme: str = "Default"
|
||||
adapter_quant_scheme: str = "Default"
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"dit_quant_scheme": self.dit_quant_scheme,
|
||||
"t5_quant_scheme": self.t5_quant_scheme,
|
||||
"clip_quant_scheme": self.clip_quant_scheme,
|
||||
"adapter_quant_scheme": self.adapter_quant_scheme,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class MemoryOptimizationConfig:
|
||||
"""Memory optimization configuration."""
|
||||
|
||||
enable_rotary_chunk: bool = False
|
||||
rotary_chunk_size: int = 100
|
||||
clean_cuda_cache: bool = False
|
||||
cpu_offload: bool = True
|
||||
offload_granularity: str = "block"
|
||||
offload_ratio: float = 1.0
|
||||
t5_cpu_offload: bool = True
|
||||
t5_offload_granularity: str = "model"
|
||||
audio_encoder_cpu_offload: bool = True
|
||||
audio_adapter_cpu_offload: bool = True
|
||||
vae_cpu_offload: bool = True
|
||||
use_tiling_vae: bool = True
|
||||
lazy_load: bool = False
|
||||
unload_after_inference: bool = False
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return self.__dict__.copy()
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoRAConfig:
|
||||
"""LoRA configuration."""
|
||||
|
||||
path: str
|
||||
strength: float = 1.0
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {"path": self.path, "strength": self.strength}
|
||||
|
||||
|
||||
@dataclass
|
||||
class TalkObjectsConfig:
|
||||
talk_objects: List[TalkObject] = field(default_factory=list)
|
||||
|
||||
def add_object(self, talk_object: TalkObject):
|
||||
self.talk_objects.append(talk_object)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {"talk_objects": [obj.to_dict() for obj in self.talk_objects]}
|
||||
|
||||
def to_list(self) -> List[Dict[str, Any]]:
|
||||
return [obj.to_dict() for obj in self.talk_objects]
|
||||
|
||||
|
||||
@dataclass
|
||||
class CombinedConfig:
|
||||
"""Combined configuration for all modules."""
|
||||
|
||||
inference: Optional[InferenceConfig] = None
|
||||
teacache: Optional[TeaCacheConfig] = None
|
||||
quantization: Optional[QuantizationConfig] = None
|
||||
memory: Optional[MemoryOptimizationConfig] = None
|
||||
lora_configs: List[LoRAConfig] = field(default_factory=list)
|
||||
talk_objects: Optional[TalkObjectsConfig] = None
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert to dictionary for pipeline."""
|
||||
result = {}
|
||||
|
||||
if self.inference:
|
||||
result.update(self.inference.to_dict())
|
||||
|
||||
if self.teacache:
|
||||
result["teacache"] = self.teacache.to_dict()
|
||||
|
||||
if self.quantization:
|
||||
result["quantization"] = self.quantization.to_dict()
|
||||
|
||||
if self.memory:
|
||||
result["memory"] = self.memory.to_dict()
|
||||
|
||||
if self.lora_configs:
|
||||
result["lora_configs"] = [lora.to_dict() for lora in self.lora_configs]
|
||||
|
||||
if self.talk_objects:
|
||||
result["talk_objects"] = self.talk_objects.to_list()
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,94 @@
|
||||
{
|
||||
"1": {
|
||||
"inputs": {
|
||||
"video": "1954627330480766977_wan2-2.mp4",
|
||||
"force_rate": 0,
|
||||
"custom_width": 0,
|
||||
"custom_height": 0,
|
||||
"frame_load_cap": 0,
|
||||
"skip_first_frames": 0,
|
||||
"select_every_nth": 1,
|
||||
"format": "AnimateDiff"
|
||||
},
|
||||
"class_type": "VHS_LoadVideo",
|
||||
"_meta": {
|
||||
"title": "Load Video (Upload) 🎥🅥🅗🅢"
|
||||
}
|
||||
},
|
||||
"4": {
|
||||
"inputs": {
|
||||
"ckpt_name": "seedvr2_ema_3b_fp8.safetensors",
|
||||
"precision": "fp8-sgl",
|
||||
"cpu_offload": false,
|
||||
"use_tiling_vae": true
|
||||
},
|
||||
"class_type": "LightX2VSeedVR2Loader",
|
||||
"_meta": {
|
||||
"title": "LightX2V SeedVR2 Loader"
|
||||
}
|
||||
},
|
||||
"5": {
|
||||
"inputs": {
|
||||
"target_height": 1920,
|
||||
"target_width": 1080,
|
||||
"infer_steps": 1,
|
||||
"segment_length": 81,
|
||||
"segment_overlap": 1,
|
||||
"seed": 3816942657,
|
||||
"source_fps": 16,
|
||||
"model": [
|
||||
"4",
|
||||
0
|
||||
],
|
||||
"images": [
|
||||
"1",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "LightX2VSeedVR2Sampler",
|
||||
"_meta": {
|
||||
"title": "LightX2V SeedVR2 Sampler"
|
||||
}
|
||||
},
|
||||
"6": {
|
||||
"inputs": {
|
||||
"video_info": [
|
||||
"1",
|
||||
3
|
||||
]
|
||||
},
|
||||
"class_type": "VHS_VideoInfo",
|
||||
"_meta": {
|
||||
"title": "Video Info 🎥🅥🅗🅢"
|
||||
}
|
||||
},
|
||||
"7": {
|
||||
"inputs": {
|
||||
"frame_rate": [
|
||||
"6",
|
||||
0
|
||||
],
|
||||
"loop_count": 0,
|
||||
"filename_prefix": "AnimateDiff",
|
||||
"format": "video/h264-mp4",
|
||||
"pix_fmt": "yuv420p",
|
||||
"crf": 19,
|
||||
"save_metadata": true,
|
||||
"trim_to_audio": false,
|
||||
"pingpong": false,
|
||||
"save_output": true,
|
||||
"images": [
|
||||
"5",
|
||||
0
|
||||
],
|
||||
"audio": [
|
||||
"1",
|
||||
2
|
||||
]
|
||||
},
|
||||
"class_type": "VHS_VideoCombine",
|
||||
"_meta": {
|
||||
"title": "Video Combine 🎥🅥🅗🅢"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
{
|
||||
"11": {
|
||||
"inputs": {
|
||||
"audio": "12秒.mp3",
|
||||
"start_time": 0,
|
||||
"duration": 0
|
||||
},
|
||||
"class_type": "VHS_LoadAudioUpload",
|
||||
"_meta": {
|
||||
"title": "Load Audio (Upload)🎥🅥🅗🅢"
|
||||
}
|
||||
},
|
||||
"13": {
|
||||
"inputs": {
|
||||
"prompt": "The video feature the person is talking. 双手不动",
|
||||
"negative_prompt": "",
|
||||
"inference_config": [
|
||||
"14",
|
||||
0
|
||||
],
|
||||
"image": [
|
||||
"20",
|
||||
0
|
||||
],
|
||||
"audio": [
|
||||
"11",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "LightX2VConfigCombinerV2",
|
||||
"_meta": {
|
||||
"title": "LightX2V Config Combiner V2"
|
||||
}
|
||||
},
|
||||
"14": {
|
||||
"inputs": {
|
||||
"model_cls": "seko_talk",
|
||||
"model_name": "SekoTalk-v2.7_beta2-bf16-step4_temp",
|
||||
"task": "rs2v",
|
||||
"infer_steps": 4,
|
||||
"seed": 4221706066,
|
||||
"cfg_scale": 1,
|
||||
"cfg_scale2": 1,
|
||||
"sample_shift": 5,
|
||||
"height": 1280,
|
||||
"width": 720,
|
||||
"duration": 5,
|
||||
"attention_type": "sage_attn2",
|
||||
"denoising_steps": "",
|
||||
"resize_mode": "adaptive",
|
||||
"fixed_area": "480p",
|
||||
"segment_length": 81,
|
||||
"prev_frame_length": 5,
|
||||
"use_tiny_vae": false
|
||||
},
|
||||
"class_type": "LightX2VInferenceConfig",
|
||||
"_meta": {
|
||||
"title": "LightX2V Inference Config"
|
||||
}
|
||||
},
|
||||
"15": {
|
||||
"inputs": {
|
||||
"prepared_config": [
|
||||
"13",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "LightX2VModularInferenceV2",
|
||||
"_meta": {
|
||||
"title": "LightX2V Modular Inference V2"
|
||||
}
|
||||
},
|
||||
"18": {
|
||||
"inputs": {
|
||||
"frame_rate": [
|
||||
"22",
|
||||
0
|
||||
],
|
||||
"loop_count": 0,
|
||||
"filename_prefix": "vigen-15ebb023",
|
||||
"format": "video/h264-mp4",
|
||||
"pix_fmt": "yuv420p",
|
||||
"crf": 23,
|
||||
"save_metadata": false,
|
||||
"trim_to_audio": false,
|
||||
"pingpong": false,
|
||||
"save_output": false,
|
||||
"images": [
|
||||
"21",
|
||||
0
|
||||
],
|
||||
"audio": [
|
||||
"15",
|
||||
1
|
||||
]
|
||||
},
|
||||
"class_type": "VHS_VideoCombine",
|
||||
"_meta": {
|
||||
"title": "Video Combine 🎥🅥🅗🅢"
|
||||
}
|
||||
},
|
||||
"20": {
|
||||
"inputs": {
|
||||
"image": "00000.jpg"
|
||||
},
|
||||
"class_type": "LoadImage",
|
||||
"_meta": {
|
||||
"title": "Load Image"
|
||||
}
|
||||
},
|
||||
"21": {
|
||||
"inputs": {
|
||||
"source_fps": 16,
|
||||
"target_fps": [
|
||||
"22",
|
||||
0
|
||||
],
|
||||
"scale": 1,
|
||||
"model_name": "flownet.pkl",
|
||||
"batch_size": 8,
|
||||
"use_fp16": true,
|
||||
"images": [
|
||||
"15",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "RIFEInterpolation",
|
||||
"_meta": {
|
||||
"title": "RIFE Frame Interpolation"
|
||||
}
|
||||
},
|
||||
"22": {
|
||||
"inputs": {
|
||||
"value": 25.000000000000007
|
||||
},
|
||||
"class_type": "FloatConstant",
|
||||
"_meta": {
|
||||
"title": "target fps"
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,205 @@
|
||||
{
|
||||
"id": "9c99a548-579c-497a-82d0-ded65496927b",
|
||||
"revision": 0,
|
||||
"last_node_id": 4,
|
||||
"last_link_id": 6,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "LoadImage",
|
||||
"pos": [
|
||||
-168.5590531329889,
|
||||
-43.34150113146973
|
||||
],
|
||||
"size": [
|
||||
282.796875,
|
||||
364
|
||||
],
|
||||
"flags": {},
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "IMAGE",
|
||||
"type": "IMAGE",
|
||||
"links": [
|
||||
5
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "MASK",
|
||||
"type": "MASK",
|
||||
"links": null
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LoadImage"
|
||||
},
|
||||
"widgets_values": [
|
||||
"sign_name.jpg",
|
||||
"image"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "LightX2VSwiftVRLoader",
|
||||
"pos": [
|
||||
-185.95492085854562,
|
||||
362
|
||||
],
|
||||
"size": [
|
||||
560.40625,
|
||||
308.234375
|
||||
],
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "model",
|
||||
"type": "SWIFTVR_MODEL",
|
||||
"links": [
|
||||
4
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LightX2VSwiftVRLoader"
|
||||
},
|
||||
"widgets_values": [
|
||||
"SwiftVR_lightx2v",
|
||||
"flash_attn3",
|
||||
"flashinfer_rope",
|
||||
24,
|
||||
0,
|
||||
0,
|
||||
false
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "LightX2VSwiftVRSampler",
|
||||
"pos": [
|
||||
428.4034139883438,
|
||||
68.58151792839192
|
||||
],
|
||||
"size": [
|
||||
399.609375,
|
||||
473.265625
|
||||
],
|
||||
"flags": {},
|
||||
"order": 2,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "model",
|
||||
"type": "SWIFTVR_MODEL",
|
||||
"link": 4
|
||||
},
|
||||
{
|
||||
"name": "images",
|
||||
"type": "IMAGE",
|
||||
"link": 5
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "images",
|
||||
"type": "IMAGE",
|
||||
"links": [
|
||||
6
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "filename",
|
||||
"type": "STRING",
|
||||
"links": null
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LightX2VSwiftVRSampler"
|
||||
},
|
||||
"widgets_values": [
|
||||
1080,
|
||||
16,
|
||||
true,
|
||||
"lightx2v_swiftvr_api/SwiftVRImage",
|
||||
"libx264",
|
||||
90,
|
||||
"ultrafast"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "PreviewImage",
|
||||
"pos": [
|
||||
895.4953696545064,
|
||||
76.70654845736883
|
||||
],
|
||||
"size": [
|
||||
354.15625,
|
||||
375.734375
|
||||
],
|
||||
"flags": {},
|
||||
"order": 3,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "images",
|
||||
"type": "IMAGE",
|
||||
"link": 6
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"properties": {
|
||||
"Node name for S&R": "PreviewImage"
|
||||
},
|
||||
"widgets_values": []
|
||||
}
|
||||
],
|
||||
"links": [
|
||||
[
|
||||
4,
|
||||
2,
|
||||
0,
|
||||
3,
|
||||
0,
|
||||
"SWIFTVR_MODEL"
|
||||
],
|
||||
[
|
||||
5,
|
||||
1,
|
||||
0,
|
||||
3,
|
||||
1,
|
||||
"IMAGE"
|
||||
],
|
||||
[
|
||||
6,
|
||||
3,
|
||||
0,
|
||||
4,
|
||||
0,
|
||||
"IMAGE"
|
||||
]
|
||||
],
|
||||
"groups": [],
|
||||
"config": {},
|
||||
"extra": {
|
||||
"ds": {
|
||||
"scale": 0.9307738921015128,
|
||||
"offset": [
|
||||
645.6355492983848,
|
||||
379.57724180337755
|
||||
]
|
||||
},
|
||||
"frontendVersion": "1.45.15",
|
||||
"VHS_latentpreview": false,
|
||||
"VHS_latentpreviewrate": 0,
|
||||
"VHS_MetadataImage": true,
|
||||
"VHS_KeepIntermediate": true
|
||||
},
|
||||
"version": 0.4
|
||||
}
|
||||
@@ -0,0 +1,380 @@
|
||||
{
|
||||
"id": "2ac26f51-8427-4fd5-a63b-8c0c35716909",
|
||||
"revision": 0,
|
||||
"last_node_id": 6,
|
||||
"last_link_id": 10,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 2,
|
||||
"type": "LightX2VSwiftVRLoader",
|
||||
"pos": [
|
||||
57.786008992636425,
|
||||
120.61016525792084
|
||||
],
|
||||
"size": [
|
||||
293.84375,
|
||||
292
|
||||
],
|
||||
"flags": {},
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "model",
|
||||
"type": "SWIFTVR_MODEL",
|
||||
"links": [
|
||||
4
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LightX2VSwiftVRLoader"
|
||||
},
|
||||
"widgets_values": [
|
||||
"SwiftVR_lightx2v",
|
||||
"flash_attn3",
|
||||
"flashinfer_rope",
|
||||
24,
|
||||
0,
|
||||
0,
|
||||
false
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "LightX2VSwiftVRSampler",
|
||||
"pos": [
|
||||
410.84880341840596,
|
||||
119.34257282956952
|
||||
],
|
||||
"size": [
|
||||
337.3125,
|
||||
320
|
||||
],
|
||||
"flags": {},
|
||||
"order": 3,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "model",
|
||||
"type": "SWIFTVR_MODEL",
|
||||
"link": 4
|
||||
},
|
||||
{
|
||||
"name": "images",
|
||||
"type": "IMAGE",
|
||||
"link": 7
|
||||
},
|
||||
{
|
||||
"name": "source_fps",
|
||||
"type": "FLOAT",
|
||||
"widget": {
|
||||
"name": "source_fps"
|
||||
},
|
||||
"link": 9
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "images",
|
||||
"type": "IMAGE",
|
||||
"links": null
|
||||
},
|
||||
{
|
||||
"name": "filename",
|
||||
"type": "STRING",
|
||||
"links": [
|
||||
6
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LightX2VSwiftVRSampler"
|
||||
},
|
||||
"widgets_values": [
|
||||
1080,
|
||||
16,
|
||||
true,
|
||||
"lightx2v_swiftvr_api/SwiftVR",
|
||||
"libx265",
|
||||
60,
|
||||
"ultrafast"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "LightX2VOutputVideoPreview",
|
||||
"pos": [
|
||||
803.9540069651823,
|
||||
143.39082786445536
|
||||
],
|
||||
"size": [
|
||||
307.875,
|
||||
703.328125
|
||||
],
|
||||
"flags": {},
|
||||
"order": 4,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "filename",
|
||||
"type": "STRING",
|
||||
"link": 6
|
||||
},
|
||||
{
|
||||
"name": "audio",
|
||||
"shape": 7,
|
||||
"type": "AUDIO",
|
||||
"link": 10
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "filename",
|
||||
"type": "STRING",
|
||||
"links": null
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LightX2VOutputVideoPreview"
|
||||
},
|
||||
"widgets_values": [
|
||||
true,
|
||||
true
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "VHS_LoadVideo",
|
||||
"pos": [
|
||||
-554.2465629475984,
|
||||
172.59098534531842
|
||||
],
|
||||
"size": [
|
||||
260.421875,
|
||||
761.6875
|
||||
],
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "meta_batch",
|
||||
"shape": 7,
|
||||
"type": "VHS_BatchManager",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "vae",
|
||||
"shape": 7,
|
||||
"type": "VAE",
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "IMAGE",
|
||||
"type": "IMAGE",
|
||||
"links": [
|
||||
7
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "frame_count",
|
||||
"type": "INT",
|
||||
"links": null
|
||||
},
|
||||
{
|
||||
"name": "audio",
|
||||
"type": "AUDIO",
|
||||
"links": [
|
||||
10
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "video_info",
|
||||
"type": "VHS_VIDEOINFO",
|
||||
"links": [
|
||||
8
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "VHS_LoadVideo"
|
||||
},
|
||||
"widgets_values": {
|
||||
"video": "飞书20260721-124954.mp4",
|
||||
"force_rate": 0,
|
||||
"custom_width": 0,
|
||||
"custom_height": 0,
|
||||
"frame_load_cap": 0,
|
||||
"skip_first_frames": 0,
|
||||
"select_every_nth": 1,
|
||||
"format": "AnimateDiff",
|
||||
"videopreview": {
|
||||
"hidden": false,
|
||||
"paused": false,
|
||||
"params": {
|
||||
"filename": "飞书20260721-124954.mp4",
|
||||
"type": "input",
|
||||
"format": "video/mp4",
|
||||
"force_rate": 0,
|
||||
"custom_width": 0,
|
||||
"custom_height": 0,
|
||||
"frame_load_cap": 0,
|
||||
"skip_first_frames": 0,
|
||||
"select_every_nth": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "VHS_VideoInfo",
|
||||
"pos": [
|
||||
-245.90446161258,
|
||||
309.43756146966683
|
||||
],
|
||||
"size": [
|
||||
233.453125,
|
||||
288
|
||||
],
|
||||
"flags": {},
|
||||
"order": 2,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "video_info",
|
||||
"type": "VHS_VIDEOINFO",
|
||||
"link": 8
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "source_fps🟨",
|
||||
"type": "FLOAT",
|
||||
"links": [
|
||||
9
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "source_frame_count🟨",
|
||||
"type": "INT",
|
||||
"links": null
|
||||
},
|
||||
{
|
||||
"name": "source_duration🟨",
|
||||
"type": "FLOAT",
|
||||
"links": null
|
||||
},
|
||||
{
|
||||
"name": "source_width🟨",
|
||||
"type": "INT",
|
||||
"links": null
|
||||
},
|
||||
{
|
||||
"name": "source_height🟨",
|
||||
"type": "INT",
|
||||
"links": null
|
||||
},
|
||||
{
|
||||
"name": "loaded_fps🟦",
|
||||
"type": "FLOAT",
|
||||
"links": null
|
||||
},
|
||||
{
|
||||
"name": "loaded_frame_count🟦",
|
||||
"type": "INT",
|
||||
"links": null
|
||||
},
|
||||
{
|
||||
"name": "loaded_duration🟦",
|
||||
"type": "FLOAT",
|
||||
"links": null
|
||||
},
|
||||
{
|
||||
"name": "loaded_width🟦",
|
||||
"type": "INT",
|
||||
"links": null
|
||||
},
|
||||
{
|
||||
"name": "loaded_height🟦",
|
||||
"type": "INT",
|
||||
"links": null
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "VHS_VideoInfo"
|
||||
},
|
||||
"widgets_values": {}
|
||||
}
|
||||
],
|
||||
"links": [
|
||||
[
|
||||
4,
|
||||
2,
|
||||
0,
|
||||
3,
|
||||
0,
|
||||
"SWIFTVR_MODEL"
|
||||
],
|
||||
[
|
||||
6,
|
||||
3,
|
||||
1,
|
||||
4,
|
||||
0,
|
||||
"STRING"
|
||||
],
|
||||
[
|
||||
7,
|
||||
5,
|
||||
0,
|
||||
3,
|
||||
1,
|
||||
"IMAGE"
|
||||
],
|
||||
[
|
||||
8,
|
||||
5,
|
||||
3,
|
||||
6,
|
||||
0,
|
||||
"VHS_VIDEOINFO"
|
||||
],
|
||||
[
|
||||
9,
|
||||
6,
|
||||
0,
|
||||
3,
|
||||
2,
|
||||
"FLOAT"
|
||||
],
|
||||
[
|
||||
10,
|
||||
5,
|
||||
2,
|
||||
4,
|
||||
1,
|
||||
"AUDIO"
|
||||
]
|
||||
],
|
||||
"groups": [],
|
||||
"config": {},
|
||||
"extra": {
|
||||
"ds": {
|
||||
"scale": 0.8400831511043174,
|
||||
"offset": [
|
||||
766.3547310556437,
|
||||
389.36680475087843
|
||||
]
|
||||
},
|
||||
"frontendVersion": "1.45.15",
|
||||
"VHS_latentpreview": false,
|
||||
"VHS_latentpreviewrate": 0,
|
||||
"VHS_MetadataImage": true,
|
||||
"VHS_KeepIntermediate": true
|
||||
},
|
||||
"version": 0.4
|
||||
}
|
||||
@@ -1,22 +1,57 @@
|
||||
{
|
||||
"id": "8e881096-4f7b-4633-b34b-6b9c0f8aa093",
|
||||
"id": "00000000-0000-0000-0000-000000000000",
|
||||
"revision": 0,
|
||||
"last_node_id": 112,
|
||||
"last_link_id": 82,
|
||||
"last_node_id": 55,
|
||||
"last_link_id": 59,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 107,
|
||||
"id": 10,
|
||||
"type": "LightX2VQuantization",
|
||||
"pos": [
|
||||
-2115.796630859375,
|
||||
-282.90557861328125
|
||||
],
|
||||
"size": [
|
||||
290.09375,
|
||||
154
|
||||
],
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "quantization_config",
|
||||
"type": "QUANT_CONFIG",
|
||||
"links": [
|
||||
52
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LightX2VQuantization"
|
||||
},
|
||||
"widgets_values": [
|
||||
"sgl",
|
||||
"fp8",
|
||||
"fp8",
|
||||
"fp16",
|
||||
"fp8"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 19,
|
||||
"type": "LoadImage",
|
||||
"pos": [
|
||||
1073.2501220703125,
|
||||
-168.2064208984375
|
||||
-1714.1234130859375,
|
||||
-20.190126419067383
|
||||
],
|
||||
"size": [
|
||||
274.080078125,
|
||||
314.0000305175781
|
||||
],
|
||||
"flags": {},
|
||||
"order": 0,
|
||||
"order": 3,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
@@ -24,7 +59,7 @@
|
||||
"name": "IMAGE",
|
||||
"type": "IMAGE",
|
||||
"links": [
|
||||
79
|
||||
57
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -33,192 +68,34 @@
|
||||
"links": null
|
||||
}
|
||||
],
|
||||
"title": "加载图像",
|
||||
"properties": {
|
||||
"Node name for S&R": "LoadImage"
|
||||
},
|
||||
"widgets_values": [
|
||||
"00.jpg",
|
||||
"input (1).png",
|
||||
"image"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 109,
|
||||
"type": "LightX2VConfigCombiner",
|
||||
"pos": [
|
||||
1410.54541015625,
|
||||
-547.0989990234375
|
||||
],
|
||||
"size": [
|
||||
239.138671875,
|
||||
126
|
||||
],
|
||||
"flags": {},
|
||||
"order": 2,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "inference_config",
|
||||
"type": "INFERENCE_CONFIG",
|
||||
"link": 76
|
||||
},
|
||||
{
|
||||
"name": "teacache_config",
|
||||
"shape": 7,
|
||||
"type": "TEACACHE_CONFIG",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "quantization_config",
|
||||
"shape": 7,
|
||||
"type": "QUANT_CONFIG",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "memory_config",
|
||||
"shape": 7,
|
||||
"type": "MEMORY_CONFIG",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "vae_config",
|
||||
"shape": 7,
|
||||
"type": "VAE_CONFIG",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "lora_chain",
|
||||
"shape": 7,
|
||||
"type": "LORA_CHAIN",
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "combined_config",
|
||||
"type": "COMBINED_CONFIG",
|
||||
"links": [
|
||||
78,
|
||||
80
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LightX2VConfigCombiner"
|
||||
},
|
||||
"widgets_values": []
|
||||
},
|
||||
{
|
||||
"id": 105,
|
||||
"type": "LightX2VInferenceConfig",
|
||||
"pos": [
|
||||
1073.62646484375,
|
||||
-580.9854125976562
|
||||
],
|
||||
"size": [
|
||||
270,
|
||||
346
|
||||
],
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "inference_config",
|
||||
"type": "INFERENCE_CONFIG",
|
||||
"links": [
|
||||
76
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LightX2VInferenceConfig"
|
||||
},
|
||||
"widgets_values": [
|
||||
"wan2.1",
|
||||
"Wan2.1-I2V-14B-480P",
|
||||
"i2v",
|
||||
40,
|
||||
1822974886,
|
||||
"fixed",
|
||||
5,
|
||||
3,
|
||||
480,
|
||||
480,
|
||||
81,
|
||||
16,
|
||||
""
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 110,
|
||||
"type": "LightX2VModularInference",
|
||||
"pos": [
|
||||
1424.9375,
|
||||
-216.26104736328125
|
||||
],
|
||||
"size": [
|
||||
400,
|
||||
200
|
||||
],
|
||||
"flags": {},
|
||||
"order": 3,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "combined_config",
|
||||
"type": "COMBINED_CONFIG",
|
||||
"link": 78
|
||||
},
|
||||
{
|
||||
"name": "image",
|
||||
"shape": 7,
|
||||
"type": "IMAGE",
|
||||
"link": 79
|
||||
},
|
||||
{
|
||||
"name": "audio",
|
||||
"shape": 7,
|
||||
"type": "AUDIO",
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "images",
|
||||
"type": "IMAGE",
|
||||
"links": [
|
||||
81
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LightX2VModularInference"
|
||||
},
|
||||
"widgets_values": [
|
||||
"太空漫步,往前跑。 ",
|
||||
""
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 108,
|
||||
"id": 21,
|
||||
"type": "VHS_VideoCombine",
|
||||
"pos": [
|
||||
1859.1119384765625,
|
||||
-211.628662109375
|
||||
-1236.06640625,
|
||||
-257.72991943359375
|
||||
],
|
||||
"size": [
|
||||
220.5830078125,
|
||||
524.5830078125
|
||||
390.9510192871094,
|
||||
334
|
||||
],
|
||||
"flags": {},
|
||||
"order": 5,
|
||||
"order": 6,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "images",
|
||||
"type": "IMAGE",
|
||||
"link": 81
|
||||
"link": 56
|
||||
},
|
||||
{
|
||||
"name": "audio",
|
||||
@@ -252,119 +129,322 @@
|
||||
"widgets_values": {
|
||||
"frame_rate": 16,
|
||||
"loop_count": 0,
|
||||
"filename_prefix": "AnimateDiff",
|
||||
"filename_prefix": "vigen-15ebb023",
|
||||
"format": "video/h264-mp4",
|
||||
"pix_fmt": "yuv420p",
|
||||
"crf": 19,
|
||||
"save_metadata": true,
|
||||
"save_metadata": false,
|
||||
"trim_to_audio": false,
|
||||
"pingpong": false,
|
||||
"save_output": true,
|
||||
"videopreview": {
|
||||
"hidden": false,
|
||||
"paused": false,
|
||||
"params": {
|
||||
"filename": "AnimateDiff_00588.mp4",
|
||||
"filename": "vigen-15ebb023_00111.mp4",
|
||||
"subfolder": "",
|
||||
"type": "output",
|
||||
"format": "video/h264-mp4",
|
||||
"frame_rate": 16
|
||||
},
|
||||
"muted": false
|
||||
"frame_rate": 16,
|
||||
"workflow": "vigen-15ebb023_00111.png",
|
||||
"fullpath": "/mnt/aigc/users/gaopeng1/ComfyUI/output/vigen-15ebb023_00111.mp4"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 111,
|
||||
"type": "easy showAnything",
|
||||
"id": 23,
|
||||
"type": "LightX2VInferenceConfig",
|
||||
"pos": [
|
||||
1773.56689453125,
|
||||
-706.2327270507812
|
||||
-2410.2958984375,
|
||||
-347.2873229980469
|
||||
],
|
||||
"size": [
|
||||
624.5454711914062,
|
||||
358.7272644042969
|
||||
270,
|
||||
490
|
||||
],
|
||||
"flags": {
|
||||
"collapsed": true
|
||||
"flags": {},
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "inference_config",
|
||||
"type": "INFERENCE_CONFIG",
|
||||
"links": [
|
||||
51
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LightX2VInferenceConfig"
|
||||
},
|
||||
"widgets_values": [
|
||||
"wan2.2_moe_distill",
|
||||
"Wan2.2-I2V-A14B-distill-fp8",
|
||||
"i2v",
|
||||
4,
|
||||
222869388,
|
||||
"randomize",
|
||||
1,
|
||||
1,
|
||||
5,
|
||||
1280,
|
||||
720,
|
||||
5,
|
||||
"flash_attn3",
|
||||
"1000,750,500,250",
|
||||
"adaptive",
|
||||
"720p",
|
||||
81,
|
||||
5,
|
||||
false
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 51,
|
||||
"type": "LightX2VMemoryOptimization",
|
||||
"pos": [
|
||||
-2113.802490234375,
|
||||
-65.72712707519531
|
||||
],
|
||||
"size": [
|
||||
311.908203125,
|
||||
370
|
||||
],
|
||||
"flags": {},
|
||||
"order": 2,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "memory_config",
|
||||
"type": "MEMORY_CONFIG",
|
||||
"links": [
|
||||
53
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LightX2VMemoryOptimization"
|
||||
},
|
||||
"widgets_values": [
|
||||
false,
|
||||
100,
|
||||
false,
|
||||
true,
|
||||
"block",
|
||||
1,
|
||||
false,
|
||||
"model",
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 52,
|
||||
"type": "LightX2VConfigCombinerV2",
|
||||
"pos": [
|
||||
-1720.9281005859375,
|
||||
-361.9546813964844
|
||||
],
|
||||
"size": [
|
||||
400,
|
||||
282
|
||||
],
|
||||
"flags": {},
|
||||
"order": 4,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "anything",
|
||||
"name": "inference_config",
|
||||
"type": "INFERENCE_CONFIG",
|
||||
"link": 51
|
||||
},
|
||||
{
|
||||
"name": "teacache_config",
|
||||
"shape": 7,
|
||||
"type": "*",
|
||||
"link": 80
|
||||
"type": "TEACACHE_CONFIG",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "quantization_config",
|
||||
"shape": 7,
|
||||
"type": "QUANT_CONFIG",
|
||||
"link": 52
|
||||
},
|
||||
{
|
||||
"name": "memory_config",
|
||||
"shape": 7,
|
||||
"type": "MEMORY_CONFIG",
|
||||
"link": 53
|
||||
},
|
||||
{
|
||||
"name": "lora_chain",
|
||||
"shape": 7,
|
||||
"type": "LORA_CHAIN",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "talk_objects_config",
|
||||
"shape": 7,
|
||||
"type": "TALK_OBJECTS_CONFIG",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "image",
|
||||
"shape": 7,
|
||||
"type": "IMAGE",
|
||||
"link": 57
|
||||
},
|
||||
{
|
||||
"name": "audio",
|
||||
"shape": 7,
|
||||
"type": "AUDIO",
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "output",
|
||||
"type": "*",
|
||||
"name": "prepared_config",
|
||||
"type": "PREPARED_CONFIG",
|
||||
"links": [
|
||||
55
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LightX2VConfigCombinerV2"
|
||||
},
|
||||
"widgets_values": [
|
||||
"The cow feature the person is talking.",
|
||||
""
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 53,
|
||||
"type": "LightX2VModularInferenceV2",
|
||||
"pos": [
|
||||
-1232.5400390625,
|
||||
-354.8329772949219
|
||||
],
|
||||
"size": [
|
||||
234.46444702148438,
|
||||
46
|
||||
],
|
||||
"flags": {},
|
||||
"order": 5,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "prepared_config",
|
||||
"type": "PREPARED_CONFIG",
|
||||
"link": 55
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "images",
|
||||
"type": "IMAGE",
|
||||
"links": [
|
||||
56
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "audio",
|
||||
"type": "AUDIO",
|
||||
"links": null
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "easy showAnything"
|
||||
"Node name for S&R": "LightX2VModularInferenceV2"
|
||||
},
|
||||
"widgets_values": [
|
||||
"{\"model_cls\": \"wan2.1\", \"model_path\": \"/mnt/aigc/users/lijiaqi2/ComfyUI/models/lightx2v/Wan2.1-I2V-14B-480P\", \"task\": \"i2v\", \"mode\": \"infer\", \"infer_steps\": 40, \"seed\": 1822974886, \"sample_guide_scale\": 5.0, \"sample_shift\": 3, \"enable_cfg\": true, \"prompt\": \"\", \"negative_prompt\": \"\", \"target_height\": 480, \"target_width\": 480, \"target_video_length\": 81, \"fps\": 16, \"vae_stride\": [4, 8, 8], \"patch_size\": [1, 2, 2], \"feature_caching\": \"NoCaching\", \"teacache_thresh\": 0.26, \"coefficients\": null, \"use_ret_steps\": false, \"dit_quant_scheme\": \"bf16\", \"t5_quant_scheme\": \"bf16\", \"clip_quant_scheme\": \"fp16\", \"quant_op\": \"vllm\", \"precision_mode\": \"fp32\", \"dit_quantized_ckpt\": null, \"t5_quantized_ckpt\": null, \"clip_quantized_ckpt\": null, \"mm_config\": {\"mm_type\": \"Default\"}, \"rotary_chunk\": false, \"rotary_chunk_size\": 100, \"clean_cuda_cache\": false, \"torch_compile\": false, \"attention_type\": \"flash_attn3\", \"self_attn_1_type\": \"flash_attn3\", \"cross_attn_1_type\": \"flash_attn3\", \"cross_attn_2_type\": \"flash_attn3\", \"cpu_offload\": false, \"offload_granularity\": \"phase\", \"offload_ratio\": 1.0, \"t5_cpu_offload\": false, \"t5_offload_granularity\": \"model\", \"lazy_load\": false, \"unload_modules\": false, \"use_tiny_vae\": false, \"tiny_vae\": false, \"tiny_vae_path\": null, \"use_tiling_vae\": false, \"lora_path\": null, \"strength_model\": 1.0, \"do_mm_calib\": false, \"parallel_attn_type\": null, \"parallel_vae\": false, \"max_area\": false, \"use_prompt_enhancer\": false, \"text_len\": 512, \"_class_name\": \"WanModel\", \"_diffusers_version\": \"0.30.0\", \"dim\": 5120, \"eps\": 1e-06, \"ffn_dim\": 13824, \"freq_dim\": 256, \"in_dim\": 36, \"model_type\": \"i2v\", \"num_heads\": 40, \"num_layers\": 40, \"out_dim\": 16}"
|
||||
]
|
||||
"widgets_values": []
|
||||
}
|
||||
],
|
||||
"links": [
|
||||
[
|
||||
76,
|
||||
105,
|
||||
51,
|
||||
23,
|
||||
0,
|
||||
109,
|
||||
52,
|
||||
0,
|
||||
"INFERENCE_CONFIG"
|
||||
],
|
||||
[
|
||||
78,
|
||||
109,
|
||||
52,
|
||||
10,
|
||||
0,
|
||||
110,
|
||||
0,
|
||||
"COMBINED_CONFIG"
|
||||
52,
|
||||
2,
|
||||
"QUANT_CONFIG"
|
||||
],
|
||||
[
|
||||
79,
|
||||
107,
|
||||
53,
|
||||
51,
|
||||
0,
|
||||
52,
|
||||
3,
|
||||
"MEMORY_CONFIG"
|
||||
],
|
||||
[
|
||||
55,
|
||||
52,
|
||||
0,
|
||||
53,
|
||||
0,
|
||||
"PREPARED_CONFIG"
|
||||
],
|
||||
[
|
||||
56,
|
||||
53,
|
||||
0,
|
||||
21,
|
||||
0,
|
||||
110,
|
||||
1,
|
||||
"IMAGE"
|
||||
],
|
||||
[
|
||||
80,
|
||||
109,
|
||||
0,
|
||||
111,
|
||||
0,
|
||||
"*"
|
||||
],
|
||||
[
|
||||
81,
|
||||
110,
|
||||
0,
|
||||
108,
|
||||
57,
|
||||
19,
|
||||
0,
|
||||
52,
|
||||
6,
|
||||
"IMAGE"
|
||||
]
|
||||
],
|
||||
"groups": [],
|
||||
"groups": [
|
||||
{
|
||||
"id": 2,
|
||||
"title": "Infer",
|
||||
"bounding": [
|
||||
-2421.39599609375,
|
||||
-422.0833740234375,
|
||||
1652.6817626953125,
|
||||
893.2465209960938
|
||||
],
|
||||
"color": "#3f789e",
|
||||
"font_size": 24,
|
||||
"flags": {}
|
||||
}
|
||||
],
|
||||
"config": {},
|
||||
"extra": {
|
||||
"ds": {
|
||||
"scale": 1,
|
||||
"scale": 0.7513148009015777,
|
||||
"offset": [
|
||||
-697.2225740954825,
|
||||
708.7013290998841
|
||||
2717.6156091040034,
|
||||
642.5150759820887
|
||||
]
|
||||
},
|
||||
"frontendVersion": "1.19.9"
|
||||
"frontendVersion": "1.23.4",
|
||||
"VHS_latentpreview": false,
|
||||
"VHS_latentpreviewrate": 0,
|
||||
"VHS_MetadataImage": true,
|
||||
"VHS_KeepIntermediate": true
|
||||
},
|
||||
"version": 0.4
|
||||
}
|
||||
@@ -1,313 +0,0 @@
|
||||
{
|
||||
"id": "8e881096-4f7b-4633-b34b-6b9c0f8aa093",
|
||||
"revision": 0,
|
||||
"last_node_id": 112,
|
||||
"last_link_id": 82,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 109,
|
||||
"type": "LightX2VConfigCombiner",
|
||||
"pos": [1531.4547119140625, -521.6442260742188],
|
||||
"size": [239.138671875, 126],
|
||||
"flags": {},
|
||||
"order": 3,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "inference_config",
|
||||
"type": "INFERENCE_CONFIG",
|
||||
"link": 76
|
||||
},
|
||||
{
|
||||
"name": "teacache_config",
|
||||
"shape": 7,
|
||||
"type": "TEACACHE_CONFIG",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "quantization_config",
|
||||
"shape": 7,
|
||||
"type": "QUANT_CONFIG",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "memory_config",
|
||||
"shape": 7,
|
||||
"type": "MEMORY_CONFIG",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "vae_config",
|
||||
"shape": 7,
|
||||
"type": "VAE_CONFIG",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "lora_chain",
|
||||
"shape": 7,
|
||||
"type": "LORA_CHAIN",
|
||||
"link": 82
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "combined_config",
|
||||
"type": "COMBINED_CONFIG",
|
||||
"links": [78, 80]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LightX2VConfigCombiner"
|
||||
},
|
||||
"widgets_values": []
|
||||
},
|
||||
{
|
||||
"id": 111,
|
||||
"type": "easy showAnything",
|
||||
"pos": [1839.9300537109375, -494.4144287109375],
|
||||
"size": [624.5454711914062, 358.7272644042969],
|
||||
"flags": {
|
||||
"collapsed": true
|
||||
},
|
||||
"order": 5,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "anything",
|
||||
"shape": 7,
|
||||
"type": "*",
|
||||
"link": 80
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "output",
|
||||
"type": "*",
|
||||
"links": null
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "easy showAnything"
|
||||
},
|
||||
"widgets_values": [
|
||||
"{\"model_cls\": \"wan2.1\", \"model_path\": \"/mnt/aigc/users/lijiaqi2/ComfyUI/models/lightx2v/Wan2.1-I2V-14B-480P\", \"task\": \"i2v\", \"mode\": \"infer\", \"infer_steps\": 4, \"seed\": 1822974886, \"sample_guide_scale\": 1.0, \"sample_shift\": 8, \"enable_cfg\": false, \"prompt\": \"\", \"negative_prompt\": \"\", \"target_height\": 480, \"target_width\": 480, \"target_video_length\": 81, \"fps\": 16, \"vae_stride\": [4, 8, 8], \"patch_size\": [1, 2, 2], \"feature_caching\": \"NoCaching\", \"teacache_thresh\": 0.26, \"coefficients\": null, \"use_ret_steps\": false, \"dit_quant_scheme\": \"bf16\", \"t5_quant_scheme\": \"bf16\", \"clip_quant_scheme\": \"fp16\", \"quant_op\": \"vllm\", \"precision_mode\": \"fp32\", \"dit_quantized_ckpt\": null, \"t5_quantized_ckpt\": null, \"clip_quantized_ckpt\": null, \"mm_config\": {\"mm_type\": \"Default\"}, \"rotary_chunk\": false, \"rotary_chunk_size\": 100, \"clean_cuda_cache\": false, \"torch_compile\": false, \"attention_type\": \"flash_attn3\", \"self_attn_1_type\": \"flash_attn3\", \"cross_attn_1_type\": \"flash_attn3\", \"cross_attn_2_type\": \"flash_attn3\", \"cpu_offload\": false, \"offload_granularity\": \"phase\", \"offload_ratio\": 1.0, \"t5_cpu_offload\": false, \"t5_offload_granularity\": \"model\", \"lazy_load\": false, \"unload_modules\": false, \"use_tiny_vae\": false, \"tiny_vae\": false, \"tiny_vae_path\": null, \"use_tiling_vae\": false, \"lora_path\": null, \"strength_model\": 1.0, \"do_mm_calib\": false, \"parallel_attn_type\": null, \"parallel_vae\": false, \"max_area\": false, \"use_prompt_enhancer\": false, \"text_len\": 512, \"denoising_step_list\": [999, 750, 500, 250], \"_class_name\": \"WanModel\", \"_diffusers_version\": \"0.30.0\", \"dim\": 5120, \"eps\": 1e-06, \"ffn_dim\": 13824, \"freq_dim\": 256, \"in_dim\": 36, \"model_type\": \"i2v\", \"num_heads\": 40, \"num_layers\": 40, \"out_dim\": 16, \"lora_configs\": [{\"path\": \"/mnt/aigc/users/lijiaqi2/ComfyUI/models/lightx2v/loras/Wan21_T2V_14B_lightx2v_cfg_step_distill_lora_rank32.safetensors\", \"strength\": 1.0}]}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 105,
|
||||
"type": "LightX2VInferenceConfig",
|
||||
"pos": [1073.62646484375, -580.9854125976562],
|
||||
"size": [270, 346],
|
||||
"flags": {},
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "inference_config",
|
||||
"type": "INFERENCE_CONFIG",
|
||||
"links": [76]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LightX2VInferenceConfig"
|
||||
},
|
||||
"widgets_values": [
|
||||
"wan2.1",
|
||||
"Wan2.1-I2V-14B-480P",
|
||||
"i2v",
|
||||
4,
|
||||
1822974886,
|
||||
"fixed",
|
||||
1,
|
||||
8,
|
||||
480,
|
||||
480,
|
||||
81,
|
||||
16,
|
||||
"999, 750, 500, 250"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 112,
|
||||
"type": "LightX2VLoRALoader",
|
||||
"pos": [1083.3714599609375, -178.75082397460938],
|
||||
"size": [270, 82],
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "lora_chain",
|
||||
"shape": 7,
|
||||
"type": "LORA_CHAIN",
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "lora_chain",
|
||||
"type": "LORA_CHAIN",
|
||||
"links": [82]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LightX2VLoRALoader"
|
||||
},
|
||||
"widgets_values": [
|
||||
"Wan21_T2V_14B_lightx2v_cfg_step_distill_lora_rank32.safetensors",
|
||||
1
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 107,
|
||||
"type": "LoadImage",
|
||||
"pos": [1082.250244140625, -34.84281921386719],
|
||||
"size": [274.080078125, 314.0000305175781],
|
||||
"flags": {},
|
||||
"order": 2,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "IMAGE",
|
||||
"type": "IMAGE",
|
||||
"links": [79]
|
||||
},
|
||||
{
|
||||
"name": "MASK",
|
||||
"type": "MASK",
|
||||
"links": null
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LoadImage"
|
||||
},
|
||||
"widgets_values": ["00.jpg", "image"]
|
||||
},
|
||||
{
|
||||
"id": 110,
|
||||
"type": "LightX2VModularInference",
|
||||
"pos": [1479.8463134765625, -292.079345703125],
|
||||
"size": [400, 200],
|
||||
"flags": {},
|
||||
"order": 4,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "combined_config",
|
||||
"type": "COMBINED_CONFIG",
|
||||
"link": 78
|
||||
},
|
||||
{
|
||||
"name": "image",
|
||||
"shape": 7,
|
||||
"type": "IMAGE",
|
||||
"link": 79
|
||||
},
|
||||
{
|
||||
"name": "audio",
|
||||
"shape": 7,
|
||||
"type": "AUDIO",
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "images",
|
||||
"type": "IMAGE",
|
||||
"links": [81]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LightX2VModularInference"
|
||||
},
|
||||
"widgets_values": ["太空漫步,往前跑。 ", ""]
|
||||
},
|
||||
{
|
||||
"id": 108,
|
||||
"type": "VHS_VideoCombine",
|
||||
"pos": [1940.93017578125, -293.4468688964844],
|
||||
"size": [220.5830078125, 524.5830078125],
|
||||
"flags": {},
|
||||
"order": 6,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "images",
|
||||
"type": "IMAGE",
|
||||
"link": 81
|
||||
},
|
||||
{
|
||||
"name": "audio",
|
||||
"shape": 7,
|
||||
"type": "AUDIO",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "meta_batch",
|
||||
"shape": 7,
|
||||
"type": "VHS_BatchManager",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "vae",
|
||||
"shape": 7,
|
||||
"type": "VAE",
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "Filenames",
|
||||
"type": "VHS_FILENAMES",
|
||||
"links": null
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "VHS_VideoCombine"
|
||||
},
|
||||
"widgets_values": {
|
||||
"frame_rate": 16,
|
||||
"loop_count": 0,
|
||||
"filename_prefix": "AnimateDiff",
|
||||
"format": "video/h264-mp4",
|
||||
"pix_fmt": "yuv420p",
|
||||
"crf": 19,
|
||||
"save_metadata": true,
|
||||
"pingpong": false,
|
||||
"save_output": true,
|
||||
"videopreview": {
|
||||
"hidden": false,
|
||||
"paused": false,
|
||||
"params": {
|
||||
"filename": "AnimateDiff_00585.mp4",
|
||||
"subfolder": "",
|
||||
"type": "output",
|
||||
"format": "video/h264-mp4",
|
||||
"frame_rate": 16
|
||||
},
|
||||
"muted": false
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"links": [
|
||||
[76, 105, 0, 109, 0, "INFERENCE_CONFIG"],
|
||||
[78, 109, 0, 110, 0, "COMBINED_CONFIG"],
|
||||
[79, 107, 0, 110, 1, "IMAGE"],
|
||||
[80, 109, 0, 111, 0, "*"],
|
||||
[81, 110, 0, 108, 0, "IMAGE"],
|
||||
[82, 112, 0, 109, 5, "LORA_CHAIN"]
|
||||
],
|
||||
"groups": [],
|
||||
"config": {},
|
||||
"extra": {
|
||||
"ds": {
|
||||
"scale": 0.7513148009015777,
|
||||
"offset": [-282.5313156431909, 675.8508927908888]
|
||||
},
|
||||
"frontendVersion": "1.19.9"
|
||||
},
|
||||
"version": 0.4
|
||||
}
|
||||
@@ -1,313 +0,0 @@
|
||||
{
|
||||
"id": "8e881096-4f7b-4633-b34b-6b9c0f8aa093",
|
||||
"revision": 0,
|
||||
"last_node_id": 112,
|
||||
"last_link_id": 82,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 109,
|
||||
"type": "LightX2VConfigCombiner",
|
||||
"pos": [1531.4547119140625, -521.6442260742188],
|
||||
"size": [239.138671875, 126],
|
||||
"flags": {},
|
||||
"order": 3,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "inference_config",
|
||||
"type": "INFERENCE_CONFIG",
|
||||
"link": 76
|
||||
},
|
||||
{
|
||||
"name": "teacache_config",
|
||||
"shape": 7,
|
||||
"type": "TEACACHE_CONFIG",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "quantization_config",
|
||||
"shape": 7,
|
||||
"type": "QUANT_CONFIG",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "memory_config",
|
||||
"shape": 7,
|
||||
"type": "MEMORY_CONFIG",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "vae_config",
|
||||
"shape": 7,
|
||||
"type": "VAE_CONFIG",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "lora_chain",
|
||||
"shape": 7,
|
||||
"type": "LORA_CHAIN",
|
||||
"link": 82
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "combined_config",
|
||||
"type": "COMBINED_CONFIG",
|
||||
"links": [78, 80]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LightX2VConfigCombiner"
|
||||
},
|
||||
"widgets_values": []
|
||||
},
|
||||
{
|
||||
"id": 107,
|
||||
"type": "LoadImage",
|
||||
"pos": [1501.387939453125, 272.5443420410156],
|
||||
"size": [274.080078125, 314.0000305175781],
|
||||
"flags": {},
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "IMAGE",
|
||||
"type": "IMAGE",
|
||||
"links": [79]
|
||||
},
|
||||
{
|
||||
"name": "MASK",
|
||||
"type": "MASK",
|
||||
"links": null
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LoadImage"
|
||||
},
|
||||
"widgets_values": ["00.jpg", "image"]
|
||||
},
|
||||
{
|
||||
"id": 105,
|
||||
"type": "LightX2VInferenceConfig",
|
||||
"pos": [1073.62646484375, -580.9854125976562],
|
||||
"size": [270, 346],
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "inference_config",
|
||||
"type": "INFERENCE_CONFIG",
|
||||
"links": [76]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LightX2VInferenceConfig"
|
||||
},
|
||||
"widgets_values": [
|
||||
"wan2.1",
|
||||
"Wan2.1-T2V-14B",
|
||||
"t2v",
|
||||
4,
|
||||
1822974886,
|
||||
"fixed",
|
||||
1,
|
||||
8,
|
||||
480,
|
||||
480,
|
||||
81,
|
||||
16,
|
||||
"999, 750, 500, 250"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 111,
|
||||
"type": "easy showAnything",
|
||||
"pos": [1839.9300537109375, -494.4144287109375],
|
||||
"size": [624.5454711914062, 358.7272644042969],
|
||||
"flags": {
|
||||
"collapsed": true
|
||||
},
|
||||
"order": 5,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "anything",
|
||||
"shape": 7,
|
||||
"type": "*",
|
||||
"link": 80
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "output",
|
||||
"type": "*",
|
||||
"links": null
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "easy showAnything"
|
||||
},
|
||||
"widgets_values": [
|
||||
"{\"model_cls\": \"wan2.1\", \"model_path\": \"/mnt/aigc/users/lijiaqi2/ComfyUI/models/lightx2v/Wan2.1-T2V-14B\", \"task\": \"t2v\", \"mode\": \"infer\", \"infer_steps\": 4, \"seed\": 1822974886, \"sample_guide_scale\": 1.0, \"sample_shift\": 8, \"enable_cfg\": false, \"prompt\": \"\", \"negative_prompt\": \"\", \"target_height\": 480, \"target_width\": 480, \"target_video_length\": 81, \"fps\": 16, \"vae_stride\": [4, 8, 8], \"patch_size\": [1, 2, 2], \"feature_caching\": \"NoCaching\", \"teacache_thresh\": 0.26, \"coefficients\": null, \"use_ret_steps\": false, \"dit_quant_scheme\": \"bf16\", \"t5_quant_scheme\": \"bf16\", \"clip_quant_scheme\": \"fp16\", \"quant_op\": \"vllm\", \"precision_mode\": \"fp32\", \"dit_quantized_ckpt\": null, \"t5_quantized_ckpt\": null, \"clip_quantized_ckpt\": null, \"mm_config\": {\"mm_type\": \"Default\"}, \"rotary_chunk\": false, \"rotary_chunk_size\": 100, \"clean_cuda_cache\": false, \"torch_compile\": false, \"attention_type\": \"flash_attn3\", \"self_attn_1_type\": \"flash_attn3\", \"cross_attn_1_type\": \"flash_attn3\", \"cross_attn_2_type\": \"flash_attn3\", \"cpu_offload\": false, \"offload_granularity\": \"phase\", \"offload_ratio\": 1.0, \"t5_cpu_offload\": false, \"t5_offload_granularity\": \"model\", \"lazy_load\": false, \"unload_modules\": false, \"use_tiny_vae\": false, \"tiny_vae\": false, \"tiny_vae_path\": null, \"use_tiling_vae\": false, \"lora_path\": null, \"strength_model\": 1.0, \"do_mm_calib\": false, \"parallel_attn_type\": null, \"parallel_vae\": false, \"max_area\": false, \"use_prompt_enhancer\": false, \"text_len\": 512, \"denoising_step_list\": [999, 750, 500, 250], \"_class_name\": \"WanModel\", \"_diffusers_version\": \"0.30.0\", \"dim\": 5120, \"eps\": 1e-06, \"ffn_dim\": 13824, \"freq_dim\": 256, \"in_dim\": 16, \"model_type\": \"t2v\", \"num_heads\": 40, \"num_layers\": 40, \"out_dim\": 16, \"lora_configs\": [{\"path\": \"/mnt/aigc/users/lijiaqi2/ComfyUI/models/lightx2v/loras/Wan21_T2V_14B_lightx2v_cfg_step_distill_lora_rank32.safetensors\", \"strength\": 1.0}]}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 110,
|
||||
"type": "LightX2VModularInference",
|
||||
"pos": [1496.8944091796875, -278.5135498046875],
|
||||
"size": [400, 200],
|
||||
"flags": {},
|
||||
"order": 4,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "combined_config",
|
||||
"type": "COMBINED_CONFIG",
|
||||
"link": 78
|
||||
},
|
||||
{
|
||||
"name": "image",
|
||||
"shape": 7,
|
||||
"type": "IMAGE",
|
||||
"link": 79
|
||||
},
|
||||
{
|
||||
"name": "audio",
|
||||
"shape": 7,
|
||||
"type": "AUDIO",
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "images",
|
||||
"type": "IMAGE",
|
||||
"links": [81]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LightX2VModularInference"
|
||||
},
|
||||
"widgets_values": ["好奇的小兔子。 ", ""]
|
||||
},
|
||||
{
|
||||
"id": 108,
|
||||
"type": "VHS_VideoCombine",
|
||||
"pos": [1943.5921630859375, -272.15087890625],
|
||||
"size": [220.5830078125, 524.5830078125],
|
||||
"flags": {},
|
||||
"order": 6,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "images",
|
||||
"type": "IMAGE",
|
||||
"link": 81
|
||||
},
|
||||
{
|
||||
"name": "audio",
|
||||
"shape": 7,
|
||||
"type": "AUDIO",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "meta_batch",
|
||||
"shape": 7,
|
||||
"type": "VHS_BatchManager",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "vae",
|
||||
"shape": 7,
|
||||
"type": "VAE",
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "Filenames",
|
||||
"type": "VHS_FILENAMES",
|
||||
"links": null
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "VHS_VideoCombine"
|
||||
},
|
||||
"widgets_values": {
|
||||
"frame_rate": 16,
|
||||
"loop_count": 0,
|
||||
"filename_prefix": "AnimateDiff",
|
||||
"format": "video/h264-mp4",
|
||||
"pix_fmt": "yuv420p",
|
||||
"crf": 19,
|
||||
"save_metadata": true,
|
||||
"pingpong": false,
|
||||
"save_output": true,
|
||||
"videopreview": {
|
||||
"hidden": false,
|
||||
"paused": false,
|
||||
"params": {
|
||||
"filename": "AnimateDiff_00585.mp4",
|
||||
"subfolder": "",
|
||||
"type": "output",
|
||||
"format": "video/h264-mp4",
|
||||
"frame_rate": 16
|
||||
},
|
||||
"muted": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 112,
|
||||
"type": "LightX2VLoRALoader",
|
||||
"pos": [1068.7301025390625, -174.75782775878906],
|
||||
"size": [270, 82],
|
||||
"flags": {},
|
||||
"order": 2,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "lora_chain",
|
||||
"shape": 7,
|
||||
"type": "LORA_CHAIN",
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "lora_chain",
|
||||
"type": "LORA_CHAIN",
|
||||
"links": [82]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LightX2VLoRALoader"
|
||||
},
|
||||
"widgets_values": [
|
||||
"Wan21_T2V_14B_lightx2v_cfg_step_distill_lora_rank32.safetensors",
|
||||
1
|
||||
]
|
||||
}
|
||||
],
|
||||
"links": [
|
||||
[76, 105, 0, 109, 0, "INFERENCE_CONFIG"],
|
||||
[78, 109, 0, 110, 0, "COMBINED_CONFIG"],
|
||||
[79, 107, 0, 110, 1, "IMAGE"],
|
||||
[80, 109, 0, 111, 0, "*"],
|
||||
[81, 110, 0, 108, 0, "IMAGE"],
|
||||
[82, 112, 0, 109, 5, "LORA_CHAIN"]
|
||||
],
|
||||
"groups": [],
|
||||
"config": {},
|
||||
"extra": {
|
||||
"ds": {
|
||||
"scale": 0.9090909090909091,
|
||||
"offset": [-335.01067565958357, 635.6502922171578]
|
||||
},
|
||||
"frontendVersion": "1.19.9"
|
||||
},
|
||||
"version": 0.4
|
||||
}
|
||||
@@ -0,0 +1,509 @@
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
import urllib.parse
|
||||
from abc import ABC, abstractmethod
|
||||
from contextlib import contextmanager
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
|
||||
import numpy as np
|
||||
import scipy.io.wavfile as wavfile
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
|
||||
class FileHandler(ABC):
|
||||
"""Base class for file handling."""
|
||||
|
||||
@abstractmethod
|
||||
def save(self, data: Any, path: str) -> str:
|
||||
"""Save data to file."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def load(self, path: str) -> Any:
|
||||
"""Load data from file."""
|
||||
pass
|
||||
|
||||
|
||||
class AudioFileHandler(FileHandler):
|
||||
"""Handler for audio files."""
|
||||
|
||||
def __init__(self):
|
||||
self.supported_formats = [".wav", ".mp3", ".flac", ".m4a"]
|
||||
|
||||
def save(
|
||||
self,
|
||||
audio_data: Union[Dict, torch.Tensor, np.ndarray, Tuple],
|
||||
path: str,
|
||||
sample_rate: Optional[int] = None,
|
||||
) -> str:
|
||||
"""Save audio data to file.
|
||||
|
||||
Args:
|
||||
audio_data: Audio data in various formats
|
||||
path: Output file path
|
||||
sample_rate: Sample rate (required if not in audio_data)
|
||||
|
||||
Returns:
|
||||
Path to saved file
|
||||
"""
|
||||
|
||||
waveform, sr = self._extract_audio_data(audio_data, sample_rate)
|
||||
|
||||
# Ensure waveform is in correct shape
|
||||
waveform = self._normalize_waveform(waveform)
|
||||
|
||||
# Always save as WAV format
|
||||
ext = os.path.splitext(path)[1].lower()
|
||||
if ext != ".wav":
|
||||
path = path.rsplit(".", 1)[0] + ".wav"
|
||||
logging.info(f"Audio will be saved as WAV format: {path}")
|
||||
|
||||
# Ensure waveform is in int16 format for WAV
|
||||
if waveform.dtype != np.int16:
|
||||
# Normalize to [-1, 1] range if not already
|
||||
if waveform.dtype == np.float32 or waveform.dtype == np.float64:
|
||||
# Clip to [-1, 1] to avoid overflow
|
||||
waveform = np.clip(waveform, -1.0, 1.0)
|
||||
waveform = (waveform * 32767).astype(np.int16)
|
||||
else:
|
||||
# Assume uint8 or other integer type
|
||||
waveform = waveform.astype(np.int16)
|
||||
|
||||
wavfile.write(path, sr, waveform)
|
||||
logging.info(f"Audio saved to {path}")
|
||||
return path
|
||||
|
||||
def load(self, path: str) -> Tuple[np.ndarray, int]:
|
||||
"""Load audio from file.
|
||||
|
||||
Returns:
|
||||
Tuple of (waveform, sample_rate)
|
||||
"""
|
||||
sample_rate, waveform = wavfile.read(path)
|
||||
return waveform, sample_rate
|
||||
|
||||
def _extract_audio_data(self, audio_data: Any, sample_rate: Optional[int] = None) -> Tuple[np.ndarray, int]:
|
||||
"""Extract waveform and sample rate from various audio formats.
|
||||
|
||||
Handles three main sources:
|
||||
1. ComfyUI LoadAudio output: {"waveform": tensor, "sample_rate": int}
|
||||
2. Tuple format: (waveform, sample_rate)
|
||||
3. Raw waveform with separate sample_rate
|
||||
"""
|
||||
if isinstance(audio_data, dict):
|
||||
if "waveform" in audio_data and "sample_rate" in audio_data:
|
||||
waveform = audio_data["waveform"]
|
||||
sr = audio_data["sample_rate"]
|
||||
|
||||
# Handle ComfyUI LoadAudio format specifically
|
||||
# ComfyUI returns waveform with shape [batch, channels, samples]
|
||||
if isinstance(waveform, torch.Tensor):
|
||||
if waveform.dim() == 3: # [batch, channels, samples]
|
||||
waveform = waveform[0] # Take first batch
|
||||
if waveform.dim() == 2 and waveform.shape[0] <= 2: # [channels, samples]
|
||||
waveform = waveform.transpose(0, 1) # -> [samples, channels]
|
||||
waveform = waveform.cpu().numpy()
|
||||
else:
|
||||
raise ValueError("Audio dict must contain 'waveform' and 'sample_rate'")
|
||||
elif isinstance(audio_data, tuple) and len(audio_data) == 2:
|
||||
waveform, sr = audio_data
|
||||
if isinstance(waveform, torch.Tensor):
|
||||
waveform = waveform.cpu().numpy()
|
||||
elif sample_rate is not None:
|
||||
waveform = audio_data
|
||||
sr = sample_rate
|
||||
if isinstance(waveform, torch.Tensor):
|
||||
waveform = waveform.cpu().numpy()
|
||||
else:
|
||||
raise ValueError("Sample rate must be provided for raw audio data")
|
||||
|
||||
return waveform, sr
|
||||
|
||||
def _normalize_waveform(self, waveform: np.ndarray) -> np.ndarray:
|
||||
"""Normalize waveform shape for saving.
|
||||
|
||||
Ensures waveform is in shape [samples, channels] or [samples] for mono.
|
||||
"""
|
||||
# Already converted to numpy in _extract_audio_data
|
||||
|
||||
# Handle different shapes
|
||||
if waveform.ndim == 3: # Shouldn't happen, but handle it
|
||||
waveform = waveform[0]
|
||||
|
||||
if waveform.ndim == 2:
|
||||
# Check if it's [channels, samples] format (channels < samples typically)
|
||||
if waveform.shape[0] <= 2 and waveform.shape[0] < waveform.shape[1]:
|
||||
waveform = waveform.T # -> [samples, channels]
|
||||
# If mono with extra dimension, squeeze it
|
||||
if waveform.shape[1] == 1:
|
||||
waveform = waveform.squeeze()
|
||||
|
||||
return waveform
|
||||
|
||||
def _save_with_wave(self, path: str, waveform: np.ndarray, sample_rate: int):
|
||||
"""Save audio using wave module."""
|
||||
import wave
|
||||
|
||||
with wave.open(path, "wb") as wav_file:
|
||||
wav_file.setnchannels(1 if waveform.ndim == 1 else waveform.shape[-1])
|
||||
wav_file.setsampwidth(2) # 16-bit
|
||||
wav_file.setframerate(sample_rate)
|
||||
|
||||
if waveform.dtype != np.int16:
|
||||
waveform = (waveform * 32767).astype(np.int16)
|
||||
|
||||
wav_file.writeframes(waveform.tobytes())
|
||||
|
||||
def _load_with_wave(self, path: str) -> Tuple[np.ndarray, int]:
|
||||
"""Load audio using wave module."""
|
||||
import wave
|
||||
|
||||
with wave.open(path, "rb") as wav_file:
|
||||
sample_rate = wav_file.getframerate()
|
||||
frames = wav_file.readframes(wav_file.getnframes())
|
||||
waveform = np.frombuffer(frames, dtype=np.int16)
|
||||
|
||||
if wav_file.getnchannels() > 1:
|
||||
waveform = waveform.reshape(-1, wav_file.getnchannels())
|
||||
|
||||
return waveform, sample_rate
|
||||
|
||||
|
||||
class ImageFileHandler(FileHandler):
|
||||
"""Handler for image files."""
|
||||
|
||||
def __init__(self):
|
||||
self.supported_formats = [".png", ".jpg", ".jpeg", ".bmp", ".tiff"]
|
||||
|
||||
def save(self, image_data: Union[torch.Tensor, np.ndarray, Image.Image], path: str) -> str:
|
||||
"""Save image data to file.
|
||||
|
||||
Args:
|
||||
image_data: Image data in various formats
|
||||
path: Output file path
|
||||
|
||||
Returns:
|
||||
Path to saved file
|
||||
"""
|
||||
if isinstance(image_data, torch.Tensor):
|
||||
# Convert from tensor [H, W, C] or [C, H, W]
|
||||
if image_data.dim() == 4: # [batch, ...]
|
||||
image_data = image_data[0]
|
||||
|
||||
image_np = image_data.cpu().numpy()
|
||||
|
||||
# Handle channel ordering
|
||||
if image_np.shape[0] in [1, 3, 4]: # [C, H, W]
|
||||
image_np = np.transpose(image_np, (1, 2, 0))
|
||||
|
||||
# Convert to uint8
|
||||
if image_np.dtype != np.uint8:
|
||||
image_np = (image_np * 255).astype(np.uint8)
|
||||
|
||||
image = Image.fromarray(image_np.squeeze())
|
||||
|
||||
elif isinstance(image_data, np.ndarray):
|
||||
if image_data.dtype != np.uint8:
|
||||
image_data = (image_data * 255).astype(np.uint8)
|
||||
image = Image.fromarray(image_data.squeeze())
|
||||
|
||||
elif isinstance(image_data, Image.Image):
|
||||
image = image_data
|
||||
else:
|
||||
raise ValueError(f"Unsupported image format: {type(image_data)}")
|
||||
|
||||
image.save(path)
|
||||
logging.info(f"Image saved to {path}")
|
||||
return path
|
||||
|
||||
def load(self, path: str) -> Image.Image:
|
||||
"""Load image from file."""
|
||||
return Image.open(path)
|
||||
|
||||
|
||||
class MaskFileHandler(ImageFileHandler):
|
||||
"""Handler specifically for mask files."""
|
||||
|
||||
def save(self, mask_data: Union[torch.Tensor, np.ndarray], path: str) -> str:
|
||||
"""Save mask data to file.
|
||||
|
||||
Args:
|
||||
mask_data: Mask data (single channel)
|
||||
path: Output file path
|
||||
|
||||
Returns:
|
||||
Path to saved file
|
||||
"""
|
||||
if isinstance(mask_data, torch.Tensor):
|
||||
if mask_data.dim() == 3: # [batch, H, W]
|
||||
mask_data = mask_data[0]
|
||||
mask_np = (mask_data.cpu().numpy() * 255).astype(np.uint8)
|
||||
elif isinstance(mask_data, np.ndarray):
|
||||
mask_np = (mask_data * 255).astype(np.uint8)
|
||||
else:
|
||||
mask_np = mask_data
|
||||
|
||||
mask_image = Image.fromarray(mask_np)
|
||||
mask_image.save(path)
|
||||
logging.info(f"Mask saved to {path}")
|
||||
return path
|
||||
|
||||
|
||||
class TempFileManager:
|
||||
def __init__(self):
|
||||
self.temp_files: List[str] = []
|
||||
self.temp_dirs: List[str] = []
|
||||
|
||||
@contextmanager
|
||||
def temp_file(self, suffix: str = "", prefix: str = "lightx2v_", delete: bool = True):
|
||||
temp_file = tempfile.NamedTemporaryFile(suffix=suffix, prefix=prefix, delete=False)
|
||||
temp_path = temp_file.name
|
||||
temp_file.close()
|
||||
|
||||
self.temp_files.append(temp_path)
|
||||
|
||||
try:
|
||||
yield temp_path
|
||||
finally:
|
||||
if delete:
|
||||
self.cleanup_file(temp_path)
|
||||
|
||||
def create_temp_file(self, suffix: str = "", prefix: str = "lightx2v_") -> str:
|
||||
with tempfile.NamedTemporaryFile(suffix=suffix, prefix=prefix, delete=False) as tmp:
|
||||
temp_path = tmp.name
|
||||
|
||||
self.temp_files.append(temp_path)
|
||||
return temp_path
|
||||
|
||||
def cleanup_file(self, path: str):
|
||||
if path in self.temp_files:
|
||||
self.temp_files.remove(path)
|
||||
|
||||
if os.path.exists(path):
|
||||
try:
|
||||
os.unlink(path)
|
||||
logging.debug(f"Cleaned up temp file: {path}")
|
||||
except Exception as e:
|
||||
logging.warning(f"Failed to clean up {path}: {e}")
|
||||
|
||||
@contextmanager
|
||||
def temp_dir(self, suffix: str = "", prefix: str = "lightx2v_", delete: bool = True):
|
||||
temp_dir = tempfile.mkdtemp(suffix=suffix, prefix=prefix)
|
||||
self.temp_dirs.append(temp_dir)
|
||||
|
||||
try:
|
||||
yield temp_dir
|
||||
finally:
|
||||
if delete:
|
||||
self.cleanup_dir(temp_dir)
|
||||
|
||||
def create_temp_dir(self, suffix: str = "", prefix: str = "lightx2v_") -> str:
|
||||
temp_dir = tempfile.mkdtemp(suffix=suffix, prefix=prefix)
|
||||
self.temp_dirs.append(temp_dir)
|
||||
return temp_dir
|
||||
|
||||
def cleanup_dir(self, path: str):
|
||||
if path in self.temp_dirs:
|
||||
self.temp_dirs.remove(path)
|
||||
|
||||
if os.path.exists(path):
|
||||
try:
|
||||
import shutil
|
||||
|
||||
shutil.rmtree(path)
|
||||
logging.debug(f"Cleaned up temp directory: {path}")
|
||||
except Exception as e:
|
||||
logging.warning(f"Failed to clean up directory {path}: {e}")
|
||||
|
||||
def cleanup_all(self):
|
||||
for temp_file in self.temp_files[:]:
|
||||
self.cleanup_file(temp_file)
|
||||
self.temp_files.clear()
|
||||
|
||||
for temp_dir in self.temp_dirs[:]:
|
||||
self.cleanup_dir(temp_dir)
|
||||
self.temp_dirs.clear()
|
||||
|
||||
def __del__(self):
|
||||
self.cleanup_all()
|
||||
|
||||
|
||||
class HTTPFileDownloader:
|
||||
"""Handler for downloading files from HTTP/HTTPS URLs."""
|
||||
|
||||
def __init__(self):
|
||||
self.temp_manager = TempFileManager()
|
||||
|
||||
@staticmethod
|
||||
def is_url(path: str) -> bool:
|
||||
"""Check if the path is an HTTP/HTTPS URL.
|
||||
|
||||
Args:
|
||||
path: Path to check
|
||||
|
||||
Returns:
|
||||
True if path is HTTP/HTTPS URL, False otherwise
|
||||
"""
|
||||
if not path:
|
||||
return False
|
||||
|
||||
parsed = urllib.parse.urlparse(path)
|
||||
return parsed.scheme in ("http", "https")
|
||||
|
||||
def download_to_input(self, url: str, filename: Optional[str] = None) -> str:
|
||||
"""Download file from URL to ComfyUI input directory.
|
||||
|
||||
Args:
|
||||
url: URL to download from
|
||||
filename: Target filename (optional, will be generated if not provided)
|
||||
|
||||
Returns:
|
||||
Absolute path to downloaded file
|
||||
|
||||
Raises:
|
||||
Exception: If download fails
|
||||
"""
|
||||
try:
|
||||
import requests
|
||||
except ImportError:
|
||||
logging.error("requests module not available for HTTP download")
|
||||
raise ImportError("requests module is required for HTTP download")
|
||||
|
||||
# Generate filename if not provided
|
||||
if not filename:
|
||||
# Extract filename from URL
|
||||
parsed_url = urllib.parse.urlparse(url)
|
||||
url_filename = os.path.basename(parsed_url.path)
|
||||
|
||||
if url_filename:
|
||||
# Use URL filename but add a unique suffix to avoid conflicts
|
||||
import uuid
|
||||
|
||||
name, ext = os.path.splitext(url_filename)
|
||||
filename = f"{name}_{uuid.uuid4().hex[:8]}{ext}"
|
||||
else:
|
||||
# Generate a completely new filename
|
||||
import uuid
|
||||
|
||||
filename = f"downloaded_{uuid.uuid4().hex[:8]}"
|
||||
|
||||
# Get input directory
|
||||
input_dir = ComfyUIFileResolver.get_input_directory()
|
||||
full_path = os.path.join(input_dir, filename)
|
||||
|
||||
# Create directory if needed
|
||||
os.makedirs(input_dir, exist_ok=True)
|
||||
|
||||
try:
|
||||
logging.info(f"Downloading file from {url} to {full_path}")
|
||||
|
||||
# Download with streaming to handle large files
|
||||
response = requests.get(url, stream=True, timeout=30)
|
||||
response.raise_for_status()
|
||||
|
||||
# Get total size for progress reporting
|
||||
total_size = int(response.headers.get("content-length", 0))
|
||||
downloaded_size = 0
|
||||
|
||||
# Write to file
|
||||
with open(full_path, "wb") as f:
|
||||
for chunk in response.iter_content(chunk_size=8192):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
downloaded_size += len(chunk)
|
||||
|
||||
# Log progress for large files
|
||||
if total_size > 0 and total_size > 1024 * 1024: # > 1MB
|
||||
progress = (downloaded_size / total_size) * 100
|
||||
if downloaded_size % (1024 * 1024) == 0: # Log every 1MB
|
||||
logging.debug(f"Download progress: {progress:.1f}%")
|
||||
|
||||
logging.info(f"Successfully downloaded file to {full_path}")
|
||||
return full_path
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
# Clean up partial file if download failed
|
||||
if os.path.exists(full_path):
|
||||
try:
|
||||
os.unlink(full_path)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
logging.error(f"Failed to download file from {url}: {e}")
|
||||
raise Exception(f"Failed to download file from {url}: {e}")
|
||||
except Exception as e:
|
||||
# Clean up partial file if download failed
|
||||
if os.path.exists(full_path):
|
||||
try:
|
||||
os.unlink(full_path)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
logging.error(f"Error downloading file: {e}")
|
||||
raise
|
||||
|
||||
def download_if_url(self, path: str, prefix: str = "downloaded") -> str:
|
||||
"""Download file if path is URL, otherwise return path as-is.
|
||||
|
||||
Args:
|
||||
path: Path or URL to process
|
||||
prefix: Prefix for downloaded filename
|
||||
|
||||
Returns:
|
||||
Absolute path to local file
|
||||
"""
|
||||
if self.is_url(path):
|
||||
# Generate filename with prefix
|
||||
import uuid
|
||||
|
||||
ext = os.path.splitext(urllib.parse.urlparse(path).path)[1] or ".bin"
|
||||
filename = f"{prefix}_{uuid.uuid4().hex[:8]}{ext}"
|
||||
return self.download_to_input(path, filename)
|
||||
|
||||
return path
|
||||
|
||||
|
||||
class ComfyUIFileResolver:
|
||||
"""Resolve file paths for ComfyUI input/output directories."""
|
||||
|
||||
@staticmethod
|
||||
def get_input_directory() -> str:
|
||||
"""Get ComfyUI input directory."""
|
||||
try:
|
||||
import folder_paths
|
||||
|
||||
return folder_paths.get_input_directory()
|
||||
except ImportError:
|
||||
# Fallback if not in ComfyUI environment
|
||||
return "input"
|
||||
|
||||
@staticmethod
|
||||
def resolve_input_path(filename: str) -> str:
|
||||
"""Resolve a filename to full path in input directory."""
|
||||
if os.path.isabs(filename):
|
||||
return filename
|
||||
|
||||
input_dir = ComfyUIFileResolver.get_input_directory()
|
||||
return os.path.join(input_dir, filename)
|
||||
|
||||
@staticmethod
|
||||
def save_to_input(data: Any, filename: str, handler: FileHandler) -> str:
|
||||
"""Save data to ComfyUI input directory.
|
||||
|
||||
Args:
|
||||
data: Data to save
|
||||
filename: Target filename
|
||||
handler: File handler to use
|
||||
|
||||
Returns:
|
||||
Full path to saved file
|
||||
"""
|
||||
input_dir = ComfyUIFileResolver.get_input_directory()
|
||||
full_path = os.path.join(input_dir, filename)
|
||||
|
||||
# Create directory if needed
|
||||
os.makedirs(input_dir, exist_ok=True)
|
||||
|
||||
return handler.save(data, full_path)
|
||||
+1
-1
Submodule lightx2v updated: df62828718...52e86de7c2
+40
-6
@@ -29,16 +29,43 @@ def scan_models() -> List[str]:
|
||||
return ["None"] + models if models else ["None"]
|
||||
|
||||
|
||||
def scan_loras() -> List[str]:
|
||||
loras = []
|
||||
base_path = get_model_base_path()
|
||||
loras_path = base_path / "loras"
|
||||
def support_model_cls_list() -> List[str]:
|
||||
return [
|
||||
"wan2.1",
|
||||
"wan2.1_distill",
|
||||
"wan2.1_vace",
|
||||
"cogvideox",
|
||||
"seko_talk",
|
||||
"wan2.2_moe",
|
||||
"wan2.2",
|
||||
"wan2.2_moe_audio",
|
||||
"wan2.2_audio",
|
||||
"wan2.2_moe_distill",
|
||||
"qwen_image",
|
||||
"seedvr2",
|
||||
]
|
||||
|
||||
if loras_path.exists():
|
||||
for item in loras_path.iterdir():
|
||||
|
||||
def get_loras_models(model_path: Path) -> List[str]:
|
||||
loras = []
|
||||
if model_path.exists():
|
||||
for item in model_path.iterdir():
|
||||
if item.is_file():
|
||||
if item.suffix.lower() in [".safetensors", ".pt", ".pth", ".ckpt"]:
|
||||
loras.append(item.name)
|
||||
return loras
|
||||
|
||||
|
||||
def scan_loras() -> List[str]:
|
||||
base_path = get_model_base_path()
|
||||
loras_path = base_path / "loras"
|
||||
loras = get_loras_models(loras_path)
|
||||
|
||||
models_base = folder_paths.models_dir
|
||||
loras_path = Path(models_base) / "loras"
|
||||
|
||||
loras2 = get_loras_models(loras_path)
|
||||
loras.extend(loras2)
|
||||
|
||||
loras.sort()
|
||||
|
||||
@@ -66,6 +93,13 @@ def get_lora_full_path(lora_name: str) -> str:
|
||||
|
||||
if lora_path.exists():
|
||||
return str(lora_path)
|
||||
|
||||
models_base = folder_paths.models_dir
|
||||
loras_path = Path(models_base) / "loras" / lora_name
|
||||
|
||||
if loras_path.exists():
|
||||
return str(loras_path)
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
|
||||
@@ -1,757 +0,0 @@
|
||||
"""Modular ComfyUI nodes for LightX2V without presets."""
|
||||
|
||||
import gc
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from comfy.utils import ProgressBar
|
||||
from PIL import Image
|
||||
|
||||
from .bridge import (
|
||||
ModularConfigManager,
|
||||
get_available_attn_ops,
|
||||
get_available_quant_ops,
|
||||
)
|
||||
from .lightx2v.lightx2v.infer import init_runner
|
||||
from .model_utils import (
|
||||
get_lora_full_path,
|
||||
get_model_full_path,
|
||||
scan_loras,
|
||||
scan_models,
|
||||
)
|
||||
|
||||
|
||||
class LightX2VInferenceConfig:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
available_models = scan_models()
|
||||
|
||||
return {
|
||||
"required": {
|
||||
"model_cls": (
|
||||
["wan2.1", "wan2.1_audio", "wan2.1_distill", "hunyuan"],
|
||||
{"default": "wan2.1", "tooltip": "Model type"},
|
||||
),
|
||||
"model_name": (
|
||||
available_models,
|
||||
{
|
||||
"default": available_models[0],
|
||||
"tooltip": "Select model from available models",
|
||||
},
|
||||
),
|
||||
"task": (
|
||||
["t2v", "i2v"],
|
||||
{
|
||||
"default": "t2v",
|
||||
"tooltip": "Task type: text-to-video or image-to-video",
|
||||
},
|
||||
),
|
||||
"infer_steps": (
|
||||
"INT",
|
||||
{"default": 40, "min": 1, "max": 100, "tooltip": "Inference steps"},
|
||||
),
|
||||
"seed": (
|
||||
"INT",
|
||||
{
|
||||
"default": 42,
|
||||
"min": -1,
|
||||
"max": 2**32 - 1,
|
||||
"tooltip": "Random seed, -1 for random",
|
||||
},
|
||||
),
|
||||
"cfg_scale": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 5.0,
|
||||
"min": 1.0,
|
||||
"max": 10.0,
|
||||
"step": 0.1,
|
||||
"tooltip": "CFG guidance strength",
|
||||
},
|
||||
),
|
||||
"sample_shift": (
|
||||
"INT",
|
||||
{"default": 5, "min": 0, "max": 10, "tooltip": "Sample shift"},
|
||||
),
|
||||
"height": (
|
||||
"INT",
|
||||
{
|
||||
"default": 480,
|
||||
"min": 64,
|
||||
"max": 2048,
|
||||
"step": 8,
|
||||
"tooltip": "Video height",
|
||||
},
|
||||
),
|
||||
"width": (
|
||||
"INT",
|
||||
{
|
||||
"default": 832,
|
||||
"min": 64,
|
||||
"max": 2048,
|
||||
"step": 8,
|
||||
"tooltip": "Video width",
|
||||
},
|
||||
),
|
||||
"video_length": (
|
||||
"INT",
|
||||
{
|
||||
"default": 81,
|
||||
"min": 16,
|
||||
"max": 120,
|
||||
"tooltip": "Video frame count",
|
||||
},
|
||||
),
|
||||
"fps": (
|
||||
"INT",
|
||||
{
|
||||
"default": 16,
|
||||
"min": 8,
|
||||
"max": 30,
|
||||
"tooltip": "Model output frame rate (cannot be changed)",
|
||||
},
|
||||
),
|
||||
},
|
||||
"optional": {
|
||||
"denoising_steps": (
|
||||
"STRING",
|
||||
{
|
||||
"default": "",
|
||||
"tooltip": "Custom denoising steps for distillation models (comma-separated, e.g., '999,750,500,250'). Leave empty to use model defaults.",
|
||||
},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("INFERENCE_CONFIG",)
|
||||
RETURN_NAMES = ("inference_config",)
|
||||
FUNCTION = "create_config"
|
||||
CATEGORY = "LightX2V/Config"
|
||||
|
||||
def create_config(
|
||||
self,
|
||||
model_cls,
|
||||
model_name,
|
||||
task,
|
||||
infer_steps,
|
||||
seed,
|
||||
cfg_scale,
|
||||
sample_shift,
|
||||
height,
|
||||
width,
|
||||
video_length,
|
||||
fps,
|
||||
denoising_steps="",
|
||||
):
|
||||
"""Create basic inference configuration."""
|
||||
model_path = get_model_full_path(model_name)
|
||||
|
||||
config = {
|
||||
"model_cls": model_cls,
|
||||
"model_path": model_path,
|
||||
"task": task,
|
||||
"infer_steps": infer_steps,
|
||||
"seed": seed if seed != -1 else np.random.randint(0, 2**32 - 1),
|
||||
"cfg_scale": cfg_scale,
|
||||
"sample_shift": sample_shift,
|
||||
"height": height,
|
||||
"width": width,
|
||||
"video_length": video_length,
|
||||
"fps": fps,
|
||||
}
|
||||
|
||||
if denoising_steps and denoising_steps.strip():
|
||||
try:
|
||||
steps_list = [int(s.strip()) for s in denoising_steps.split(",")]
|
||||
config["denoising_step_list"] = steps_list
|
||||
config["infer_steps"] = len(steps_list)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return (config,)
|
||||
|
||||
|
||||
class LightX2VTeaCache:
|
||||
"""TeaCache configuration node."""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"enable": (
|
||||
"BOOLEAN",
|
||||
{"default": False, "tooltip": "Enable TeaCache feature caching"},
|
||||
),
|
||||
"threshold": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.26,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "Cache threshold, lower values provide more speedup: 0.1 ~2x speedup, 0.2 ~3x speedup",
|
||||
},
|
||||
),
|
||||
"use_ret_steps": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": False,
|
||||
"tooltip": "Only cache key steps to balance quality and speed",
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("TEACACHE_CONFIG",)
|
||||
RETURN_NAMES = ("teacache_config",)
|
||||
FUNCTION = "create_config"
|
||||
CATEGORY = "LightX2V/Config"
|
||||
|
||||
def create_config(self, enable, threshold, use_ret_steps):
|
||||
config = {
|
||||
"enable": enable,
|
||||
"threshold": threshold,
|
||||
"use_ret_steps": use_ret_steps,
|
||||
}
|
||||
return (config,)
|
||||
|
||||
|
||||
class LightX2VQuantization:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
available_ops = get_available_quant_ops()
|
||||
quant_backends = []
|
||||
|
||||
for op_name, is_available in available_ops:
|
||||
if is_available:
|
||||
quant_backends.append(op_name)
|
||||
|
||||
# Always have at least one option
|
||||
if not quant_backends:
|
||||
quant_backends = ["none"]
|
||||
|
||||
return {
|
||||
"required": {
|
||||
"dit_precision": (
|
||||
["bf16", "int8", "fp8"],
|
||||
{"default": "bf16", "tooltip": "DIT model quantization precision"},
|
||||
),
|
||||
"t5_precision": (
|
||||
["bf16", "int8", "fp8"],
|
||||
{"default": "bf16", "tooltip": "T5 encoder quantization precision"},
|
||||
),
|
||||
"clip_precision": (
|
||||
["fp16", "int8", "fp8"],
|
||||
{
|
||||
"default": "fp16",
|
||||
"tooltip": "CLIP encoder quantization precision",
|
||||
},
|
||||
),
|
||||
"quant_backend": (
|
||||
quant_backends,
|
||||
{
|
||||
"default": quant_backends[0],
|
||||
"tooltip": "Quantization computation backend",
|
||||
},
|
||||
),
|
||||
"sensitive_layers_precision": (
|
||||
["fp32", "bf16"],
|
||||
{
|
||||
"default": "fp32",
|
||||
"tooltip": "Sensitive layers (normalization and embedding) precision",
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("QUANT_CONFIG",)
|
||||
RETURN_NAMES = ("quantization_config",)
|
||||
FUNCTION = "create_config"
|
||||
CATEGORY = "LightX2V/Config"
|
||||
|
||||
def create_config(
|
||||
self,
|
||||
dit_precision,
|
||||
t5_precision,
|
||||
clip_precision,
|
||||
quant_backend,
|
||||
sensitive_layers_precision,
|
||||
):
|
||||
"""Create quantization configuration."""
|
||||
config = {
|
||||
"dit_precision": dit_precision,
|
||||
"t5_precision": t5_precision,
|
||||
"clip_precision": clip_precision,
|
||||
"quant_backend": quant_backend,
|
||||
"sensitive_layers_precision": sensitive_layers_precision,
|
||||
}
|
||||
return (config,)
|
||||
|
||||
|
||||
class LightX2VMemoryOptimization:
|
||||
"""Memory optimization configuration node."""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
available_attn = get_available_attn_ops()
|
||||
attn_types = []
|
||||
|
||||
for op_name, is_available in available_attn:
|
||||
if is_available:
|
||||
attn_types.append(op_name)
|
||||
|
||||
if "torch_sdpa" not in attn_types:
|
||||
attn_types.append("torch_sdpa")
|
||||
|
||||
return {
|
||||
"required": {
|
||||
"optimization_level": (
|
||||
["none", "low", "medium", "high", "extreme"],
|
||||
{
|
||||
"default": "none",
|
||||
"tooltip": "Memory optimization level, higher levels save more memory but may affect speed",
|
||||
},
|
||||
),
|
||||
"attention_type": (
|
||||
attn_types,
|
||||
{"default": attn_types[0], "tooltip": "Attention mechanism type"},
|
||||
),
|
||||
},
|
||||
"optional": {
|
||||
# GPU optimization
|
||||
"enable_rotary_chunk": (
|
||||
"BOOLEAN",
|
||||
{"default": False, "tooltip": "Enable rotary encoding chunking"},
|
||||
),
|
||||
"rotary_chunk_size": (
|
||||
"INT",
|
||||
{"default": 100, "min": 100, "max": 10000, "step": 100},
|
||||
),
|
||||
"clean_cuda_cache": (
|
||||
"BOOLEAN",
|
||||
{"default": False, "tooltip": "Clean CUDA cache promptly"},
|
||||
),
|
||||
# CPU offloading
|
||||
"enable_cpu_offload": (
|
||||
"BOOLEAN",
|
||||
{"default": False, "tooltip": "Enable CPU offloading"},
|
||||
),
|
||||
"offload_granularity": (
|
||||
["block", "phase"],
|
||||
{"default": "phase", "tooltip": "Offload granularity"},
|
||||
),
|
||||
"offload_ratio": (
|
||||
"FLOAT",
|
||||
{"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.1},
|
||||
),
|
||||
# Module management
|
||||
"lazy_load": (
|
||||
"BOOLEAN",
|
||||
{"default": False, "tooltip": "Lazy load model"},
|
||||
),
|
||||
"unload_after_inference": (
|
||||
"BOOLEAN",
|
||||
{"default": False, "tooltip": "Unload modules after inference"},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("MEMORY_CONFIG",)
|
||||
RETURN_NAMES = ("memory_config",)
|
||||
FUNCTION = "create_config"
|
||||
CATEGORY = "LightX2V/Config"
|
||||
|
||||
def create_config(
|
||||
self,
|
||||
optimization_level,
|
||||
attention_type,
|
||||
enable_rotary_chunk=False,
|
||||
rotary_chunk_size=100,
|
||||
clean_cuda_cache=False,
|
||||
enable_cpu_offload=False,
|
||||
offload_granularity="phase",
|
||||
offload_ratio=1.0,
|
||||
lazy_load=False,
|
||||
unload_after_inference=False,
|
||||
):
|
||||
config = {
|
||||
"optimization_level": optimization_level,
|
||||
"attention_type": attention_type,
|
||||
"enable_rotary_chunk": enable_rotary_chunk,
|
||||
"rotary_chunk_size": rotary_chunk_size,
|
||||
"clean_cuda_cache": clean_cuda_cache,
|
||||
"enable_cpu_offload": enable_cpu_offload,
|
||||
"offload_granularity": offload_granularity,
|
||||
"offload_ratio": offload_ratio,
|
||||
"lazy_load": lazy_load,
|
||||
"unload_after_inference": unload_after_inference,
|
||||
}
|
||||
return (config,)
|
||||
|
||||
|
||||
class LightX2VLightweightVAE:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"use_tiny_vae": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": False,
|
||||
"tooltip": "Use lightweight VAE to accelerate decoding",
|
||||
},
|
||||
),
|
||||
"use_tiling_vae": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": False,
|
||||
"tooltip": "Use VAE tiling inference to reduce VRAM usage",
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("VAE_CONFIG",)
|
||||
RETURN_NAMES = ("vae_config",)
|
||||
FUNCTION = "create_config"
|
||||
CATEGORY = "LightX2V/Config"
|
||||
|
||||
def create_config(self, use_tiny_vae, use_tiling_vae):
|
||||
config = {
|
||||
"use_tiny_vae": use_tiny_vae,
|
||||
"use_tiling_vae": use_tiling_vae,
|
||||
}
|
||||
return (config,)
|
||||
|
||||
|
||||
class LightX2VLoRALoader:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
available_loras = scan_loras()
|
||||
|
||||
return {
|
||||
"required": {
|
||||
"lora_name": (
|
||||
available_loras,
|
||||
{
|
||||
"default": available_loras[0],
|
||||
"tooltip": "Select LoRA from available LoRAs",
|
||||
},
|
||||
),
|
||||
"strength": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 1.0,
|
||||
"min": 0.0,
|
||||
"max": 2.0,
|
||||
"step": 0.1,
|
||||
"tooltip": "LoRA strength",
|
||||
},
|
||||
),
|
||||
},
|
||||
"optional": {
|
||||
"lora_chain": (
|
||||
"LORA_CHAIN",
|
||||
{"tooltip": "Previous LoRA chain to append to"},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("LORA_CHAIN",)
|
||||
RETURN_NAMES = ("lora_chain",)
|
||||
FUNCTION = "load_lora"
|
||||
CATEGORY = "LightX2V/LoRA"
|
||||
|
||||
def load_lora(self, lora_name, strength, lora_chain=None):
|
||||
if lora_chain is None:
|
||||
lora_chain = []
|
||||
else:
|
||||
lora_chain = lora_chain.copy()
|
||||
|
||||
lora_path = get_lora_full_path(lora_name)
|
||||
|
||||
if lora_path:
|
||||
lora_config = {"path": lora_path, "strength": strength}
|
||||
lora_chain.append(lora_config)
|
||||
|
||||
return (lora_chain,)
|
||||
|
||||
|
||||
class LightX2VConfigCombiner:
|
||||
def __init__(self):
|
||||
self.config_manager = ModularConfigManager()
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"inference_config": (
|
||||
"INFERENCE_CONFIG",
|
||||
{"tooltip": "Basic inference configuration"},
|
||||
),
|
||||
},
|
||||
"optional": {
|
||||
"teacache_config": (
|
||||
"TEACACHE_CONFIG",
|
||||
{"tooltip": "TeaCache configuration"},
|
||||
),
|
||||
"quantization_config": (
|
||||
"QUANT_CONFIG",
|
||||
{"tooltip": "Quantization configuration"},
|
||||
),
|
||||
"memory_config": (
|
||||
"MEMORY_CONFIG",
|
||||
{"tooltip": "Memory optimization configuration"},
|
||||
),
|
||||
"vae_config": ("VAE_CONFIG", {"tooltip": "VAE configuration"}),
|
||||
"lora_chain": ("LORA_CHAIN", {"tooltip": "LoRA chain configuration"}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("COMBINED_CONFIG",)
|
||||
RETURN_NAMES = ("combined_config",)
|
||||
FUNCTION = "combine_configs"
|
||||
CATEGORY = "LightX2V/Config"
|
||||
|
||||
def combine_configs(
|
||||
self,
|
||||
inference_config,
|
||||
teacache_config=None,
|
||||
quantization_config=None,
|
||||
memory_config=None,
|
||||
vae_config=None,
|
||||
lora_chain=None,
|
||||
):
|
||||
configs = {
|
||||
"inference": inference_config,
|
||||
}
|
||||
|
||||
if teacache_config:
|
||||
configs["teacache"] = teacache_config
|
||||
if quantization_config:
|
||||
configs["quantization"] = quantization_config
|
||||
if memory_config:
|
||||
configs["memory"] = memory_config
|
||||
if vae_config:
|
||||
configs["vae"] = vae_config
|
||||
|
||||
config = self.config_manager.build_final_config(configs)
|
||||
|
||||
if lora_chain:
|
||||
config.lora_configs = lora_chain
|
||||
|
||||
return (config,)
|
||||
|
||||
|
||||
class LightX2VModularInference:
|
||||
def __init__(self):
|
||||
self._current_runner = None
|
||||
self._current_config_hash = None
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"combined_config": (
|
||||
"COMBINED_CONFIG",
|
||||
{"tooltip": "Combined configuration from config combiner"},
|
||||
),
|
||||
"prompt": (
|
||||
"STRING",
|
||||
{"multiline": True, "default": "", "tooltip": "Generation prompt"},
|
||||
),
|
||||
"negative_prompt": (
|
||||
"STRING",
|
||||
{"multiline": True, "default": "", "tooltip": "Negative prompt"},
|
||||
),
|
||||
},
|
||||
"optional": {
|
||||
"image": ("IMAGE", {"tooltip": "Input image for i2v task"}),
|
||||
"audio": (
|
||||
"AUDIO",
|
||||
{"tooltip": "Input audio for audio-driven generation"},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
RETURN_NAMES = ("images",)
|
||||
FUNCTION = "generate"
|
||||
CATEGORY = "LightX2V/Inference"
|
||||
|
||||
def _get_config_hash(self, config) -> str:
|
||||
relevant_configs = {
|
||||
"model_cls": getattr(config, "model_cls", None),
|
||||
"model_path": getattr(config, "model_path", None),
|
||||
"dit_quantized": getattr(config, "dit_quantized", False),
|
||||
"t5_quantized": getattr(config, "t5_quantized", False),
|
||||
"clip_quantized": getattr(config, "clip_quantized", False),
|
||||
"lora_configs": getattr(config, "lora_configs", None),
|
||||
"lazy_load": getattr(config, "lazy_load", False),
|
||||
}
|
||||
|
||||
config_str = json.dumps(relevant_configs, sort_keys=True)
|
||||
return hashlib.md5(config_str.encode()).hexdigest()
|
||||
|
||||
def generate(
|
||||
self,
|
||||
combined_config,
|
||||
prompt,
|
||||
negative_prompt,
|
||||
image=None,
|
||||
audio=None,
|
||||
**kwargs,
|
||||
):
|
||||
os.environ["TOKENIZERS_PARALLELISM"] = "false"
|
||||
if "DTYPE" not in os.environ:
|
||||
os.environ["DTYPE"] = "BF16"
|
||||
if "ENABLE_GRAPH_MODE" not in os.environ:
|
||||
os.environ["ENABLE_GRAPH_MODE"] = "false"
|
||||
if "ENABLE_PROFILING_DEBUG" not in os.environ:
|
||||
os.environ["ENABLE_PROFILING_DEBUG"] = "true"
|
||||
|
||||
config = combined_config
|
||||
|
||||
config.prompt = prompt
|
||||
config.negative_prompt = negative_prompt
|
||||
|
||||
if config.task == "i2v" and image is None:
|
||||
raise ValueError("i2v task requires input image")
|
||||
|
||||
temp_files = []
|
||||
|
||||
try:
|
||||
if config.task == "i2v" and image is not None:
|
||||
image_np = (image[0].cpu().numpy() * 255).astype(np.uint8)
|
||||
pil_image = Image.fromarray(image_np)
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as tmp:
|
||||
pil_image.save(tmp.name)
|
||||
config.image_path = tmp.name
|
||||
temp_files.append(tmp.name)
|
||||
|
||||
if (
|
||||
audio is not None
|
||||
and hasattr(config, "model_cls")
|
||||
and "audio" in config.model_cls
|
||||
):
|
||||
if isinstance(audio, tuple) and len(audio) == 2:
|
||||
waveform, sample_rate = audio
|
||||
|
||||
if isinstance(waveform, torch.Tensor):
|
||||
waveform = waveform.cpu().numpy()
|
||||
|
||||
with tempfile.NamedTemporaryFile(
|
||||
suffix=".wav", delete=False
|
||||
) as tmp:
|
||||
try:
|
||||
import scipy.io.wavfile as wavfile
|
||||
except ImportError:
|
||||
import wave
|
||||
|
||||
with wave.open(tmp.name, "wb") as wav_file:
|
||||
wav_file.setnchannels(
|
||||
1 if waveform.ndim == 1 else waveform.shape[-1]
|
||||
)
|
||||
wav_file.setsampwidth(2) # 16-bit
|
||||
wav_file.setframerate(sample_rate)
|
||||
if waveform.dtype != np.int16:
|
||||
waveform = (waveform * 32767).astype(np.int16)
|
||||
wav_file.writeframes(waveform.tobytes())
|
||||
else:
|
||||
if waveform.ndim == 1:
|
||||
wavfile.write(tmp.name, sample_rate, waveform)
|
||||
else:
|
||||
if waveform.shape[0] < waveform.shape[1]:
|
||||
waveform = waveform.T
|
||||
wavfile.write(tmp.name, sample_rate, waveform)
|
||||
|
||||
config.audio_path = tmp.name
|
||||
temp_files.append(tmp.name)
|
||||
|
||||
config_hash = self._get_config_hash(config)
|
||||
needs_reinit = (
|
||||
self._current_runner is None
|
||||
or self._current_config_hash != config_hash
|
||||
or getattr(config, "lazy_load", False)
|
||||
)
|
||||
|
||||
if needs_reinit:
|
||||
if self._current_runner is not None:
|
||||
del self._current_runner
|
||||
torch.cuda.empty_cache()
|
||||
gc.collect()
|
||||
|
||||
self._current_runner = init_runner(config)
|
||||
self._current_config_hash = config_hash
|
||||
else:
|
||||
if hasattr(self._current_runner, "config"):
|
||||
self._current_runner.config = config
|
||||
|
||||
total_steps = getattr(config, "infer_steps", 40)
|
||||
progress = ProgressBar(total_steps)
|
||||
|
||||
def update_progress(current_step, total):
|
||||
progress.update_absolute(current_step)
|
||||
|
||||
if hasattr(self._current_runner, "set_progress_callback"):
|
||||
self._current_runner.set_progress_callback(update_progress)
|
||||
|
||||
if hasattr(self._current_runner, "run_pipeline"):
|
||||
images = self._current_runner.run_pipeline(save_video=False)
|
||||
else:
|
||||
images = self._current_runner()
|
||||
|
||||
if getattr(config, "unload_after_inference", False):
|
||||
del self._current_runner
|
||||
self._current_runner = None
|
||||
self._current_config_hash = None
|
||||
|
||||
torch.cuda.empty_cache()
|
||||
gc.collect()
|
||||
|
||||
images = (images + 1) / 2
|
||||
images = images.squeeze(0).permute(1, 2, 3, 0).cpu()
|
||||
images = torch.clamp(images, 0, 1)
|
||||
|
||||
return (images,)
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Error during inference: {e}")
|
||||
raise
|
||||
|
||||
# finally:
|
||||
# for temp_file in temp_files:
|
||||
# if os.path.exists(temp_file):
|
||||
# try:
|
||||
# os.unlink(temp_file)
|
||||
# except Exception:
|
||||
# pass
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"LightX2VInferenceConfig": LightX2VInferenceConfig,
|
||||
"LightX2VTeaCache": LightX2VTeaCache,
|
||||
"LightX2VQuantization": LightX2VQuantization,
|
||||
"LightX2VMemoryOptimization": LightX2VMemoryOptimization,
|
||||
"LightX2VLightweightVAE": LightX2VLightweightVAE,
|
||||
"LightX2VLoRALoader": LightX2VLoRALoader,
|
||||
"LightX2VConfigCombiner": LightX2VConfigCombiner,
|
||||
"LightX2VModularInference": LightX2VModularInference,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"LightX2VInferenceConfig": "LightX2V Inference Config",
|
||||
"LightX2VTeaCache": "LightX2V TeaCache",
|
||||
"LightX2VQuantization": "LightX2V Quantization",
|
||||
"LightX2VMemoryOptimization": "LightX2V Memory Optimization",
|
||||
"LightX2VLightweightVAE": "LightX2V Lightweight VAE",
|
||||
"LightX2VLoRALoader": "LightX2V LoRA Loader",
|
||||
"LightX2VConfigCombiner": "LightX2V Config Combiner",
|
||||
"LightX2VModularInference": "LightX2V Modular Inference",
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
"""ComfyUI node definitions for LightX2V.
|
||||
|
||||
Each submodule groups a category of nodes:
|
||||
- ``config`` : per-feature configuration nodes (inference / teacache / quant / memory)
|
||||
- ``lora`` : LoRA chain loader
|
||||
- ``talk`` : talk-object input/combiner nodes
|
||||
- ``combiner`` : config combiners (V1/V2/V3) that aggregate the above
|
||||
- ``inference`` : the modular inference runner
|
||||
- ``file_input``: validated, upload-backed media paths
|
||||
- ``seedvr`` : SeedVR2 super-resolution runner
|
||||
- ``swiftvr`` : SwiftVR restoration runner
|
||||
"""
|
||||
|
||||
from .combiner import (
|
||||
LightX2VConfigCombinerV2,
|
||||
LightX2VConfigCombinerV3,
|
||||
)
|
||||
from .config import (
|
||||
LightX2VInferenceConfig,
|
||||
LightX2VMemoryOptimization,
|
||||
LightX2VQuantization,
|
||||
LightX2VTeaCache,
|
||||
)
|
||||
from .file_input import LightX2VInputVideoPath
|
||||
from .inference import LightX2VModularInferenceV2
|
||||
from .lora import LightX2VLoRALoader
|
||||
from .seedvr import LightX2VOutputVideoPreview, LightX2VSeedVR2FileSampler, LightX2VSeedVR2Loader, LightX2VSeedVR2Sampler
|
||||
from .swiftvr import LightX2VSwiftVRFileSampler, LightX2VSwiftVRLoader, LightX2VSwiftVRSampler
|
||||
from .talk import (
|
||||
TalkObjectInput,
|
||||
TalkObjectsCombiner,
|
||||
TalkObjectsFromFiles,
|
||||
TalkObjectsFromJSON,
|
||||
)
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"LightX2VInferenceConfig": LightX2VInferenceConfig,
|
||||
"LightX2VTeaCache": LightX2VTeaCache,
|
||||
"LightX2VQuantization": LightX2VQuantization,
|
||||
"LightX2VMemoryOptimization": LightX2VMemoryOptimization,
|
||||
"LightX2VLoRALoader": LightX2VLoRALoader,
|
||||
"LightX2VConfigCombinerV2": LightX2VConfigCombinerV2,
|
||||
"LightX2VConfigCombinerV3": LightX2VConfigCombinerV3,
|
||||
"LightX2VModularInferenceV2": LightX2VModularInferenceV2,
|
||||
"LightX2VInputVideoPath": LightX2VInputVideoPath,
|
||||
"LightX2VSeedVR2Loader": LightX2VSeedVR2Loader,
|
||||
"LightX2VSeedVR2Sampler": LightX2VSeedVR2Sampler,
|
||||
"LightX2VSeedVR2FileSampler": LightX2VSeedVR2FileSampler,
|
||||
"LightX2VSwiftVRLoader": LightX2VSwiftVRLoader,
|
||||
"LightX2VSwiftVRSampler": LightX2VSwiftVRSampler,
|
||||
"LightX2VSwiftVRFileSampler": LightX2VSwiftVRFileSampler,
|
||||
"LightX2VOutputVideoPreview": LightX2VOutputVideoPreview,
|
||||
"LightX2VTalkObjectInput": TalkObjectInput,
|
||||
"LightX2VTalkObjectsCombiner": TalkObjectsCombiner,
|
||||
"LightX2VTalkObjectsFromJSON": TalkObjectsFromJSON,
|
||||
"LightX2VTalkObjectsFromFiles": TalkObjectsFromFiles,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"LightX2VInferenceConfig": "LightX2V Inference Config",
|
||||
"LightX2VTeaCache": "LightX2V TeaCache",
|
||||
"LightX2VQuantization": "LightX2V Quantization",
|
||||
"LightX2VMemoryOptimization": "LightX2V Memory Optimization",
|
||||
"LightX2VLoRALoader": "LightX2V LoRA Loader",
|
||||
"LightX2VConfigCombinerV2": "LightX2V Config Combiner V2",
|
||||
"LightX2VConfigCombinerV3": "LightX2V Config Combiner V3",
|
||||
"LightX2VModularInferenceV2": "LightX2V Modular Inference V2",
|
||||
"LightX2VInputVideoPath": "LightX2V Input Video Path (Upload)",
|
||||
"LightX2VSeedVR2Loader": "LightX2V SeedVR2 Loader",
|
||||
"LightX2VSeedVR2Sampler": "LightX2V SeedVR2 Sampler",
|
||||
"LightX2VSeedVR2FileSampler": "LightX2V SeedVR2 File Sampler",
|
||||
"LightX2VSwiftVRLoader": "LightX2V SwiftVR Loader",
|
||||
"LightX2VSwiftVRSampler": "LightX2V SwiftVR Sampler",
|
||||
"LightX2VSwiftVRFileSampler": "LightX2V SwiftVR File Sampler",
|
||||
"LightX2VOutputVideoPreview": "LightX2V Output Video Preview",
|
||||
"LightX2VTalkObjectInput": "LightX2V Talk Object Input (Single)",
|
||||
"LightX2VTalkObjectsCombiner": "LightX2V Talk Objects Combiner",
|
||||
"LightX2VTalkObjectsFromFiles": "LightX2V Talk Objects From Files",
|
||||
"LightX2VTalkObjectsFromJSON": "LightX2V Talk Objects From JSON (API)",
|
||||
}
|
||||
|
||||
__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"]
|
||||
@@ -0,0 +1,127 @@
|
||||
"""In-memory shim for lightx2v.utils.audio_io.load_audio_file.
|
||||
|
||||
Routes ComfyUI AUDIO tensors into the runner without round-tripping through
|
||||
a temp WAV file. Sentinel paths start with SENTINEL_PREFIX; the patched
|
||||
loader returns a stashed tensor instead of touching disk.
|
||||
|
||||
Scope: only the single-AUDIO ComfyUI input path uses this shim. V3 multi-talker
|
||||
padding still writes real WAV files (its inputs are external file paths / URLs,
|
||||
not ComfyUI tensors), and those reads fall through to the original loader.
|
||||
|
||||
Patching strategy: lightx2v consumers do `from lightx2v.utils.audio_io import
|
||||
load_audio_file`, which captures the original function at import time. So
|
||||
patching only `lightx2v.utils.audio_io.load_audio_file` would miss them — we
|
||||
rebind every known import site too. New consumers must be added to _PATCH_SITES.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import uuid
|
||||
from typing import Dict, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
SENTINEL_PREFIX = "<lightx2v-mem-audio:"
|
||||
SENTINEL_SUFFIX = ">"
|
||||
|
||||
_REGISTRY: Dict[str, Tuple[torch.Tensor, int]] = {}
|
||||
_LOCK = threading.Lock()
|
||||
_PATCHED = False
|
||||
|
||||
# (module dotted-path, attribute name). Each entry rebinds that module's
|
||||
# `load_audio_file` attribute to the shim. Add new early-binders here as
|
||||
# upstream changes.
|
||||
_PATCH_SITES = (
|
||||
("lightx2v.utils.audio_io", "load_audio_file"),
|
||||
("lightx2v.models.runners.wan.wan_audio_runner", "load_audio_file"),
|
||||
("lightx2v.shot_runner.rs2v_infer", "load_audio_file"),
|
||||
("lightx2v.shot_runner.stream_infer", "load_audio_file"),
|
||||
)
|
||||
|
||||
|
||||
def register(waveform: torch.Tensor, sample_rate: int) -> str:
|
||||
"""Register a [C, T] float32 waveform; return a sentinel "path" for it."""
|
||||
if waveform.dim() != 2:
|
||||
raise ValueError(f"waveform must be [C, T]; got shape {tuple(waveform.shape)}")
|
||||
token = uuid.uuid4().hex
|
||||
sentinel = f"{SENTINEL_PREFIX}{token}{SENTINEL_SUFFIX}"
|
||||
stashed = waveform.detach().to(torch.float32).cpu().contiguous()
|
||||
with _LOCK:
|
||||
_REGISTRY[sentinel] = (stashed, int(sample_rate))
|
||||
return sentinel
|
||||
|
||||
|
||||
def release(sentinel: str) -> None:
|
||||
with _LOCK:
|
||||
_REGISTRY.pop(sentinel, None)
|
||||
|
||||
|
||||
def is_sentinel(path) -> bool:
|
||||
return isinstance(path, str) and path.startswith(SENTINEL_PREFIX)
|
||||
|
||||
|
||||
def comfyui_audio_to_loader_pair(audio_dict) -> Tuple[torch.Tensor, int]:
|
||||
"""ComfyUI AUDIO {"waveform": [B, C, T], "sample_rate": int} -> ([C, T], sr).
|
||||
|
||||
ComfyUI gives waveform as [B, C, T] float32 in [-1, 1] (B usually 1).
|
||||
lightx2v's load_audio_file returns [C, T] (channels_first=True). We
|
||||
squeeze the batch dim here; mono-down and resample happen downstream in
|
||||
AudioProcessor / ShotRS2VPipeline so the shim stays format-agnostic.
|
||||
"""
|
||||
waveform = audio_dict["waveform"]
|
||||
if waveform.dim() == 3:
|
||||
waveform = waveform[0]
|
||||
if waveform.dim() != 2:
|
||||
raise ValueError(f"Unexpected ComfyUI AUDIO waveform shape {tuple(waveform.shape)}")
|
||||
return waveform, int(audio_dict["sample_rate"])
|
||||
|
||||
|
||||
def install() -> None:
|
||||
"""Idempotent monkey-patch. Safe to call multiple times."""
|
||||
global _PATCHED
|
||||
if _PATCHED:
|
||||
return
|
||||
import importlib
|
||||
|
||||
original = None
|
||||
targets = []
|
||||
for module_path, attr in _PATCH_SITES:
|
||||
try:
|
||||
module = importlib.import_module(module_path)
|
||||
except ImportError:
|
||||
logging.warning(f"_audio_shim: {module_path} not importable; skipping")
|
||||
continue
|
||||
fn = getattr(module, attr, None)
|
||||
if fn is None:
|
||||
logging.warning(f"_audio_shim: {module_path}.{attr} missing; skipping")
|
||||
continue
|
||||
if original is None:
|
||||
original = fn
|
||||
targets.append((module, attr))
|
||||
|
||||
if original is None:
|
||||
raise RuntimeError("_audio_shim.install: no patch sites resolved; lightx2v not installed?")
|
||||
|
||||
def _patched(uri, frame_offset: int = 0, num_frames: int = -1, channels_first: bool = True):
|
||||
if is_sentinel(uri):
|
||||
with _LOCK:
|
||||
entry = _REGISTRY.get(uri)
|
||||
if entry is None:
|
||||
raise FileNotFoundError(f"Stale lightx2v in-memory audio sentinel: {uri}")
|
||||
tensor, sr = entry
|
||||
# Slice semantics mirror torchaudio.load(frame_offset, num_frames).
|
||||
if frame_offset > 0 or num_frames > 0:
|
||||
end = tensor.shape[-1] if num_frames < 0 else frame_offset + num_frames
|
||||
tensor = tensor[..., frame_offset:end]
|
||||
if not channels_first:
|
||||
tensor = tensor.transpose(0, 1).contiguous()
|
||||
return tensor, sr
|
||||
return original(uri, frame_offset=frame_offset, num_frames=num_frames, channels_first=channels_first)
|
||||
|
||||
for module, attr in targets:
|
||||
setattr(module, attr, _patched)
|
||||
|
||||
_PATCHED = True
|
||||
logging.info(f"_audio_shim installed at {len(targets)} site(s)")
|
||||
@@ -0,0 +1,530 @@
|
||||
"""Config combiner nodes.
|
||||
|
||||
- V2 ``LightX2VConfigCombinerV2`` : config aggregation + data prep (image/audio/talk_objects),
|
||||
emits ``PREPARED_CONFIG``.
|
||||
- V3 ``LightX2VConfigCombinerV3`` : V2 + equal-duration audio padding and background-mask
|
||||
synthesis for multi-talker setups (used when the user's
|
||||
per-speaker audios differ in length and must be aligned).
|
||||
|
||||
V2 and V3 share INPUT_TYPES and most of ``prepare_config``; the shared scaffolding lives
|
||||
in the private ``_BaseConfigCombiner`` below. V3 only overrides the multi-talker branch
|
||||
to add padding + background track synthesis.
|
||||
"""
|
||||
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess as sp
|
||||
import wave
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
from ..config_builder import ConfigBuilder
|
||||
from ..data_models import (
|
||||
InferenceConfig,
|
||||
MemoryOptimizationConfig,
|
||||
QuantizationConfig,
|
||||
TeaCacheConfig,
|
||||
)
|
||||
from ..file_handlers import (
|
||||
AudioFileHandler,
|
||||
ComfyUIFileResolver,
|
||||
HTTPFileDownloader,
|
||||
ImageFileHandler,
|
||||
TempFileManager,
|
||||
)
|
||||
|
||||
|
||||
class _BaseConfigCombiner:
|
||||
"""Shared scaffolding for V2 / V3.
|
||||
|
||||
Subclasses must implement ``_process_talk_objects(src_objects, max_duration)``
|
||||
returning the final ``processed_talk_objects`` list (V2 passes through;
|
||||
V3 pads to equal length and appends a background talker).
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.config_builder = ConfigBuilder()
|
||||
self.temp_manager = TempFileManager()
|
||||
self.image_handler = ImageFileHandler()
|
||||
self.audio_handler = AudioFileHandler()
|
||||
self.resolver = ComfyUIFileResolver()
|
||||
self.http_downloader = HTTPFileDownloader()
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"inference_config": (
|
||||
"INFERENCE_CONFIG",
|
||||
{"tooltip": "Basic inference configuration"},
|
||||
),
|
||||
"prompt": (
|
||||
"STRING",
|
||||
{"multiline": True, "default": "", "tooltip": "Generation prompt"},
|
||||
),
|
||||
"negative_prompt": (
|
||||
"STRING",
|
||||
{"multiline": True, "default": "", "tooltip": "Negative prompt"},
|
||||
),
|
||||
},
|
||||
"optional": {
|
||||
"teacache_config": (
|
||||
"TEACACHE_CONFIG",
|
||||
{"tooltip": "TeaCache configuration"},
|
||||
),
|
||||
"quantization_config": (
|
||||
"QUANT_CONFIG",
|
||||
{"tooltip": "Quantization configuration"},
|
||||
),
|
||||
"memory_config": (
|
||||
"MEMORY_CONFIG",
|
||||
{"tooltip": "Memory optimization configuration"},
|
||||
),
|
||||
"lora_chain": ("LORA_CHAIN", {"tooltip": "LoRA chain configuration"}),
|
||||
"talk_objects_config": ("TALK_OBJECTS_CONFIG", {"tooltip": "Talk objects configuration"}),
|
||||
"image": ("IMAGE", {"tooltip": "Input image for i2v or s2v or rs2v task"}),
|
||||
"audio": (
|
||||
"AUDIO",
|
||||
{"tooltip": "Input audio for audio-driven generation for s2v or rs2v task"},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("PREPARED_CONFIG",)
|
||||
RETURN_NAMES = ("prepared_config",)
|
||||
FUNCTION = "prepare_config"
|
||||
CATEGORY = "LightX2V/ConfigV2"
|
||||
|
||||
# --- pipeline ---------------------------------------------------------
|
||||
|
||||
def prepare_config(
|
||||
self,
|
||||
inference_config,
|
||||
prompt,
|
||||
negative_prompt,
|
||||
teacache_config=None,
|
||||
quantization_config=None,
|
||||
memory_config=None,
|
||||
lora_chain=None,
|
||||
talk_objects_config=None,
|
||||
image=None,
|
||||
audio=None,
|
||||
):
|
||||
config = self._build_base_config(
|
||||
inference_config,
|
||||
prompt,
|
||||
negative_prompt,
|
||||
teacache_config,
|
||||
quantization_config,
|
||||
memory_config,
|
||||
lora_chain,
|
||||
talk_objects_config,
|
||||
)
|
||||
self._save_image_if_needed(config, image)
|
||||
self._save_single_audio_if_needed(config, audio)
|
||||
self._handle_talk_objects(config)
|
||||
|
||||
logging.info("lightx2v prepared config: " + json.dumps(config, indent=2, ensure_ascii=False))
|
||||
return (config,)
|
||||
|
||||
# --- shared helpers ---------------------------------------------------
|
||||
|
||||
def _build_base_config(
|
||||
self,
|
||||
inference_config,
|
||||
prompt,
|
||||
negative_prompt,
|
||||
teacache_config,
|
||||
quantization_config,
|
||||
memory_config,
|
||||
lora_chain,
|
||||
talk_objects_config,
|
||||
):
|
||||
inf_config = InferenceConfig(**inference_config) if isinstance(inference_config, dict) else inference_config
|
||||
tea_config = TeaCacheConfig(**teacache_config) if teacache_config and isinstance(teacache_config, dict) else teacache_config
|
||||
quant_config = (
|
||||
QuantizationConfig(**quantization_config) if quantization_config and isinstance(quantization_config, dict) else quantization_config
|
||||
)
|
||||
mem_config = MemoryOptimizationConfig(**memory_config) if memory_config and isinstance(memory_config, dict) else memory_config
|
||||
|
||||
config = self.config_builder.combine_configs(
|
||||
inference_config=inf_config,
|
||||
teacache_config=tea_config,
|
||||
quantization_config=quant_config,
|
||||
memory_config=mem_config,
|
||||
lora_chain=lora_chain,
|
||||
talk_objects_config=talk_objects_config,
|
||||
)
|
||||
config.prompt = prompt
|
||||
config.negative_prompt = negative_prompt
|
||||
return config
|
||||
|
||||
def _save_image_if_needed(self, config, image):
|
||||
if config.task not in ["i2v", "s2v", "rs2v"]:
|
||||
return
|
||||
if image is None:
|
||||
raise ValueError("i2v or s2v or rs2v task requires input image")
|
||||
|
||||
image_np = (image[0].cpu().numpy() * 255).astype(np.uint8)
|
||||
pil_image = Image.fromarray(image_np)
|
||||
temp_path = self.temp_manager.create_temp_file(suffix=".png")
|
||||
pil_image.save(temp_path)
|
||||
config.image_path = temp_path
|
||||
logging.info(f"Image saved to {temp_path}")
|
||||
|
||||
def _save_single_audio_if_needed(self, config, audio):
|
||||
# Route ComfyUI AUDIO straight into the runner via the in-memory shim
|
||||
# (no WAV temp file, no soundfile round-trip). The shim only intercepts
|
||||
# this single-AUDIO path; V3 multi-talker padding still produces real
|
||||
# files since its inputs are external paths/URLs, not ComfyUI tensors.
|
||||
if audio is None or not hasattr(config, "model_cls") or "seko" not in config.model_cls:
|
||||
return
|
||||
from ._audio_shim import comfyui_audio_to_loader_pair, install, register
|
||||
|
||||
install()
|
||||
waveform, sr = comfyui_audio_to_loader_pair(audio)
|
||||
sentinel = register(waveform, sr)
|
||||
config.audio_path = sentinel
|
||||
logging.info(f"Routed ComfyUI AUDIO ({waveform.shape[0]}ch @ {sr}Hz, {waveform.shape[1]} samples) via in-memory shim")
|
||||
|
||||
def _handle_talk_objects(self, config):
|
||||
if not getattr(config, "talk_objects", None):
|
||||
return
|
||||
|
||||
src_objects, max_duration = self._resolve_talk_object_paths(config.talk_objects)
|
||||
processed_objects = self._process_talk_objects(src_objects, max_duration)
|
||||
self._commit_talk_objects(config, processed_objects)
|
||||
|
||||
def _resolve_talk_object_paths(self, talk_objects):
|
||||
"""Pull (audio, optional mask) per talker; resolve URLs and ComfyUI-relative paths.
|
||||
|
||||
Always captures per-object duration so subclasses that pad can use it; V2 ignores it.
|
||||
Returns ``(src_objects, max_duration)``.
|
||||
"""
|
||||
src_objects = []
|
||||
for talk_obj in talk_objects:
|
||||
obj = {}
|
||||
if "audio" in talk_obj:
|
||||
obj["audio"] = talk_obj["audio"]
|
||||
if "mask" in talk_obj:
|
||||
obj["mask"] = talk_obj["mask"]
|
||||
if "audio" in obj:
|
||||
src_objects.append(obj)
|
||||
|
||||
max_duration = None
|
||||
for obj in src_objects:
|
||||
audio_path = obj.get("audio")
|
||||
if audio_path:
|
||||
obj["audio"] = self._resolve_one_asset(audio_path, kind="audio")
|
||||
if obj["audio"] and os.path.exists(obj["audio"]):
|
||||
try:
|
||||
duration = self._probe_audio_duration(obj["audio"])
|
||||
obj["duration"] = duration
|
||||
if max_duration is None or duration > max_duration:
|
||||
max_duration = duration
|
||||
except Exception as e:
|
||||
logging.warning(f"Failed to probe audio duration for {obj['audio']}: {e}")
|
||||
|
||||
mask_path = obj.get("mask")
|
||||
if mask_path:
|
||||
obj["mask"] = self._resolve_one_asset(mask_path, kind="mask")
|
||||
|
||||
return src_objects, max_duration
|
||||
|
||||
def _resolve_one_asset(self, path, kind):
|
||||
"""Resolve URL → downloaded path; resolve ComfyUI-relative → absolute. Warn on missing."""
|
||||
if self.http_downloader.is_url(path):
|
||||
try:
|
||||
downloaded = self.http_downloader.download_if_url(path, prefix=kind)
|
||||
logging.info(f"Downloaded {kind} from URL: {path} -> {downloaded}")
|
||||
path = downloaded
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to download {kind} from {path}: {e}")
|
||||
return path
|
||||
elif not os.path.isabs(path) and not path.startswith("/tmp"):
|
||||
resolved = self.resolver.resolve_input_path(path)
|
||||
logging.info(f"Resolved {kind} path: {path} -> {resolved}")
|
||||
path = resolved
|
||||
|
||||
if not os.path.exists(path):
|
||||
logging.warning(f"{kind.capitalize()} file not found: {path}")
|
||||
return path
|
||||
|
||||
@staticmethod
|
||||
def _probe_audio_duration(input_path: str) -> float:
|
||||
cmd_probe = [
|
||||
"ffprobe",
|
||||
"-v",
|
||||
"error",
|
||||
"-select_streams",
|
||||
"a:0",
|
||||
"-show_entries",
|
||||
"stream=duration,sample_rate,bit_rate,channels",
|
||||
"-of",
|
||||
"json",
|
||||
input_path,
|
||||
]
|
||||
output = sp.check_output(cmd_probe, encoding="utf-8", errors="replace")
|
||||
data = json.loads(output)
|
||||
streams = data.get("streams", [])
|
||||
if not streams:
|
||||
raise ValueError(f"Failed to get audio stream information: {input_path}")
|
||||
return float(streams[0].get("duration", 0))
|
||||
|
||||
def _commit_talk_objects(self, config, processed_objects):
|
||||
"""Single talker w/o mask → set audio_path directly. Otherwise dump talk_objects.json."""
|
||||
if not processed_objects:
|
||||
return
|
||||
if len(processed_objects) == 1 and not processed_objects[0].get("mask", "").strip():
|
||||
config.audio_path = processed_objects[0]["audio"]
|
||||
logging.info(f"Convert Processed 1 talk object to audio path: {config.audio_path}")
|
||||
return
|
||||
|
||||
temp_dir = self.temp_manager.create_temp_dir()
|
||||
with open(os.path.join(temp_dir, "config.json"), "w") as f:
|
||||
json.dump({"talk_objects": processed_objects}, f)
|
||||
config.audio_path = temp_dir
|
||||
logging.info(f"Processed {len(processed_objects)} talk objects")
|
||||
|
||||
# --- hook for subclasses ---------------------------------------------
|
||||
|
||||
def _process_talk_objects(self, src_objects, max_duration):
|
||||
"""Default: pass through. V3 overrides this to pad + synthesize a bg talker."""
|
||||
return src_objects
|
||||
|
||||
|
||||
class LightX2VConfigCombinerV2(_BaseConfigCombiner):
|
||||
"""Aggregates configs and prepares image/audio/talk_objects. No multi-talker padding."""
|
||||
|
||||
# Inherits everything; explicit no-op override here so the class isn't empty
|
||||
# and so the per-class identity / categorization stay distinct from V3.
|
||||
pass
|
||||
|
||||
|
||||
class LightX2VConfigCombinerV3(_BaseConfigCombiner):
|
||||
"""V2 + equal-duration audio padding and background-mask synthesis for multi-talker setups."""
|
||||
|
||||
def _process_talk_objects(self, src_objects, max_duration):
|
||||
if len(src_objects) <= 1:
|
||||
return src_objects
|
||||
return self._pad_and_synthesize_bg(src_objects, max_duration)
|
||||
|
||||
# --- V3-only multi-talker alignment ----------------------------------
|
||||
|
||||
def _pad_and_synthesize_bg(self, src_objects, max_duration):
|
||||
"""Pad each talker's audio to ``max_duration`` and append a (bg_audio, bg_mask) talker.
|
||||
|
||||
The background talker carries silence-like white noise + a mask covering pixels
|
||||
that none of the per-speaker masks claim, so the runner has someone to "speak"
|
||||
for the rest of the frame.
|
||||
"""
|
||||
processed = []
|
||||
mask_img_paths = []
|
||||
extend_count = 0
|
||||
|
||||
for obj in src_objects:
|
||||
dst_obj = {"audio": obj["audio"]}
|
||||
src_audio_duration = obj.get("duration", max_duration)
|
||||
if max_duration - src_audio_duration > 0.1:
|
||||
dst_audio_path = self.temp_manager.create_temp_file(suffix=".mp3")
|
||||
self.extend_mp3(obj["audio"], dst_audio_path, max_duration)
|
||||
dst_obj["audio"] = dst_audio_path
|
||||
extend_count += 1
|
||||
src_mask = obj.get("mask")
|
||||
if src_mask:
|
||||
dst_obj["mask"] = src_mask
|
||||
mask_img_paths.append(src_mask)
|
||||
processed.append(dst_obj)
|
||||
logging.info(f"Extended {extend_count} audio files")
|
||||
|
||||
bg_mask_io = self.generate_background_mask(mask_img_paths)
|
||||
bg_mask_path = self.temp_manager.create_temp_file(suffix=".jpg")
|
||||
with open(bg_mask_path, "wb") as f:
|
||||
f.write(bg_mask_io.getvalue())
|
||||
|
||||
bg_noise = self.generate_white_noise(
|
||||
duration=max_duration,
|
||||
framerate=16000,
|
||||
n_channels=1,
|
||||
rms=0.00232,
|
||||
std_dev=0.00232,
|
||||
)
|
||||
wav_io = io.BytesIO()
|
||||
self.save_wav_file(audio_data=bg_noise, output_path=wav_io, framerate=16000, sample_width=2)
|
||||
bg_audio_path = self.temp_manager.create_temp_file(suffix=".wav")
|
||||
with open(bg_audio_path, "wb") as f:
|
||||
f.write(wav_io.getvalue())
|
||||
|
||||
processed.append({"audio": bg_audio_path, "mask": bg_mask_path})
|
||||
logging.info(f"Generated background mask and audio: {bg_mask_path}, {bg_audio_path}")
|
||||
return processed
|
||||
|
||||
# --- V3-only static utilities (kept here, not on the base) -----------
|
||||
|
||||
@staticmethod
|
||||
def extend_mp3(input_path: str, output_path: str, duration: float) -> bool:
|
||||
"""Pad audio to ``duration`` seconds; truncate if input is at most 0.1s longer.
|
||||
|
||||
Errors if input exceeds duration by more than 0.1s.
|
||||
"""
|
||||
cmd_probe = [
|
||||
"ffprobe",
|
||||
"-v",
|
||||
"error",
|
||||
"-select_streams",
|
||||
"a:0",
|
||||
"-show_entries",
|
||||
"stream=duration,sample_rate,bit_rate,channels",
|
||||
"-of",
|
||||
"json",
|
||||
input_path,
|
||||
]
|
||||
try:
|
||||
output = sp.check_output(cmd_probe, encoding="utf-8", errors="replace")
|
||||
data = json.loads(output)
|
||||
streams = data.get("streams", [])
|
||||
if not streams:
|
||||
raise ValueError(f"Failed to get audio stream information: {input_path}")
|
||||
|
||||
stream_info = streams[0]
|
||||
input_duration = float(stream_info.get("duration", 0))
|
||||
sample_rate = stream_info.get("sample_rate", "44100")
|
||||
bit_rate = stream_info.get("bit_rate", "128000")
|
||||
channels = stream_info.get("channels", 2)
|
||||
|
||||
if input_duration > duration:
|
||||
raise ValueError(f"Input audio duration ({input_duration:.2f}s) exceeds target duration + 0.1s ({duration + 0.1:.2f}s)")
|
||||
pad_duration = duration - input_duration
|
||||
cmd = [
|
||||
"ffmpeg",
|
||||
"-i",
|
||||
input_path,
|
||||
"-af",
|
||||
f"apad=pad_dur={pad_duration}",
|
||||
"-ar",
|
||||
str(sample_rate),
|
||||
"-b:a",
|
||||
str(bit_rate),
|
||||
"-ac",
|
||||
str(channels),
|
||||
"-c:a",
|
||||
"libmp3lame",
|
||||
"-y",
|
||||
output_path,
|
||||
]
|
||||
sp.run(cmd, capture_output=True, text=True, check=True, encoding="utf-8", errors="replace")
|
||||
return True
|
||||
|
||||
except sp.CalledProcessError as e:
|
||||
if e.stderr:
|
||||
logging.error(f"Subprocess execution failed, stderr: {e.stderr}")
|
||||
raise
|
||||
except json.JSONDecodeError:
|
||||
raise ValueError(f"Failed to parse audio information: {input_path}")
|
||||
|
||||
@staticmethod
|
||||
def generate_white_noise(
|
||||
duration: float,
|
||||
framerate: int,
|
||||
n_channels: int = 1,
|
||||
rms: float = None,
|
||||
std_dev: float = None,
|
||||
seed: int = None,
|
||||
) -> np.ndarray:
|
||||
if seed is not None:
|
||||
np.random.seed(seed)
|
||||
|
||||
n_samples = int(duration * framerate)
|
||||
if n_channels == 1:
|
||||
noise = np.random.normal(0, 1, n_samples).astype(np.float32)
|
||||
else:
|
||||
noise = np.random.normal(0, 1, (n_samples, n_channels)).astype(np.float32)
|
||||
|
||||
if std_dev is not None:
|
||||
current_std = np.std(noise)
|
||||
if current_std > 0:
|
||||
noise = noise * (std_dev / current_std)
|
||||
elif rms is not None:
|
||||
current_rms = np.sqrt(np.mean(noise**2))
|
||||
if current_rms > 0:
|
||||
noise = noise * (rms / current_rms)
|
||||
return noise
|
||||
|
||||
@staticmethod
|
||||
def save_wav_file(audio_data: np.ndarray, output_path, framerate: int, sample_width: int = 2) -> None:
|
||||
if audio_data.ndim == 1:
|
||||
n_channels = 1
|
||||
audio_data = audio_data.reshape(-1, 1)
|
||||
else:
|
||||
n_channels = audio_data.shape[1]
|
||||
|
||||
audio_data = np.clip(audio_data, -1.0, 1.0)
|
||||
if sample_width == 1:
|
||||
audio_int = ((audio_data + 1.0) * 127.5).astype(np.uint8)
|
||||
elif sample_width == 2:
|
||||
audio_int = (audio_data * 32767).astype(np.int16)
|
||||
elif sample_width == 4:
|
||||
audio_int = (audio_data * 2147483647).astype(np.int32)
|
||||
else:
|
||||
raise ValueError(f"Unsupported sample width: {sample_width}")
|
||||
|
||||
if n_channels == 1:
|
||||
audio_int = audio_int.flatten()
|
||||
else:
|
||||
audio_int = audio_int.reshape(-1, n_channels)
|
||||
|
||||
with wave.open(output_path, "wb") as wav_file:
|
||||
wav_file.setnchannels(n_channels)
|
||||
wav_file.setsampwidth(sample_width)
|
||||
wav_file.setframerate(framerate)
|
||||
wav_file.writeframes(audio_int.tobytes())
|
||||
|
||||
@staticmethod
|
||||
def generate_background_mask(positive_mask_paths):
|
||||
"""White where all positive masks are ~zero (background), black elsewhere."""
|
||||
width = height = None
|
||||
opened_imgs = []
|
||||
for path in positive_mask_paths:
|
||||
img = Image.open(path)
|
||||
if width is None:
|
||||
width = img.width
|
||||
elif width != img.width:
|
||||
raise ValueError(f"Widths of masks are not the same: {width} != {img.width}")
|
||||
if height is None:
|
||||
height = img.height
|
||||
elif height != img.height:
|
||||
raise ValueError(f"Heights of masks are not the same: {height} != {img.height}")
|
||||
opened_imgs.append(img)
|
||||
|
||||
img_arrays = []
|
||||
for img in opened_imgs:
|
||||
arr = np.array(img)
|
||||
if arr.ndim == 2:
|
||||
arr = arr[:, :, np.newaxis]
|
||||
img_arrays.append(arr)
|
||||
|
||||
threshold = 1
|
||||
zero_masks = []
|
||||
for arr in img_arrays:
|
||||
if arr.shape[-1] == 1:
|
||||
zero_mask = arr[:, :, 0] <= threshold
|
||||
else:
|
||||
zero_mask = np.all(arr <= threshold, axis=-1)
|
||||
zero_masks.append(zero_mask)
|
||||
|
||||
if zero_masks:
|
||||
all_zero_mask = np.logical_and.reduce(zero_masks)
|
||||
bg_array = np.where(all_zero_mask, 255, 0).astype(np.uint8)
|
||||
else:
|
||||
bg_array = np.full((height, width), 255, dtype=np.uint8)
|
||||
|
||||
bg_img = Image.fromarray(bg_array, mode="L")
|
||||
img_io = io.BytesIO()
|
||||
bg_img.save(img_io, format="JPEG")
|
||||
img_io.seek(0)
|
||||
for img in opened_imgs:
|
||||
img.close()
|
||||
return img_io
|
||||
+448
@@ -0,0 +1,448 @@
|
||||
"""Per-feature configuration nodes: inference / teacache / quantization / memory."""
|
||||
|
||||
from ..bridge import get_available_attn_ops, get_available_quant_ops
|
||||
from ..config_builder import InferenceConfigBuilder
|
||||
from ..data_models import (
|
||||
MemoryOptimizationConfig,
|
||||
QuantizationConfig,
|
||||
TeaCacheConfig,
|
||||
)
|
||||
from ..model_utils import scan_models, support_model_cls_list
|
||||
|
||||
|
||||
class LightX2VInferenceConfig:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
available_models = scan_models()
|
||||
support_model_classes = support_model_cls_list()
|
||||
available_attn = get_available_attn_ops()
|
||||
attn_types = []
|
||||
|
||||
for op_name, is_available in available_attn:
|
||||
if is_available:
|
||||
attn_types.append(op_name)
|
||||
|
||||
if "torch_sdpa" not in attn_types:
|
||||
attn_types.append("torch_sdpa")
|
||||
|
||||
return {
|
||||
"required": {
|
||||
"model_cls": (
|
||||
support_model_classes,
|
||||
{"default": "wan2.1", "tooltip": "Model type"},
|
||||
),
|
||||
"model_name": (
|
||||
available_models,
|
||||
{
|
||||
"default": available_models[0],
|
||||
"tooltip": "Select model from available models",
|
||||
},
|
||||
),
|
||||
"task": (
|
||||
["t2v", "i2v", "s2v", "rs2v"],
|
||||
{
|
||||
"default": "i2v",
|
||||
"tooltip": "Task type: text-to-video or image-to-video or reference_image and audio to video (shot)",
|
||||
},
|
||||
),
|
||||
"infer_steps": (
|
||||
"INT",
|
||||
{"default": 4, "min": 1, "max": 100, "tooltip": "Inference steps"},
|
||||
),
|
||||
"seed": (
|
||||
"INT",
|
||||
{
|
||||
"default": 42,
|
||||
"min": -1,
|
||||
"max": 2**32 - 1,
|
||||
"tooltip": "Random seed, -1 for random",
|
||||
},
|
||||
),
|
||||
"cfg_scale": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 5.0,
|
||||
"min": 1.0,
|
||||
"max": 10.0,
|
||||
"step": 0.1,
|
||||
"tooltip": "CFG guidance strength",
|
||||
},
|
||||
),
|
||||
"cfg_scale2": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 5.0,
|
||||
"min": 1.0,
|
||||
"max": 10.0,
|
||||
"step": 0.1,
|
||||
"tooltip": "CFG guidance, lower noise when model cls is Wan2.2 MoE",
|
||||
},
|
||||
),
|
||||
"sample_shift": (
|
||||
"INT",
|
||||
{"default": 5, "min": 0, "max": 10, "tooltip": "Sample shift"},
|
||||
),
|
||||
"height": (
|
||||
"INT",
|
||||
{
|
||||
"default": 1280,
|
||||
"min": 64,
|
||||
"max": 2048,
|
||||
"step": 8,
|
||||
"tooltip": "Video height",
|
||||
},
|
||||
),
|
||||
"width": (
|
||||
"INT",
|
||||
{
|
||||
"default": 720,
|
||||
"min": 64,
|
||||
"max": 2048,
|
||||
"step": 8,
|
||||
"tooltip": "Video width",
|
||||
},
|
||||
),
|
||||
"duration": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 5.0,
|
||||
"min": 1.0,
|
||||
"max": 999,
|
||||
"step": 0.1,
|
||||
"tooltip": "Video duration in seconds",
|
||||
},
|
||||
),
|
||||
"attention_type": (
|
||||
attn_types,
|
||||
{"default": attn_types[0], "tooltip": "Attention mechanism type"},
|
||||
),
|
||||
},
|
||||
"optional": {
|
||||
"denoising_steps": (
|
||||
"STRING",
|
||||
{
|
||||
"default": "",
|
||||
"tooltip": "Custom denoising steps for distillation models (comma-separated, e.g., '999,750,500,250'). Leave empty to use model defaults.",
|
||||
},
|
||||
),
|
||||
"resize_mode": (
|
||||
[
|
||||
"adaptive",
|
||||
"keep_ratio_fixed_area",
|
||||
"fixed_min_area",
|
||||
"fixed_max_area",
|
||||
"fixed_shape",
|
||||
"fixed_min_side",
|
||||
],
|
||||
{
|
||||
"default": "adaptive",
|
||||
"tooltip": "Adaptive resize input image to target aspect ratio",
|
||||
},
|
||||
),
|
||||
"fixed_area": (
|
||||
"STRING",
|
||||
{
|
||||
"default": "720p",
|
||||
"tooltip": "Fixed shape for input image, e.g., '720p', '480p', when resize_mode is 'keep_ratio_fixed_area' or 'fixed_min_side'",
|
||||
},
|
||||
),
|
||||
"segment_length": (
|
||||
"INT",
|
||||
{
|
||||
"default": 81,
|
||||
"min": 16,
|
||||
"max": 256,
|
||||
"tooltip": "Segment length in frames for sekotalk models (target_video_length)",
|
||||
},
|
||||
),
|
||||
"prev_frame_length": (
|
||||
"INT",
|
||||
{
|
||||
"default": 5,
|
||||
"min": 0,
|
||||
"max": 16,
|
||||
"tooltip": "Previous frame overlap for sekotalk models",
|
||||
},
|
||||
),
|
||||
"use_tiny_vae": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": False,
|
||||
"tooltip": "Use lightweight VAE to accelerate decoding",
|
||||
},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("INFERENCE_CONFIG",)
|
||||
RETURN_NAMES = ("inference_config",)
|
||||
FUNCTION = "create_config"
|
||||
CATEGORY = "LightX2V/Config"
|
||||
|
||||
def create_config(
|
||||
self,
|
||||
model_cls,
|
||||
model_name,
|
||||
task,
|
||||
infer_steps,
|
||||
seed,
|
||||
cfg_scale,
|
||||
cfg_scale2,
|
||||
sample_shift,
|
||||
height,
|
||||
width,
|
||||
duration,
|
||||
attention_type,
|
||||
denoising_steps="",
|
||||
resize_mode="adaptive",
|
||||
fixed_area="720p",
|
||||
segment_length=81,
|
||||
prev_frame_length=5,
|
||||
use_tiny_vae=False,
|
||||
):
|
||||
"""Create basic inference configuration."""
|
||||
builder = InferenceConfigBuilder()
|
||||
|
||||
config = builder.build(
|
||||
model_cls=model_cls,
|
||||
model_name=model_name,
|
||||
task=task,
|
||||
infer_steps=infer_steps,
|
||||
seed=seed,
|
||||
cfg_scale=cfg_scale,
|
||||
cfg_scale2=cfg_scale2,
|
||||
sample_shift=sample_shift,
|
||||
height=height,
|
||||
width=width,
|
||||
duration=duration,
|
||||
attention_type=attention_type,
|
||||
denoising_steps=denoising_steps,
|
||||
resize_mode=resize_mode,
|
||||
fixed_area=fixed_area,
|
||||
segment_length=segment_length,
|
||||
prev_frame_length=prev_frame_length,
|
||||
use_tiny_vae=use_tiny_vae,
|
||||
)
|
||||
|
||||
return (config.to_dict(),)
|
||||
|
||||
|
||||
class LightX2VTeaCache:
|
||||
"""TeaCache configuration node."""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"enable": (
|
||||
"BOOLEAN",
|
||||
{"default": False, "tooltip": "Enable TeaCache feature caching"},
|
||||
),
|
||||
"threshold": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.26,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "Cache threshold, lower values provide more speedup: 0.1 ~2x speedup, 0.2 ~3x speedup",
|
||||
},
|
||||
),
|
||||
"use_ret_steps": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": False,
|
||||
"tooltip": "Only cache key steps to balance quality and speed",
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("TEACACHE_CONFIG",)
|
||||
RETURN_NAMES = ("teacache_config",)
|
||||
FUNCTION = "create_config"
|
||||
CATEGORY = "LightX2V/Config"
|
||||
|
||||
def create_config(self, enable, threshold, use_ret_steps):
|
||||
"""Create TeaCache configuration."""
|
||||
config = TeaCacheConfig(enable=enable, threshold=threshold, use_ret_steps=use_ret_steps)
|
||||
return (config.to_dict(),)
|
||||
|
||||
|
||||
class LightX2VQuantization:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
available_ops = get_available_quant_ops()
|
||||
quant_backends = []
|
||||
|
||||
for op_name, is_available in available_ops:
|
||||
if is_available:
|
||||
quant_backends.append(op_name)
|
||||
|
||||
common_schema = ["fp8", "int8"]
|
||||
supported_quant_schemes = ["Default"]
|
||||
for schema in common_schema:
|
||||
for backend in quant_backends:
|
||||
supported_quant_schemes.append(f"{schema}-{backend}")
|
||||
|
||||
return {
|
||||
"required": {
|
||||
"dit_quant_scheme": (
|
||||
supported_quant_schemes,
|
||||
{
|
||||
"default": supported_quant_schemes[0],
|
||||
"tooltip": "DIT model quantization precision",
|
||||
},
|
||||
),
|
||||
"t5_quant_scheme": (
|
||||
supported_quant_schemes,
|
||||
{
|
||||
"default": supported_quant_schemes[0],
|
||||
"tooltip": "T5 encoder quantization precision",
|
||||
},
|
||||
),
|
||||
"clip_quant_scheme": (
|
||||
supported_quant_schemes,
|
||||
{
|
||||
"default": supported_quant_schemes[0],
|
||||
"tooltip": "CLIP encoder quantization precision",
|
||||
},
|
||||
),
|
||||
"adapter_quant_scheme": (
|
||||
supported_quant_schemes,
|
||||
{
|
||||
"default": supported_quant_schemes[0],
|
||||
"tooltip": "Adapter quantization precision",
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("QUANT_CONFIG",)
|
||||
RETURN_NAMES = ("quantization_config",)
|
||||
FUNCTION = "create_config"
|
||||
CATEGORY = "LightX2V/Config"
|
||||
|
||||
def create_config(
|
||||
self,
|
||||
dit_quant_scheme,
|
||||
t5_quant_scheme,
|
||||
clip_quant_scheme,
|
||||
adapter_quant_scheme,
|
||||
):
|
||||
"""Create quantization configuration."""
|
||||
config = QuantizationConfig(
|
||||
dit_quant_scheme=dit_quant_scheme,
|
||||
t5_quant_scheme=t5_quant_scheme,
|
||||
clip_quant_scheme=clip_quant_scheme,
|
||||
adapter_quant_scheme=adapter_quant_scheme,
|
||||
)
|
||||
return (config.to_dict(),)
|
||||
|
||||
|
||||
class LightX2VMemoryOptimization:
|
||||
"""Memory optimization configuration node."""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"enable_rotary_chunk": (
|
||||
"BOOLEAN",
|
||||
{"default": False, "tooltip": "Enable rotary encoding chunking"},
|
||||
),
|
||||
"rotary_chunk_size": (
|
||||
"INT",
|
||||
{"default": 100, "min": 100, "max": 10000, "step": 100},
|
||||
),
|
||||
"clean_cuda_cache": (
|
||||
"BOOLEAN",
|
||||
{"default": False, "tooltip": "Clean CUDA cache promptly"},
|
||||
),
|
||||
"cpu_offload": (
|
||||
"BOOLEAN",
|
||||
{"default": True, "tooltip": "Enable CPU offloading"},
|
||||
),
|
||||
"offload_granularity": (
|
||||
["block", "phase", "model"],
|
||||
{"default": "block", "tooltip": "Offload granularity"},
|
||||
),
|
||||
"offload_ratio": (
|
||||
"FLOAT",
|
||||
{"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.1},
|
||||
),
|
||||
"t5_cpu_offload": (
|
||||
"BOOLEAN",
|
||||
{"default": True, "tooltip": "Enable T5 CPU offloading"},
|
||||
),
|
||||
"t5_offload_granularity": (
|
||||
["model", "block"],
|
||||
{"default": "model", "tooltip": "T5 offload granularity"},
|
||||
),
|
||||
"audio_encoder_cpu_offload": (
|
||||
"BOOLEAN",
|
||||
{"default": True, "tooltip": "Enable audio encoder CPU offloading"},
|
||||
),
|
||||
"audio_adapter_cpu_offload": (
|
||||
"BOOLEAN",
|
||||
{"default": True, "tooltip": "Enable audio adapter CPU offloading"},
|
||||
),
|
||||
"vae_cpu_offload": (
|
||||
"BOOLEAN",
|
||||
{"default": True, "tooltip": "Enable VAE CPU offloading"},
|
||||
),
|
||||
"use_tiling_vae": (
|
||||
"BOOLEAN",
|
||||
{"default": True, "tooltip": "Enable VAE tiling inference"},
|
||||
),
|
||||
"lazy_load": (
|
||||
"BOOLEAN",
|
||||
{"default": False, "tooltip": "Lazy load model"},
|
||||
),
|
||||
"unload_after_inference": (
|
||||
"BOOLEAN",
|
||||
{"default": False, "tooltip": "Unload modules after inference"},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("MEMORY_CONFIG",)
|
||||
RETURN_NAMES = ("memory_config",)
|
||||
FUNCTION = "create_config"
|
||||
CATEGORY = "LightX2V/Config"
|
||||
|
||||
def create_config(
|
||||
self,
|
||||
enable_rotary_chunk=False,
|
||||
rotary_chunk_size=100,
|
||||
clean_cuda_cache=False,
|
||||
cpu_offload=False,
|
||||
offload_granularity="phase",
|
||||
offload_ratio=1.0,
|
||||
t5_cpu_offload=True,
|
||||
t5_offload_granularity="model",
|
||||
audio_encoder_cpu_offload=False,
|
||||
audio_adapter_cpu_offload=False,
|
||||
vae_cpu_offload=False,
|
||||
use_tiling_vae=False,
|
||||
lazy_load=False,
|
||||
unload_after_inference=False,
|
||||
):
|
||||
"""Create memory optimization configuration."""
|
||||
config = MemoryOptimizationConfig(
|
||||
enable_rotary_chunk=enable_rotary_chunk,
|
||||
rotary_chunk_size=rotary_chunk_size,
|
||||
clean_cuda_cache=clean_cuda_cache,
|
||||
cpu_offload=cpu_offload,
|
||||
offload_granularity=offload_granularity,
|
||||
offload_ratio=offload_ratio,
|
||||
t5_cpu_offload=t5_cpu_offload,
|
||||
t5_offload_granularity=t5_offload_granularity,
|
||||
audio_encoder_cpu_offload=audio_encoder_cpu_offload,
|
||||
audio_adapter_cpu_offload=audio_adapter_cpu_offload,
|
||||
vae_cpu_offload=vae_cpu_offload,
|
||||
use_tiling_vae=use_tiling_vae,
|
||||
lazy_load=lazy_load,
|
||||
unload_after_inference=unload_after_inference,
|
||||
)
|
||||
return (config.to_dict(),)
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Validated file-path inputs for native LightX2V media runners."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import folder_paths
|
||||
|
||||
|
||||
def _input_video_files():
|
||||
input_dir = folder_paths.get_input_directory()
|
||||
files, _ = folder_paths.recursive_search(input_dir)
|
||||
return sorted(folder_paths.filter_files_content_types(files, ["video"]))
|
||||
|
||||
|
||||
def resolve_input_video_path(filename) -> Path:
|
||||
"""Resolve a ComfyUI input filename without allowing directory escape."""
|
||||
|
||||
raw = str(filename or "").strip()
|
||||
if not raw:
|
||||
raise ValueError("video is required")
|
||||
|
||||
input_dir = Path(folder_paths.get_input_directory()).resolve()
|
||||
candidate = Path(folder_paths.get_annotated_filepath(raw)).resolve()
|
||||
try:
|
||||
candidate.relative_to(input_dir)
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"Expected a video under ComfyUI input, got: {filename}") from exc
|
||||
|
||||
if not candidate.is_file():
|
||||
raise FileNotFoundError(f"Input video does not exist: {candidate}")
|
||||
if not folder_paths.filter_files_content_types([candidate.name], ["video"]):
|
||||
raise ValueError(f"Input file is not recognized as video: {candidate}")
|
||||
return candidate
|
||||
|
||||
|
||||
def probe_video_file(video_path: Path):
|
||||
"""Read only video metadata and the first frame dimensions via decord."""
|
||||
|
||||
from decord import VideoReader
|
||||
|
||||
reader = VideoReader(str(video_path))
|
||||
if len(reader) < 1:
|
||||
raise ValueError(f"Input video contains no frames: {video_path}")
|
||||
first_frame = reader[0]
|
||||
height, width = int(first_frame.shape[0]), int(first_frame.shape[1])
|
||||
fps = float(reader.get_avg_fps() or 0.0)
|
||||
if height <= 0 or width <= 0:
|
||||
raise ValueError(f"Invalid input video dimensions: {width}x{height}")
|
||||
return width, height, fps
|
||||
|
||||
|
||||
class LightX2VInputVideoPath:
|
||||
"""Upload/select a video under ComfyUI input and expose its absolute path."""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"video": (
|
||||
_input_video_files(),
|
||||
{
|
||||
"video_upload": True,
|
||||
"tooltip": "Upload or select a video under ComfyUI input. The absolute path is resolved only while executing.",
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("STRING",)
|
||||
RETURN_NAMES = ("video_path",)
|
||||
FUNCTION = "resolve"
|
||||
CATEGORY = "LightX2V/Input"
|
||||
|
||||
def resolve(self, video):
|
||||
return (str(resolve_input_video_path(video)),)
|
||||
|
||||
@classmethod
|
||||
def IS_CHANGED(cls, video):
|
||||
path = resolve_input_video_path(video)
|
||||
stat = path.stat()
|
||||
return f"{stat.st_mtime_ns}:{stat.st_size}"
|
||||
|
||||
@classmethod
|
||||
def VALIDATE_INPUTS(cls, video):
|
||||
try:
|
||||
resolve_input_video_path(video)
|
||||
except (OSError, ValueError) as exc:
|
||||
return str(exc)
|
||||
return True
|
||||
@@ -0,0 +1,185 @@
|
||||
"""Modular inference runner that consumes a PREPARED_CONFIG."""
|
||||
|
||||
import gc
|
||||
import logging
|
||||
|
||||
import torch
|
||||
from comfy.utils import ProgressBar
|
||||
|
||||
from ..config_builder import ConfigBuilder
|
||||
from ..lightx2v.lightx2v.infer import init_runner
|
||||
from ..lightx2v.lightx2v.utils.input_info import init_empty_input_info, update_input_info_from_dict
|
||||
from ..lightx2v.lightx2v.utils.set_config import auto_calc_config, set_args2config
|
||||
|
||||
|
||||
class LightX2VModularInferenceV2:
|
||||
"""Pure inference node that takes prepared config and runs inference."""
|
||||
|
||||
_current_runner = None
|
||||
_current_config_hash = None
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"prepared_config": (
|
||||
"PREPARED_CONFIG",
|
||||
{"tooltip": "Fully prepared configuration from ConfigCombinerV2"},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "AUDIO")
|
||||
RETURN_NAMES = ("images", "audio")
|
||||
FUNCTION = "generate"
|
||||
CATEGORY = "LightX2V/InferenceV2"
|
||||
|
||||
@classmethod
|
||||
def _release_runner(cls):
|
||||
"""Drop the singleton runner + force VRAM teardown via DefaultRunner.__del__.
|
||||
|
||||
Callers MUST drop their own local refs to the old runner *before* invoking
|
||||
this — otherwise the refcount stays > 0, __del__ doesn't fire, and the
|
||||
next model load OOMs (model_a + model_b alive on GPU at the same time).
|
||||
"""
|
||||
cls._current_runner = None
|
||||
cls._current_config_hash = None
|
||||
gc.collect()
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
def _build_rs2v_shot_config(self, config):
|
||||
from ..lightx2v.lightx2v.shot_runner.shot_base import load_clip_configs
|
||||
|
||||
config_json = config.get("config_json")
|
||||
if config_json:
|
||||
main_cfg = config_json
|
||||
elif config.get("clip_configs"):
|
||||
main_cfg = config
|
||||
else:
|
||||
# load_clip_configs only runs set_config() on the "path" branch; for
|
||||
# in-memory clip configs we have to do it ourselves, otherwise framework
|
||||
# defaults (vae_stride, patch_size, ...) and the model's config.json
|
||||
# never get merged — rs2v_infer then KeyErrors on config["vae_stride"].
|
||||
if "task" not in config:
|
||||
config["task"] = "rs2v"
|
||||
# set_config = set_args2config + auto_calc_config. set_args2config strips
|
||||
# any key that's part of an InputInfo dataclass (target_video_length,
|
||||
# infer_steps, seed, ...). For CLI runs auto_calc_config recovers them
|
||||
# by merging --config_json, but we have no external JSON, so
|
||||
# auto_calc_config's `config["target_video_length"]` access KeyErrors.
|
||||
# Inject the bridge between set_args2config and auto_calc_config.
|
||||
target_video_length = config.get("target_video_length", config.get("segment_length", config.get("video_length", 81)))
|
||||
formatted = set_args2config(config)
|
||||
formatted["target_video_length"] = target_video_length
|
||||
formatted = auto_calc_config(formatted)
|
||||
main_cfg = {
|
||||
"lightx2v_path": "",
|
||||
"clip_configs": [
|
||||
{
|
||||
"name": "rs2v_clip",
|
||||
"config": formatted,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
if isinstance(main_cfg, dict) and "lightx2v_path" not in main_cfg:
|
||||
main_cfg = dict(main_cfg)
|
||||
main_cfg["lightx2v_path"] = ""
|
||||
|
||||
return load_clip_configs(main_cfg)
|
||||
|
||||
def generate(self, prepared_config):
|
||||
"""Run inference with prepared configuration."""
|
||||
|
||||
config = prepared_config
|
||||
# Combiner may have stashed the input AUDIO in the in-memory shim and
|
||||
# set audio_path to a sentinel — release it on exit so the tensor isn't
|
||||
# retained across runs (one ComfyUI graph tick = one sentinel).
|
||||
from ._audio_shim import is_sentinel as _is_audio_sentinel
|
||||
from ._audio_shim import release as _release_audio_sentinel
|
||||
|
||||
_audio_sentinel = config.get("audio_path") if isinstance(config, dict) else getattr(config, "audio_path", None)
|
||||
if not _is_audio_sentinel(_audio_sentinel):
|
||||
_audio_sentinel = None
|
||||
|
||||
try:
|
||||
config_hash = ConfigBuilder.get_config_hash(config)
|
||||
|
||||
current_runner = getattr(self.__class__, "_current_runner", None)
|
||||
current_config_hash = getattr(self.__class__, "_current_config_hash", None)
|
||||
|
||||
needs_reinit = current_runner is None or current_config_hash != config_hash or getattr(config, "lazy_load", False)
|
||||
|
||||
logging.info(f"Needs reinit: {needs_reinit}, old config hash: {current_config_hash}, new config hash: {config_hash}")
|
||||
if needs_reinit:
|
||||
if current_runner is not None:
|
||||
# Free old runner VRAM BEFORE constructing the new one, otherwise
|
||||
# both models live on GPU during the second load -> OOM (seen when
|
||||
# switching v2.5 s2v -> v2.7 rs2v).
|
||||
current_runner = None
|
||||
self._release_runner()
|
||||
if config.get("task") == "rs2v":
|
||||
from ..lightx2v.lightx2v.shot_runner.rs2v_infer import ShotRS2VPipeline
|
||||
|
||||
shot_cfg = self._build_rs2v_shot_config(config)
|
||||
self.__class__._current_runner = ShotRS2VPipeline(shot_cfg)
|
||||
else:
|
||||
# set_args2config strips InputInfo-dataclass keys (target_video_length,
|
||||
# infer_steps, ...). CLI flows recover them via --config_json merging
|
||||
# inside auto_calc_config; our in-memory flow has no external JSON, so
|
||||
# we bridge target_video_length manually between the two halves so
|
||||
# auto_calc_config:194's modulo check on s2v/i2v doesn't KeyError.
|
||||
target_video_length = config.get(
|
||||
"target_video_length",
|
||||
config.get("segment_length", config.get("video_length", 81)),
|
||||
)
|
||||
formatted_config = set_args2config(config)
|
||||
formatted_config["target_video_length"] = target_video_length
|
||||
formatted_config = auto_calc_config(formatted_config)
|
||||
self.__class__._current_runner = init_runner(formatted_config)
|
||||
self.__class__._current_config_hash = config_hash
|
||||
|
||||
progress = ProgressBar(100)
|
||||
|
||||
def update_progress(current_step, _total):
|
||||
progress.update_absolute(current_step)
|
||||
|
||||
current_runner = self.__class__._current_runner
|
||||
|
||||
if hasattr(current_runner, "set_progress_callback"):
|
||||
current_runner.set_progress_callback(update_progress)
|
||||
|
||||
config["return_result_tensor"] = True
|
||||
config["save_result_path"] = ""
|
||||
config["negative_prompt"] = config.get("negative_prompt", "")
|
||||
if config.get("task") == "rs2v":
|
||||
result_dict = current_runner.run_pipeline(config)
|
||||
else:
|
||||
input_data = init_empty_input_info(config.task)
|
||||
update_input_info_from_dict(input_data, config)
|
||||
current_runner.set_config(config)
|
||||
result_dict = current_runner.run_pipeline(input_data)
|
||||
|
||||
images = result_dict.get("video", None)
|
||||
audio = result_dict.get("audio", None)
|
||||
|
||||
if images is not None and images.numel() > 0:
|
||||
images = images.cpu()
|
||||
if images.dtype != torch.float32:
|
||||
images = images.float()
|
||||
|
||||
if getattr(config, "unload_after_inference", False):
|
||||
current_runner = None # drop local ref so __del__ can run
|
||||
self._release_runner()
|
||||
else:
|
||||
torch.cuda.empty_cache()
|
||||
gc.collect()
|
||||
|
||||
return (images, audio)
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Error during inference: {e}")
|
||||
raise
|
||||
finally:
|
||||
if _audio_sentinel is not None:
|
||||
_release_audio_sentinel(_audio_sentinel)
|
||||
@@ -0,0 +1,48 @@
|
||||
"""LoRA chain loader node."""
|
||||
|
||||
from ..config_builder import LoRAChainBuilder
|
||||
from ..model_utils import scan_loras
|
||||
|
||||
|
||||
class LightX2VLoRALoader:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
available_loras = scan_loras()
|
||||
|
||||
return {
|
||||
"required": {
|
||||
"lora_name": (
|
||||
available_loras,
|
||||
{
|
||||
"default": available_loras[0],
|
||||
"tooltip": "Select LoRA from available LoRAs",
|
||||
},
|
||||
),
|
||||
"strength": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 1.0,
|
||||
"min": 0.0,
|
||||
"max": 2.0,
|
||||
"step": 0.1,
|
||||
"tooltip": "LoRA strength",
|
||||
},
|
||||
),
|
||||
},
|
||||
"optional": {
|
||||
"lora_chain": (
|
||||
"LORA_CHAIN",
|
||||
{"tooltip": "Previous LoRA chain to append to"},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("LORA_CHAIN",)
|
||||
RETURN_NAMES = ("lora_chain",)
|
||||
FUNCTION = "load_lora"
|
||||
CATEGORY = "LightX2V/LoRA"
|
||||
|
||||
def load_lora(self, lora_name, strength, lora_chain=None):
|
||||
"""Load and chain LoRA configurations."""
|
||||
chain = LoRAChainBuilder.build_chain(lora_name=lora_name, strength=strength, existing_chain=lora_chain)
|
||||
return (chain,)
|
||||
+761
@@ -0,0 +1,761 @@
|
||||
"""SeedVR2 super-resolution nodes for ComfyUI.
|
||||
|
||||
Split into:
|
||||
- LightX2VSeedVR2Loader: pick a SeedVR2 DiT checkpoint under
|
||||
models/lightx2v/seedvr2/, load it into VRAM, return a SEEDVR_MODEL handle.
|
||||
- LightX2VSeedVR2Sampler: takes SEEDVR_MODEL + IMAGE + per-call params,
|
||||
returns upscaled IMAGE frames.
|
||||
- LightX2VSeedVR2FileSampler: takes a validated input video path and streams
|
||||
the restored result to ComfyUI output while preserving source audio.
|
||||
|
||||
The sampler installs a small shim on the runner so input frames come from the
|
||||
IMAGE tensor (no temp file, no re-encode); the runner's segmenting logic still
|
||||
runs and slices our in-memory tensor.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import gc
|
||||
import logging
|
||||
import math
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import types
|
||||
import wave
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
|
||||
import folder_paths
|
||||
import torch
|
||||
from comfy.utils import ProgressBar
|
||||
|
||||
from .file_input import probe_video_file, resolve_input_video_path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SEEDVR_RUN_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def _seedvr2_model_dir() -> Path:
|
||||
return Path(folder_paths.models_dir) / "lightx2v" / "seedvr2"
|
||||
|
||||
|
||||
def _scan_seedvr2_ckpts():
|
||||
d = _seedvr2_model_dir()
|
||||
if not d.exists():
|
||||
return ["None"]
|
||||
items = sorted(f.name for f in d.iterdir() if f.is_file() and f.suffix == ".safetensors")
|
||||
return items or ["None"]
|
||||
|
||||
|
||||
def _prepare_output_video(filename_prefix, width, height):
|
||||
full_output_folder, filename, counter, subfolder, _ = folder_paths.get_save_image_path(
|
||||
filename_prefix,
|
||||
folder_paths.get_output_directory(),
|
||||
int(width),
|
||||
int(height),
|
||||
)
|
||||
file = f"{filename}_{counter:05}_.mp4"
|
||||
full_path = Path(full_output_folder) / file
|
||||
return full_path, file, subfolder
|
||||
|
||||
|
||||
def _split_output_filename(filename):
|
||||
raw = str(filename or "").strip().replace("\\", "/")
|
||||
if not raw:
|
||||
raise ValueError("filename is required")
|
||||
|
||||
output_dir = Path(folder_paths.get_output_directory()).resolve()
|
||||
raw_path = Path(raw)
|
||||
if raw_path.is_absolute():
|
||||
full_path = raw_path.resolve()
|
||||
try:
|
||||
relative_path = full_path.relative_to(output_dir)
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"Expected a file under ComfyUI output, got: {filename}") from exc
|
||||
else:
|
||||
parts = raw_path.parts
|
||||
if parts and parts[0] == "output":
|
||||
parts = parts[1:]
|
||||
relative_path = Path(*parts) if parts else Path()
|
||||
if ".." in relative_path.parts:
|
||||
raise ValueError(f"Output filename cannot contain '..': {filename}")
|
||||
|
||||
if not relative_path.name:
|
||||
raise ValueError(f"Expected an output video filename, got: {filename}")
|
||||
|
||||
full_path = (output_dir / relative_path).resolve()
|
||||
try:
|
||||
full_path.relative_to(output_dir)
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"Expected a file under ComfyUI output, got: {filename}") from exc
|
||||
|
||||
subfolder = relative_path.parent.as_posix()
|
||||
if subfolder == ".":
|
||||
subfolder = ""
|
||||
return relative_path.name, subfolder, relative_path.as_posix(), full_path
|
||||
|
||||
|
||||
def _output_video_file_info(filename, validate_exists=True):
|
||||
file, subfolder, relative_name, full_path = _split_output_filename(filename)
|
||||
if validate_exists and not full_path.is_file():
|
||||
raise FileNotFoundError(f"Output video does not exist: {full_path}")
|
||||
return {"filename": file, "subfolder": subfolder, "type": "output"}, relative_name
|
||||
|
||||
|
||||
def _output_video_full_path(filename, validate_exists=True):
|
||||
_, _, _, full_path = _split_output_filename(filename)
|
||||
if validate_exists and not full_path.is_file():
|
||||
raise FileNotFoundError(f"Output video does not exist: {full_path}")
|
||||
return full_path
|
||||
|
||||
|
||||
def _audio_to_wav(audio, wav_path):
|
||||
if not isinstance(audio, Mapping):
|
||||
logger.info("[LightX2VOutputVideoPreview] skip audio mux: unsupported AUDIO input type=%s", type(audio).__name__)
|
||||
return False
|
||||
if audio.get("waveform") is None or audio.get("sample_rate") is None:
|
||||
logger.info("[LightX2VOutputVideoPreview] skip audio mux: AUDIO has no waveform/sample_rate")
|
||||
return False
|
||||
|
||||
waveform = audio["waveform"]
|
||||
sample_rate = int(audio["sample_rate"])
|
||||
if sample_rate <= 0:
|
||||
logger.info("[LightX2VOutputVideoPreview] skip audio mux: invalid sample_rate=%s", sample_rate)
|
||||
return False
|
||||
if not torch.is_tensor(waveform) or waveform.numel() == 0:
|
||||
logger.info("[LightX2VOutputVideoPreview] skip audio mux: empty waveform")
|
||||
return False
|
||||
if waveform.dim() == 3:
|
||||
waveform = waveform[0]
|
||||
if waveform.dim() == 1:
|
||||
waveform = waveform.unsqueeze(0)
|
||||
if waveform.dim() != 2:
|
||||
logger.info("[LightX2VOutputVideoPreview] skip audio mux: unsupported waveform shape=%s", tuple(audio["waveform"].shape))
|
||||
return False
|
||||
|
||||
waveform_i16 = (waveform.detach().cpu().float().clamp(-1.0, 1.0) * 32767.0).to(torch.int16)
|
||||
interleaved = waveform_i16.transpose(0, 1).contiguous().numpy()
|
||||
with wave.open(str(wav_path), "wb") as wav:
|
||||
wav.setnchannels(int(waveform_i16.shape[0]))
|
||||
wav.setsampwidth(2)
|
||||
wav.setframerate(sample_rate)
|
||||
wav.writeframes(interleaved.tobytes())
|
||||
return True
|
||||
|
||||
|
||||
def _audio_mux_path(video_path):
|
||||
video_path = Path(video_path)
|
||||
return video_path.with_name(f"{video_path.stem}-audio{video_path.suffix}")
|
||||
|
||||
|
||||
def _mux_audio_into_video(video_path, audio):
|
||||
from imageio_ffmpeg import get_ffmpeg_exe
|
||||
|
||||
video_path = Path(video_path)
|
||||
if not video_path.is_file():
|
||||
raise FileNotFoundError(f"Output video does not exist: {video_path}")
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix=".lightx2v_audio_mux.", dir=str(video_path.parent)) as tmp_dir:
|
||||
wav_path = Path(tmp_dir) / "audio.wav"
|
||||
muxed_tmp_path = Path(tmp_dir) / "muxed.mp4"
|
||||
muxed_path = _audio_mux_path(video_path)
|
||||
if not _audio_to_wav(audio, wav_path):
|
||||
return None
|
||||
|
||||
command = [
|
||||
get_ffmpeg_exe(),
|
||||
"-y",
|
||||
"-i",
|
||||
str(video_path),
|
||||
"-i",
|
||||
str(wav_path),
|
||||
"-map",
|
||||
"0:v:0",
|
||||
"-map",
|
||||
"1:a:0",
|
||||
"-c:v",
|
||||
"copy",
|
||||
"-c:a",
|
||||
"aac",
|
||||
"-b:a",
|
||||
"192k",
|
||||
"-shortest",
|
||||
str(muxed_tmp_path),
|
||||
]
|
||||
process = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
|
||||
if process.returncode != 0:
|
||||
stderr = process.stderr.decode("utf-8", errors="replace").strip()
|
||||
raise RuntimeError(f"FFmpeg audio mux failed: {stderr}")
|
||||
shutil.copy2(muxed_tmp_path, muxed_path)
|
||||
muxed_tmp_path.replace(video_path)
|
||||
logger.info("[LightX2VOutputVideoPreview] muxed audio into %s and updated %s", muxed_path, video_path)
|
||||
return muxed_path
|
||||
|
||||
|
||||
def _install_tensor_input_shim(runner, frames_u8, fps):
|
||||
"""Patch runner methods so input frames come from `frames_u8` instead of disk.
|
||||
|
||||
frames_u8: torch.uint8 [T, C, H, W] on CPU (same format as torchvision.io.read_video output).
|
||||
"""
|
||||
if not hasattr(runner, "_lightx2v_original_run_input_encoder_local_sr"):
|
||||
runner._lightx2v_original_run_input_encoder_local_sr = runner._run_input_encoder_local_sr.__func__
|
||||
if not hasattr(runner, "_lightx2v_original_run_input_encoder"):
|
||||
runner._lightx2v_original_run_input_encoder = runner.run_input_encoder
|
||||
|
||||
runner._tensor_input = frames_u8
|
||||
runner._tensor_input_fps = float(fps)
|
||||
|
||||
def _probe_video(self, video_path): # noqa: ARG001
|
||||
total = self._tensor_input.shape[0]
|
||||
self._set_output_fps(self._tensor_input_fps)
|
||||
return total, self._tensor_input_fps, None
|
||||
|
||||
def _read_video_segment(self, video_path, start_idx, end_idx): # noqa: ARG001
|
||||
seg = self._tensor_input[start_idx:end_idx]
|
||||
if seg.shape[0] == 0:
|
||||
return torch.empty(0, 3, 0, 0, dtype=torch.uint8)
|
||||
return seg
|
||||
|
||||
original_encoder = runner._lightx2v_original_run_input_encoder_local_sr
|
||||
|
||||
def _run_input_encoder_local_sr(self):
|
||||
if getattr(self, "_sr_segment", None) is None:
|
||||
self._sr_segment = (0, self._tensor_input.shape[0])
|
||||
try:
|
||||
return original_encoder(self)
|
||||
finally:
|
||||
self._sr_segment = None
|
||||
return original_encoder(self)
|
||||
|
||||
runner._probe_video = types.MethodType(_probe_video, runner)
|
||||
runner._read_video_segment = types.MethodType(_read_video_segment, runner)
|
||||
runner._run_input_encoder_local_sr = types.MethodType(_run_input_encoder_local_sr, runner)
|
||||
runner.run_input_encoder = runner._run_input_encoder_local_sr
|
||||
|
||||
|
||||
def _clear_tensor_input_shim(runner):
|
||||
for attr in ("_tensor_input", "_tensor_input_fps"):
|
||||
if hasattr(runner, attr):
|
||||
delattr(runner, attr)
|
||||
for attr in ("_probe_video", "_read_video_segment", "_run_input_encoder_local_sr"):
|
||||
if attr in runner.__dict__:
|
||||
delattr(runner, attr)
|
||||
original_run_input_encoder = getattr(runner, "_lightx2v_original_run_input_encoder", None)
|
||||
if original_run_input_encoder is not None:
|
||||
runner.run_input_encoder = original_run_input_encoder
|
||||
|
||||
|
||||
class LightX2VSeedVR2Loader:
|
||||
"""Load a SeedVR2 DiT checkpoint from models/lightx2v/seedvr2/."""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
ckpts = _scan_seedvr2_ckpts()
|
||||
return {
|
||||
"required": {
|
||||
"ckpt_name": (
|
||||
ckpts,
|
||||
{"default": ckpts[0], "tooltip": "DiT .safetensors under models/lightx2v/seedvr2/"},
|
||||
),
|
||||
"precision": (
|
||||
["auto", "bf16", "fp8-sgl", "fp8-q8f", "fp8-vllm"],
|
||||
{
|
||||
"default": "auto",
|
||||
"tooltip": "auto = bf16 for fp16/bf16 weights, fp8-sgl for fp8 weights. fp8-sgl needs sgl-kernel (H100/SM90); fp8-q8f is the 4090 path.",
|
||||
},
|
||||
),
|
||||
"cpu_offload": (
|
||||
"BOOLEAN",
|
||||
{"default": False, "tooltip": "Offload DiT blocks to CPU between forwards (slower; only needed on small VRAM)"},
|
||||
),
|
||||
"use_tiling_vae": ("BOOLEAN", {"default": False, "tooltip": "Tile VAE to reduce peak memory; usually keep off on 32GB+ GPUs."}),
|
||||
"vae_tile_size": (
|
||||
"INT",
|
||||
{
|
||||
"default": 1024,
|
||||
"min": 256,
|
||||
"max": 2048,
|
||||
"step": 64,
|
||||
"tooltip": "Output-space VAE tile size when tiling is enabled. Larger is faster but uses more VRAM.",
|
||||
},
|
||||
),
|
||||
"vae_tile_overlap": (
|
||||
"INT",
|
||||
{
|
||||
"default": 32,
|
||||
"min": 0,
|
||||
"max": 256,
|
||||
"step": 8,
|
||||
"tooltip": "Output-space VAE tile overlap when tiling is enabled. Smaller is faster but may increase tile seams.",
|
||||
},
|
||||
),
|
||||
"vae_causal_slice_size": (
|
||||
"INT",
|
||||
{
|
||||
"default": 16,
|
||||
"min": 0,
|
||||
"max": 64,
|
||||
"step": 1,
|
||||
"tooltip": "Temporal VAE slice size. 0 disables causal slicing. Larger is faster but uses more VRAM.",
|
||||
},
|
||||
),
|
||||
"vae_memory_limit_gb": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 2.0,
|
||||
"min": 0.0,
|
||||
"max": 16.0,
|
||||
"step": 0.25,
|
||||
"tooltip": "Per-op VAE conv/norm memory limit in GiB. 0 disables this extra splitting.",
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("SEEDVR_MODEL",)
|
||||
RETURN_NAMES = ("model",)
|
||||
FUNCTION = "load"
|
||||
CATEGORY = "LightX2V/SeedVR"
|
||||
|
||||
def load(
|
||||
self,
|
||||
ckpt_name,
|
||||
precision,
|
||||
cpu_offload,
|
||||
use_tiling_vae,
|
||||
vae_tile_size,
|
||||
vae_tile_overlap,
|
||||
vae_causal_slice_size,
|
||||
vae_memory_limit_gb,
|
||||
):
|
||||
from ..lightx2v.lightx2v.infer import init_runner
|
||||
from ..lightx2v.lightx2v.utils.set_config import set_config
|
||||
|
||||
model_dir = _seedvr2_model_dir()
|
||||
if ckpt_name == "None":
|
||||
raise FileNotFoundError(f"No .safetensors checkpoints found in {model_dir}")
|
||||
|
||||
for required in ("ema_vae.pth", "pos_emb.pt", "neg_emb.pt"):
|
||||
p = model_dir / required
|
||||
if not p.is_file():
|
||||
raise FileNotFoundError(
|
||||
f"Missing {p}. SeedVR2 needs VAE + pre-computed text embeddings (pos_emb.pt / neg_emb.pt) in the same directory as the DiT checkpoint."
|
||||
)
|
||||
ckpt_path = model_dir / ckpt_name
|
||||
if not ckpt_path.is_file():
|
||||
raise FileNotFoundError(str(ckpt_path))
|
||||
|
||||
if precision == "auto":
|
||||
precision = "fp8-sgl" if "fp8" in ckpt_name.lower() else "bf16"
|
||||
|
||||
config = {
|
||||
"model_cls": "seedvr2",
|
||||
"task": "sr",
|
||||
"model_path": str(model_dir),
|
||||
"infer_steps": 1,
|
||||
"fps": 16,
|
||||
"target_video_length": 81,
|
||||
"target_height": 1080,
|
||||
"target_width": 1920,
|
||||
"use_tiling_vae": bool(use_tiling_vae),
|
||||
"vae_tile_size": int(vae_tile_size),
|
||||
"vae_tile_overlap": int(vae_tile_overlap),
|
||||
"vae_causal_slice_size": int(vae_causal_slice_size),
|
||||
"vae_memory_limit_gb": float(vae_memory_limit_gb),
|
||||
"cpu_offload": bool(cpu_offload),
|
||||
}
|
||||
if "7b" in ckpt_name.lower():
|
||||
config["model_size"] = "7b"
|
||||
|
||||
if precision.startswith("fp8-"):
|
||||
config["dit_quantized_ckpt"] = str(ckpt_path)
|
||||
config["dit_quant_scheme"] = precision
|
||||
config["dit_quantized"] = True
|
||||
else:
|
||||
config["dit_original_ckpt"] = str(ckpt_path)
|
||||
|
||||
formatted = set_config(argparse.Namespace(**config))
|
||||
runner = init_runner(formatted)
|
||||
logger.info(
|
||||
"[SeedVR2Loader] loaded %s (%s); cpu_offload=%s, tile_vae=%s, tile=%s, overlap=%s, slice=%s, mem_limit=%sGiB",
|
||||
ckpt_name,
|
||||
precision,
|
||||
cpu_offload,
|
||||
use_tiling_vae,
|
||||
vae_tile_size,
|
||||
vae_tile_overlap,
|
||||
vae_causal_slice_size,
|
||||
vae_memory_limit_gb,
|
||||
)
|
||||
return ({"runner": runner, "precision": precision, "ckpt": ckpt_name},)
|
||||
|
||||
|
||||
class LightX2VSeedVR2Sampler:
|
||||
"""Run SeedVR2 SR on an input frame tensor; return upscaled frames as IMAGE."""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"model": ("SEEDVR_MODEL",),
|
||||
"images": ("IMAGE",),
|
||||
"target_width": ("INT", {"default": 1920, "min": 64, "max": 7680, "step": 8}),
|
||||
"target_height": (
|
||||
"INT",
|
||||
{
|
||||
"default": 1080,
|
||||
"min": 64,
|
||||
"max": 4320,
|
||||
"step": 8,
|
||||
"tooltip": "Target output frame height. NaDiT preserves input aspect ratio; the geometric mean of target_h * target_w is the effective resolution cap.",
|
||||
},
|
||||
),
|
||||
"infer_steps": ("INT", {"default": 1, "min": 1, "max": 50}),
|
||||
"segment_length": (
|
||||
"INT",
|
||||
{"default": 81, "min": 16, "max": 512, "step": 1, "tooltip": "Frames per SR pass. Long videos are auto-segmented."},
|
||||
),
|
||||
"segment_overlap": ("INT", {"default": 1, "min": 0, "max": 32}),
|
||||
"seed": ("INT", {"default": 42, "min": 0, "max": 2**32 - 1}),
|
||||
"source_fps": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 16.0,
|
||||
"min": 1.0,
|
||||
"max": 120.0,
|
||||
"step": 0.5,
|
||||
"tooltip": "FPS of the input frames (passed through to the runner for any internal timing logic)",
|
||||
},
|
||||
),
|
||||
"save_to_output_file": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": False,
|
||||
"tooltip": "Save the SR result directly under ComfyUI output and return a filename instead of returning the full IMAGE tensor.",
|
||||
},
|
||||
),
|
||||
"filename_prefix": ("STRING", {"default": "lightx2v_seedvr2/SeedVR2"}),
|
||||
"color_fix": (
|
||||
["gpu", "off", "cpu"],
|
||||
{
|
||||
"default": "gpu",
|
||||
"tooltip": "SeedVR color correction after VAE decode. gpu is faster on high-VRAM GPUs; off is fastest; cpu matches the original path.",
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "STRING")
|
||||
RETURN_NAMES = ("images", "filename")
|
||||
FUNCTION = "sample"
|
||||
CATEGORY = "LightX2V/SeedVR"
|
||||
|
||||
def sample(
|
||||
self,
|
||||
model,
|
||||
images,
|
||||
target_height,
|
||||
target_width,
|
||||
infer_steps,
|
||||
segment_length,
|
||||
segment_overlap,
|
||||
seed,
|
||||
source_fps,
|
||||
save_to_output_file,
|
||||
filename_prefix,
|
||||
color_fix,
|
||||
):
|
||||
from ..lightx2v.lightx2v.utils.input_info import init_empty_input_info, update_input_info_from_dict
|
||||
|
||||
runner = model["runner"]
|
||||
|
||||
if images.dim() != 4 or images.shape[-1] not in (3, 4):
|
||||
raise ValueError(f"Expected IMAGE [T, H, W, C], got shape {tuple(images.shape)}")
|
||||
# ComfyUI IMAGE: [T, H, W, C] float[0,1] → [T, C, H, W] uint8 (read_video's contract)
|
||||
ori_h, ori_w = int(images.shape[1]), int(images.shape[2])
|
||||
frames = images[..., :3].permute(0, 3, 1, 2).contiguous()
|
||||
frames_u8 = (frames.clamp(0.0, 1.0) * 255.0).to(torch.uint8).cpu()
|
||||
|
||||
# Derive sr_ratio from input vs target. The runner uses
|
||||
# resolution = min(sqrt(ori_h*ori_w) * sr_ratio, sqrt(target_h*target_w))
|
||||
# so we pick sr_ratio so the min lands on the target term (clamped to >=1
|
||||
# to avoid asking the SR model to downscale).
|
||||
ori_geom = math.sqrt(ori_h * ori_w)
|
||||
target_geom = math.sqrt(target_height * target_width)
|
||||
sr_ratio = max(target_geom / ori_geom, 1.0) if ori_geom > 0 else 1.0
|
||||
if target_geom < ori_geom:
|
||||
logger.warning(f"[SeedVR2] target ({target_height}x{target_width}) smaller than input ({ori_h}x{ori_w}); SR will run at input scale.")
|
||||
|
||||
_install_tensor_input_shim(runner, frames_u8, source_fps)
|
||||
save_path = ""
|
||||
output_file = ""
|
||||
output_subfolder = ""
|
||||
if save_to_output_file:
|
||||
full_path, output_file, output_subfolder = _prepare_output_video(filename_prefix, target_width, target_height)
|
||||
save_path = str(full_path)
|
||||
|
||||
# runner.config is a LockableDict (locked after init); set_config uses temporarily_unlocked.
|
||||
runner.set_config(
|
||||
{
|
||||
"sr_ratio": float(sr_ratio),
|
||||
"target_height": int(target_height),
|
||||
"target_width": int(target_width),
|
||||
"target_video_length": int(segment_length), # vestigial for SR; keep aligned with segment_length
|
||||
"sr_segment_length": int(segment_length),
|
||||
"sr_overlap": int(segment_overlap),
|
||||
"infer_steps": int(infer_steps),
|
||||
"seed": int(seed),
|
||||
"fps": float(source_fps),
|
||||
"video_path": "<tensor>", # truthy sentinel so segmenting logic runs; shim bypasses file I/O
|
||||
"image_path": "",
|
||||
"prompt": "",
|
||||
"negative_prompt": "",
|
||||
"save_result_path": save_path,
|
||||
"return_result_tensor": not bool(save_to_output_file),
|
||||
"color_fix": str(color_fix),
|
||||
}
|
||||
)
|
||||
|
||||
input_info = init_empty_input_info("sr")
|
||||
update_input_info_from_dict(
|
||||
input_info,
|
||||
{
|
||||
"video_path": "<tensor>",
|
||||
"image_path": "",
|
||||
"prompt": "",
|
||||
"negative_prompt": "",
|
||||
"seed": int(seed),
|
||||
"sr_ratio": float(sr_ratio),
|
||||
"save_result_path": save_path,
|
||||
"return_result_tensor": not bool(save_to_output_file),
|
||||
},
|
||||
)
|
||||
|
||||
progress = ProgressBar(100)
|
||||
if hasattr(runner, "set_progress_callback"):
|
||||
runner.set_progress_callback(lambda cur, _tot: progress.update_absolute(cur))
|
||||
|
||||
try:
|
||||
result = runner.run_pipeline(input_info)
|
||||
finally:
|
||||
_clear_tensor_input_shim(runner)
|
||||
torch.cuda.empty_cache()
|
||||
gc.collect()
|
||||
|
||||
video = result.get("video") if isinstance(result, dict) else result
|
||||
if save_to_output_file:
|
||||
if not Path(save_path).is_file():
|
||||
raise RuntimeError(f"SeedVR2 did not create expected output video: {save_path}")
|
||||
placeholder = torch.zeros((1, 1, 1, 3), dtype=torch.float32)
|
||||
relative_name = str(Path(output_subfolder) / output_file) if output_subfolder else output_file
|
||||
return (placeholder, relative_name)
|
||||
|
||||
if video is None or video.numel() == 0:
|
||||
raise RuntimeError("SeedVR2 returned empty result")
|
||||
|
||||
# wan_vae_to_comfy already gives [T, H, W, C] float[0,1] on CPU
|
||||
video = video.detach().cpu().float().clamp(0.0, 1.0)
|
||||
return (video, "")
|
||||
|
||||
|
||||
class LightX2VSeedVR2FileSampler:
|
||||
"""Run SeedVR2 on an input video using segmented file I/O."""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"model": ("SEEDVR_MODEL",),
|
||||
"video_path": (
|
||||
"STRING",
|
||||
{
|
||||
"forceInput": True,
|
||||
"tooltip": "Absolute path produced by LightX2V Input Video Path. The file must remain under ComfyUI input.",
|
||||
},
|
||||
),
|
||||
"target_width": ("INT", {"default": 1920, "min": 64, "max": 7680, "step": 8}),
|
||||
"target_height": (
|
||||
"INT",
|
||||
{
|
||||
"default": 1080,
|
||||
"min": 64,
|
||||
"max": 4320,
|
||||
"step": 8,
|
||||
"tooltip": "Target output frame height.",
|
||||
},
|
||||
),
|
||||
"infer_steps": ("INT", {"default": 1, "min": 1, "max": 50}),
|
||||
"segment_length": (
|
||||
"INT",
|
||||
{
|
||||
"default": 81,
|
||||
"min": 16,
|
||||
"max": 512,
|
||||
"step": 1,
|
||||
"tooltip": "Frames decoded and restored per segment. Long videos do not materialize as a full IMAGE batch.",
|
||||
},
|
||||
),
|
||||
"segment_overlap": ("INT", {"default": 1, "min": 0, "max": 32}),
|
||||
"seed": ("INT", {"default": 42, "min": 0, "max": 2**32 - 1}),
|
||||
"filename_prefix": ("STRING", {"default": "lightx2v_seedvr2/SeedVR2"}),
|
||||
"color_fix": (
|
||||
["gpu", "off", "cpu"],
|
||||
{
|
||||
"default": "gpu",
|
||||
"tooltip": "SeedVR color correction after VAE decode.",
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("STRING",)
|
||||
RETURN_NAMES = ("filename",)
|
||||
FUNCTION = "sample"
|
||||
CATEGORY = "LightX2V/SeedVR"
|
||||
|
||||
def sample(
|
||||
self,
|
||||
model,
|
||||
video_path,
|
||||
target_height,
|
||||
target_width,
|
||||
infer_steps,
|
||||
segment_length,
|
||||
segment_overlap,
|
||||
seed,
|
||||
filename_prefix,
|
||||
color_fix,
|
||||
):
|
||||
from ..lightx2v.lightx2v.utils.input_info import init_empty_input_info, update_input_info_from_dict
|
||||
|
||||
input_path = resolve_input_video_path(video_path)
|
||||
source_width, source_height, source_fps = probe_video_file(input_path)
|
||||
effective_fps = source_fps if source_fps > 0 else 16.0
|
||||
source_geom = math.sqrt(source_height * source_width)
|
||||
target_geom = math.sqrt(int(target_height) * int(target_width))
|
||||
sr_ratio = max(target_geom / source_geom, 1.0) if source_geom > 0 else 1.0
|
||||
if target_geom < source_geom:
|
||||
logger.warning(
|
||||
"[SeedVR2FileSampler] target (%sx%s) is smaller than input (%sx%s); SR will run at input scale before final sizing.",
|
||||
target_width,
|
||||
target_height,
|
||||
source_width,
|
||||
source_height,
|
||||
)
|
||||
|
||||
full_path, output_file, output_subfolder = _prepare_output_video(filename_prefix, target_width, target_height)
|
||||
save_path = str(full_path)
|
||||
input_info = init_empty_input_info("sr")
|
||||
update_input_info_from_dict(
|
||||
input_info,
|
||||
{
|
||||
"video_path": str(input_path),
|
||||
"image_path": "",
|
||||
"prompt": "",
|
||||
"negative_prompt": "",
|
||||
"seed": int(seed),
|
||||
"sr_ratio": float(sr_ratio),
|
||||
"save_result_path": save_path,
|
||||
"return_result_tensor": False,
|
||||
},
|
||||
)
|
||||
|
||||
runner = model["runner"]
|
||||
progress = ProgressBar(100)
|
||||
logger.info(
|
||||
"[SeedVR2FileSampler] input=%s (%sx%s @ %.3f fps), target=%sx%s, segment=%s/%s",
|
||||
input_path,
|
||||
source_width,
|
||||
source_height,
|
||||
source_fps,
|
||||
target_width,
|
||||
target_height,
|
||||
segment_length,
|
||||
segment_overlap,
|
||||
)
|
||||
try:
|
||||
with _SEEDVR_RUN_LOCK:
|
||||
_clear_tensor_input_shim(runner)
|
||||
runner.set_config(
|
||||
{
|
||||
"sr_ratio": float(sr_ratio),
|
||||
"target_height": int(target_height),
|
||||
"target_width": int(target_width),
|
||||
"target_video_length": int(segment_length),
|
||||
"sr_segment_length": int(segment_length),
|
||||
"sr_overlap": int(segment_overlap),
|
||||
"stream_save_video": True,
|
||||
"infer_steps": int(infer_steps),
|
||||
"seed": int(seed),
|
||||
"fps": float(effective_fps),
|
||||
"video_path": str(input_path),
|
||||
"image_path": "",
|
||||
"prompt": "",
|
||||
"negative_prompt": "",
|
||||
"save_result_path": save_path,
|
||||
"return_result_tensor": False,
|
||||
"color_fix": str(color_fix),
|
||||
}
|
||||
)
|
||||
if hasattr(runner, "set_progress_callback"):
|
||||
runner.set_progress_callback(lambda current, _total: progress.update_absolute(current))
|
||||
runner.run_pipeline(input_info)
|
||||
finally:
|
||||
torch.cuda.empty_cache()
|
||||
gc.collect()
|
||||
|
||||
if not Path(save_path).is_file():
|
||||
raise RuntimeError(f"SeedVR2 did not create expected output video: {save_path}")
|
||||
relative_name = str(Path(output_subfolder) / output_file) if output_subfolder else output_file
|
||||
return (relative_name,)
|
||||
|
||||
|
||||
class LightX2VOutputVideoPreview:
|
||||
"""Expose an existing ComfyUI output video to the history/view API."""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"filename": (
|
||||
"STRING",
|
||||
{
|
||||
"default": "",
|
||||
"forceInput": True,
|
||||
"tooltip": "Video path under ComfyUI output, e.g. file.mp4, subfolder/file.mp4, or output/subfolder/file.mp4.",
|
||||
},
|
||||
),
|
||||
"validate_exists": (
|
||||
"BOOLEAN",
|
||||
{"default": True, "tooltip": "Check that the output video exists before creating the preview entry."},
|
||||
),
|
||||
"mux_audio": (
|
||||
"BOOLEAN",
|
||||
{"default": True, "tooltip": "Mux optional AUDIO input into the output video before previewing."},
|
||||
),
|
||||
},
|
||||
"optional": {
|
||||
"audio": (
|
||||
"AUDIO",
|
||||
{"tooltip": "Optional audio from Load Video/Get Video Components to merge into the output MP4."},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("STRING",)
|
||||
RETURN_NAMES = ("filename",)
|
||||
FUNCTION = "preview"
|
||||
OUTPUT_NODE = True
|
||||
CATEGORY = "LightX2V/Output"
|
||||
|
||||
def preview(self, filename, validate_exists, mux_audio, audio=None):
|
||||
preview_filename = filename
|
||||
if bool(mux_audio) and audio is not None:
|
||||
video_path = _output_video_full_path(filename, bool(validate_exists))
|
||||
muxed_path = _mux_audio_into_video(video_path, audio)
|
||||
if muxed_path is not None:
|
||||
preview_filename = str(muxed_path)
|
||||
file_info, relative_name = _output_video_file_info(preview_filename, bool(validate_exists))
|
||||
return {"ui": {"images": [file_info], "animated": (True,)}, "result": (relative_name,)}
|
||||
@@ -0,0 +1,544 @@
|
||||
"""SwiftVR restoration nodes for ComfyUI."""
|
||||
|
||||
import argparse
|
||||
import gc
|
||||
import logging
|
||||
import tempfile
|
||||
import threading
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
import folder_paths
|
||||
import torch
|
||||
from comfy.utils import ProgressBar
|
||||
|
||||
from .file_input import probe_video_file, resolve_input_video_path
|
||||
from .seedvr import _prepare_output_video
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SWIFTVR_RUN_LOCK = threading.Lock()
|
||||
_REQUIRED_MODEL_FILES = (
|
||||
"transformer/config.json",
|
||||
"transformer/diffusion_pytorch_model.safetensors",
|
||||
"reae.safetensors",
|
||||
"prompt_embedding.safetensors",
|
||||
)
|
||||
_MAX_OUTPUT_DIMENSION = 8192
|
||||
_MAX_SR_RATIO = 8.0
|
||||
|
||||
|
||||
def _swiftvr_model_root() -> Path:
|
||||
return Path(folder_paths.models_dir) / "lightx2v"
|
||||
|
||||
|
||||
def _is_swiftvr_model(path: Path) -> bool:
|
||||
return path.is_dir() and all((path / relative_path).is_file() for relative_path in _REQUIRED_MODEL_FILES)
|
||||
|
||||
|
||||
def _scan_swiftvr_models():
|
||||
root = _swiftvr_model_root()
|
||||
if not root.is_dir():
|
||||
return ["None"]
|
||||
models = sorted(path.name for path in root.iterdir() if _is_swiftvr_model(path))
|
||||
return models or ["None"]
|
||||
|
||||
|
||||
def _prepare_output_image(filename_prefix, width, height):
|
||||
full_output_folder, filename, counter, subfolder, _ = folder_paths.get_save_image_path(
|
||||
filename_prefix,
|
||||
folder_paths.get_output_directory(),
|
||||
int(width),
|
||||
int(height),
|
||||
)
|
||||
file = f"{filename}_{counter:05}_.png"
|
||||
full_path = Path(full_output_folder) / file
|
||||
return full_path, file, subfolder
|
||||
|
||||
|
||||
def _resolve_output_size(source_width: int, source_height: int, target_short_edge: int, *, require_even: bool):
|
||||
"""Build an aspect-preserving public output size for SwiftVR.
|
||||
|
||||
Native SwiftVR pads this public size to a multiple of 32 internally and
|
||||
crops the restored result back, so network alignment is intentionally not
|
||||
exposed through the ComfyUI interface.
|
||||
"""
|
||||
|
||||
source_width = int(source_width)
|
||||
source_height = int(source_height)
|
||||
target_short_edge = int(target_short_edge)
|
||||
if source_width <= 0 or source_height <= 0:
|
||||
raise ValueError(f"SwiftVR source size must be positive, got {source_width}x{source_height}")
|
||||
if target_short_edge <= 0:
|
||||
raise ValueError(f"SwiftVR target_short_edge must be positive, got {target_short_edge}")
|
||||
|
||||
source_short_edge = min(source_width, source_height)
|
||||
if target_short_edge < source_short_edge:
|
||||
raise ValueError(
|
||||
f"SwiftVR only performs restoration/upscaling: target_short_edge {target_short_edge} "
|
||||
f"is smaller than the aligned input short edge {source_short_edge}"
|
||||
)
|
||||
|
||||
scale = target_short_edge / source_short_edge
|
||||
if scale > _MAX_SR_RATIO:
|
||||
raise ValueError(
|
||||
f"SwiftVR scale {scale:.3f}x exceeds the supported maximum {_MAX_SR_RATIO:.1f}x; "
|
||||
f"lower target_short_edge"
|
||||
)
|
||||
if source_width <= source_height:
|
||||
output_width = target_short_edge
|
||||
output_height = int(round(source_height * scale))
|
||||
else:
|
||||
output_height = target_short_edge
|
||||
output_width = int(round(source_width * scale))
|
||||
|
||||
if require_even:
|
||||
output_width = max(2, (output_width + 1) // 2 * 2)
|
||||
output_height = max(2, (output_height + 1) // 2 * 2)
|
||||
if max(output_width, output_height) > _MAX_OUTPUT_DIMENSION:
|
||||
raise ValueError(
|
||||
f"SwiftVR output {output_width}x{output_height} exceeds the maximum supported dimension "
|
||||
f"{_MAX_OUTPUT_DIMENSION}; lower target_short_edge"
|
||||
)
|
||||
return output_height, output_width, scale
|
||||
|
||||
|
||||
class _TensorVideoReader:
|
||||
"""Small decord-compatible reader backed by ComfyUI IMAGE frames."""
|
||||
|
||||
def __init__(self, frames: torch.Tensor, fps: float):
|
||||
self.frames = frames
|
||||
self.fps = float(fps)
|
||||
|
||||
def __len__(self):
|
||||
return int(self.frames.shape[0])
|
||||
|
||||
def __getitem__(self, index):
|
||||
return self.frames[index]
|
||||
|
||||
def get_batch(self, indices):
|
||||
return self.frames[indices]
|
||||
|
||||
def get_avg_fps(self):
|
||||
return self.fps
|
||||
|
||||
|
||||
class _TensorVideoWriter:
|
||||
"""imageio-compatible writer that keeps restored frames in memory."""
|
||||
|
||||
def __init__(self):
|
||||
self.frames = []
|
||||
|
||||
def append_data(self, frame):
|
||||
self.frames.append(torch.from_numpy(frame.copy()))
|
||||
|
||||
def close(self):
|
||||
return None
|
||||
|
||||
def as_images(self):
|
||||
if not self.frames:
|
||||
raise RuntimeError("SwiftVR produced no output frames")
|
||||
return torch.stack(self.frames).to(torch.float32).div_(255.0)
|
||||
|
||||
|
||||
class LightX2VSwiftVRLoader:
|
||||
"""Load a native LightX2V SwiftVR model and keep it resident."""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
models = _scan_swiftvr_models()
|
||||
return {
|
||||
"required": {
|
||||
"model_name": (
|
||||
models,
|
||||
{
|
||||
"default": models[0],
|
||||
"tooltip": "SwiftVR model directory under models/lightx2v/ containing transformer/, reae.safetensors, and prompt_embedding.safetensors.",
|
||||
},
|
||||
),
|
||||
"attention_backend": (
|
||||
["flash_attn3", "flash_attn2", "sage_attn2", "torch_sdpa"],
|
||||
{"default": "flash_attn3", "tooltip": "flash_attn3 is recommended on H100/SM90."},
|
||||
),
|
||||
"rope_type": (
|
||||
["flashinfer_rope", "torch_real_rope"],
|
||||
{"default": "flashinfer_rope"},
|
||||
),
|
||||
"clip_length": (
|
||||
"INT",
|
||||
{"default": 24, "min": 4, "max": 96, "step": 4, "tooltip": "Frames per streaming chunk; must be a multiple of 4."},
|
||||
),
|
||||
"dit_overlap": (
|
||||
"INT",
|
||||
{"default": 0, "min": 0, "max": 16, "step": 1, "tooltip": "Latent overlap retained between SwiftVR DiT chunks."},
|
||||
),
|
||||
"reae_frame_batch_size": (
|
||||
"INT",
|
||||
{"default": 0, "min": 0, "max": 32, "step": 1, "tooltip": "REAE frame batch size; 0 uses the native automatic path."},
|
||||
),
|
||||
"use_compile": (
|
||||
"BOOLEAN",
|
||||
{"default": False, "tooltip": "Enable torch.compile. First execution at a new resolution takes substantially longer."},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("SWIFTVR_MODEL",)
|
||||
RETURN_NAMES = ("model",)
|
||||
FUNCTION = "load"
|
||||
CATEGORY = "LightX2V/SwiftVR"
|
||||
|
||||
def load(self, model_name, attention_backend, rope_type, clip_length, dit_overlap, reae_frame_batch_size, use_compile):
|
||||
from ..lightx2v.lightx2v.infer import init_runner
|
||||
from ..lightx2v.lightx2v.utils.set_config import set_config
|
||||
|
||||
model_path = _swiftvr_model_root() / model_name
|
||||
if model_name == "None" or not _is_swiftvr_model(model_path):
|
||||
missing = [str(model_path / relative_path) for relative_path in _REQUIRED_MODEL_FILES if not (model_path / relative_path).is_file()]
|
||||
detail = f" Missing: {', '.join(missing)}" if missing else ""
|
||||
raise FileNotFoundError(f"No complete SwiftVR model found at {model_path}.{detail}")
|
||||
if int(clip_length) % 4:
|
||||
raise ValueError(f"SwiftVR clip_length must be a multiple of 4, got {clip_length}")
|
||||
|
||||
config = {
|
||||
"model_cls": "swiftvr",
|
||||
"task": "sr",
|
||||
"model_path": str(model_path),
|
||||
"attention_backend": str(attention_backend),
|
||||
"cross_attention_backend": str(attention_backend),
|
||||
"rope_type": str(rope_type),
|
||||
"clip_len": int(clip_length),
|
||||
"dit_overlap": int(dit_overlap),
|
||||
"reae_frame_batch_size": int(reae_frame_batch_size),
|
||||
"video_codec": "libx265",
|
||||
"quality": 60,
|
||||
"ffmpeg_preset": "ultrafast",
|
||||
"queue_size": 3,
|
||||
"cpu_offload": False,
|
||||
"parallel": False,
|
||||
"use_compile": bool(use_compile),
|
||||
}
|
||||
runner = init_runner(set_config(argparse.Namespace(**config)))
|
||||
logger.info(
|
||||
"[SwiftVRLoader] loaded %s; attention=%s, rope=%s, clip=%s, overlap=%s, reae_batch=%s, compile=%s",
|
||||
model_name,
|
||||
attention_backend,
|
||||
rope_type,
|
||||
clip_length,
|
||||
dit_overlap,
|
||||
reae_frame_batch_size,
|
||||
use_compile,
|
||||
)
|
||||
return ({"runner": runner, "model_name": model_name},)
|
||||
|
||||
|
||||
class LightX2VSwiftVRSampler:
|
||||
"""Restore a ComfyUI image or video-frame batch with a resident SwiftVR runner."""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"model": ("SWIFTVR_MODEL",),
|
||||
"images": ("IMAGE",),
|
||||
"target_short_edge": (
|
||||
"INT",
|
||||
{
|
||||
"default": 1080,
|
||||
"min": 64,
|
||||
"max": _MAX_OUTPUT_DIMENSION,
|
||||
"step": 8,
|
||||
"tooltip": "Output short edge. SwiftVR preserves aspect ratio; network padding and cropping are handled internally.",
|
||||
},
|
||||
),
|
||||
"source_fps": ("FLOAT", {"default": 16.0, "min": 1.0, "max": 120.0, "step": 0.5}),
|
||||
"save_to_output_file": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": False,
|
||||
"tooltip": "Save directly under ComfyUI output as PNG for one image or MP4 for multiple frames. Restored images are always returned for one-image input.",
|
||||
},
|
||||
),
|
||||
"filename_prefix": ("STRING", {"default": "lightx2v_swiftvr/SwiftVR"}),
|
||||
"video_codec": (["libx265", "libx264"], {"default": "libx265"}),
|
||||
"quality": ("INT", {"default": 60, "min": 0, "max": 100, "step": 1}),
|
||||
"ffmpeg_preset": (
|
||||
["ultrafast", "superfast", "veryfast", "faster", "fast", "medium", "slow"],
|
||||
{"default": "ultrafast"},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "STRING")
|
||||
RETURN_NAMES = ("images", "filename")
|
||||
FUNCTION = "sample"
|
||||
CATEGORY = "LightX2V/SwiftVR"
|
||||
|
||||
def sample(
|
||||
self,
|
||||
model,
|
||||
images,
|
||||
target_short_edge,
|
||||
source_fps,
|
||||
save_to_output_file,
|
||||
filename_prefix,
|
||||
video_codec,
|
||||
quality,
|
||||
ffmpeg_preset,
|
||||
):
|
||||
from lightx2v.models.runners.swiftvr import swiftvr_runner as swiftvr_module
|
||||
|
||||
from ..lightx2v.lightx2v.utils.input_info import init_empty_input_info, update_input_info_from_dict
|
||||
|
||||
if images.dim() != 4 or images.shape[-1] not in (3, 4):
|
||||
raise ValueError(f"Expected IMAGE [T, H, W, C], got shape {tuple(images.shape)}")
|
||||
if images.shape[0] < 1:
|
||||
raise ValueError("SwiftVR requires at least one input frame")
|
||||
|
||||
frames_u8 = (images[..., :3].detach().clamp(0.0, 1.0) * 255.0).round().to(torch.uint8).cpu().contiguous()
|
||||
is_image = int(frames_u8.shape[0]) == 1
|
||||
raw_height, raw_width = int(frames_u8.shape[1]), int(frames_u8.shape[2])
|
||||
source_height, source_width = raw_height // 8 * 8, raw_width // 8 * 8
|
||||
if source_height <= 0 or source_width <= 0:
|
||||
raise ValueError(f"SwiftVR input is too small after 8-pixel alignment: {raw_height}x{raw_width}")
|
||||
frames_u8 = frames_u8[:, :source_height, :source_width]
|
||||
output_height, output_width, sr_ratio = _resolve_output_size(
|
||||
source_width,
|
||||
source_height,
|
||||
target_short_edge,
|
||||
require_even=not is_image,
|
||||
)
|
||||
target_shape = [output_height, output_width]
|
||||
logger.info(
|
||||
"[SwiftVRSampler] aligned input=%sx%s, target_short_edge=%s, output=%sx%s, scale=%.4f",
|
||||
source_width,
|
||||
source_height,
|
||||
target_short_edge,
|
||||
output_width,
|
||||
output_height,
|
||||
sr_ratio,
|
||||
)
|
||||
|
||||
runner = model["runner"]
|
||||
output_file = ""
|
||||
output_subfolder = ""
|
||||
temp_dir = None
|
||||
if save_to_output_file:
|
||||
prepare_output = _prepare_output_image if is_image else _prepare_output_video
|
||||
full_path, output_file, output_subfolder = prepare_output(filename_prefix, output_width, output_height)
|
||||
save_path = str(full_path)
|
||||
memory_writer = None
|
||||
elif is_image:
|
||||
save_path = ""
|
||||
memory_writer = None
|
||||
else:
|
||||
temp_dir = tempfile.TemporaryDirectory(prefix="lightx2v_swiftvr_")
|
||||
save_path = str(Path(temp_dir.name) / "memory-output.mp4")
|
||||
memory_writer = _TensorVideoWriter()
|
||||
|
||||
image_path = "<tensor>" if is_image else ""
|
||||
video_path = "" if is_image else "<tensor>"
|
||||
return_result_tensor = is_image
|
||||
|
||||
input_info = init_empty_input_info("sr")
|
||||
update_input_info_from_dict(
|
||||
input_info,
|
||||
{
|
||||
"video_path": video_path,
|
||||
"image_path": image_path,
|
||||
"sr_ratio": float(sr_ratio),
|
||||
"target_shape": target_shape,
|
||||
"save_result_path": save_path,
|
||||
"return_result_tensor": return_result_tensor,
|
||||
},
|
||||
)
|
||||
|
||||
progress = ProgressBar(100)
|
||||
restored_images = None
|
||||
try:
|
||||
with _SWIFTVR_RUN_LOCK:
|
||||
original_video_reader = swiftvr_module.VideoReader
|
||||
original_mux_audio = swiftvr_module.mux_audio_from_video
|
||||
had_instance_writer = "open_video_writer" in runner.__dict__
|
||||
original_instance_writer = runner.__dict__.get("open_video_writer")
|
||||
had_instance_image_reader = "read_image_frame" in runner.__dict__
|
||||
original_instance_image_reader = runner.__dict__.get("read_image_frame")
|
||||
try:
|
||||
runner.set_config(
|
||||
{
|
||||
"fps": float(source_fps),
|
||||
"video_codec": str(video_codec),
|
||||
"quality": int(quality),
|
||||
"ffmpeg_preset": str(ffmpeg_preset),
|
||||
"video_path": video_path,
|
||||
"image_path": image_path,
|
||||
"sr_ratio": float(sr_ratio),
|
||||
"target_shape": target_shape,
|
||||
"return_result_tensor": return_result_tensor,
|
||||
}
|
||||
)
|
||||
if hasattr(runner, "set_progress_callback"):
|
||||
runner.set_progress_callback(lambda current, _total: progress.update_absolute(current))
|
||||
if is_image:
|
||||
image_frames = frames_u8.permute(0, 3, 1, 2).contiguous()
|
||||
runner.read_image_frame = types.MethodType(
|
||||
lambda _runner, _path: (image_frames, source_height, source_width),
|
||||
runner,
|
||||
)
|
||||
else:
|
||||
swiftvr_module.VideoReader = lambda _path: _TensorVideoReader(frames_u8, source_fps)
|
||||
swiftvr_module.mux_audio_from_video = lambda *_args, **_kwargs: None
|
||||
if not is_image and memory_writer is not None:
|
||||
runner.open_video_writer = types.MethodType(lambda _runner, _path, _fps: memory_writer, runner)
|
||||
result = runner.run_pipeline(input_info)
|
||||
if is_image:
|
||||
restored_images = result.get("images") if isinstance(result, dict) else result
|
||||
finally:
|
||||
swiftvr_module.VideoReader = original_video_reader
|
||||
swiftvr_module.mux_audio_from_video = original_mux_audio
|
||||
if memory_writer is not None:
|
||||
if had_instance_writer:
|
||||
runner.open_video_writer = original_instance_writer
|
||||
elif "open_video_writer" in runner.__dict__:
|
||||
del runner.open_video_writer
|
||||
if had_instance_image_reader:
|
||||
runner.read_image_frame = original_instance_image_reader
|
||||
elif "read_image_frame" in runner.__dict__:
|
||||
del runner.read_image_frame
|
||||
finally:
|
||||
if temp_dir is not None:
|
||||
temp_dir.cleanup()
|
||||
torch.cuda.empty_cache()
|
||||
gc.collect()
|
||||
|
||||
if is_image:
|
||||
if not torch.is_tensor(restored_images) or restored_images.numel() == 0:
|
||||
raise RuntimeError("SwiftVR produced no output image")
|
||||
restored_images = restored_images.to(device="cpu", dtype=torch.float32).clamp_(0.0, 1.0)
|
||||
if save_to_output_file:
|
||||
swiftvr_module.save_to_image(restored_images, save_path)
|
||||
if not Path(save_path).is_file():
|
||||
raise RuntimeError(f"SwiftVR did not create expected output image: {save_path}")
|
||||
relative_name = str(Path(output_subfolder) / output_file) if output_subfolder else output_file
|
||||
return (restored_images, relative_name)
|
||||
return (restored_images, "")
|
||||
|
||||
if save_to_output_file:
|
||||
if not Path(save_path).is_file():
|
||||
raise RuntimeError(f"SwiftVR did not create expected output video: {save_path}")
|
||||
placeholder = torch.zeros((1, 1, 1, 3), dtype=torch.float32)
|
||||
relative_name = str(Path(output_subfolder) / output_file) if output_subfolder else output_file
|
||||
return (placeholder, relative_name)
|
||||
|
||||
return (memory_writer.as_images().clamp_(0.0, 1.0), "")
|
||||
|
||||
|
||||
class LightX2VSwiftVRFileSampler:
|
||||
"""Restore an input video from disk without materializing it as IMAGE."""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"model": ("SWIFTVR_MODEL",),
|
||||
"video_path": (
|
||||
"STRING",
|
||||
{
|
||||
"forceInput": True,
|
||||
"tooltip": "Absolute path produced by LightX2V Input Video Path. The file must remain under ComfyUI input.",
|
||||
},
|
||||
),
|
||||
"target_short_edge": (
|
||||
"INT",
|
||||
{
|
||||
"default": 1080,
|
||||
"min": 64,
|
||||
"max": _MAX_OUTPUT_DIMENSION,
|
||||
"step": 8,
|
||||
"tooltip": "Output short edge. SwiftVR preserves aspect ratio.",
|
||||
},
|
||||
),
|
||||
"filename_prefix": ("STRING", {"default": "lightx2v_swiftvr/SwiftVR"}),
|
||||
"video_codec": (["libx265", "libx264"], {"default": "libx265"}),
|
||||
"quality": ("INT", {"default": 60, "min": 0, "max": 100, "step": 1}),
|
||||
"ffmpeg_preset": (
|
||||
["ultrafast", "superfast", "veryfast", "faster", "fast", "medium", "slow"],
|
||||
{"default": "ultrafast"},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("STRING",)
|
||||
RETURN_NAMES = ("filename",)
|
||||
FUNCTION = "sample"
|
||||
CATEGORY = "LightX2V/SwiftVR"
|
||||
|
||||
def sample(self, model, video_path, target_short_edge, filename_prefix, video_codec, quality, ffmpeg_preset):
|
||||
from ..lightx2v.lightx2v.utils.input_info import init_empty_input_info, update_input_info_from_dict
|
||||
|
||||
input_path = resolve_input_video_path(video_path)
|
||||
raw_width, raw_height, source_fps = probe_video_file(input_path)
|
||||
source_height, source_width = raw_height // 8 * 8, raw_width // 8 * 8
|
||||
if source_height <= 0 or source_width <= 0:
|
||||
raise ValueError(f"SwiftVR input is too small after 8-pixel alignment: {raw_height}x{raw_width}")
|
||||
|
||||
output_height, output_width, sr_ratio = _resolve_output_size(
|
||||
source_width,
|
||||
source_height,
|
||||
target_short_edge,
|
||||
require_even=True,
|
||||
)
|
||||
full_path, output_file, output_subfolder = _prepare_output_video(filename_prefix, output_width, output_height)
|
||||
save_path = str(full_path)
|
||||
target_shape = [output_height, output_width]
|
||||
|
||||
input_info = init_empty_input_info("sr")
|
||||
update_input_info_from_dict(
|
||||
input_info,
|
||||
{
|
||||
"video_path": str(input_path),
|
||||
"image_path": "",
|
||||
"sr_ratio": float(sr_ratio),
|
||||
"target_shape": target_shape,
|
||||
"save_result_path": save_path,
|
||||
"return_result_tensor": False,
|
||||
},
|
||||
)
|
||||
|
||||
runner = model["runner"]
|
||||
progress = ProgressBar(100)
|
||||
logger.info(
|
||||
"[SwiftVRFileSampler] input=%s (%sx%s @ %.3f fps), output=%sx%s",
|
||||
input_path,
|
||||
source_width,
|
||||
source_height,
|
||||
source_fps,
|
||||
output_width,
|
||||
output_height,
|
||||
)
|
||||
try:
|
||||
with _SWIFTVR_RUN_LOCK:
|
||||
runner.set_config(
|
||||
{
|
||||
"fps": 0.0,
|
||||
"video_codec": str(video_codec),
|
||||
"quality": int(quality),
|
||||
"ffmpeg_preset": str(ffmpeg_preset),
|
||||
"video_path": str(input_path),
|
||||
"image_path": "",
|
||||
"sr_ratio": float(sr_ratio),
|
||||
"target_shape": target_shape,
|
||||
"return_result_tensor": False,
|
||||
}
|
||||
)
|
||||
if hasattr(runner, "set_progress_callback"):
|
||||
runner.set_progress_callback(lambda current, _total: progress.update_absolute(current))
|
||||
runner.run_pipeline(input_info)
|
||||
finally:
|
||||
torch.cuda.empty_cache()
|
||||
gc.collect()
|
||||
|
||||
if not Path(save_path).is_file():
|
||||
raise RuntimeError(f"SwiftVR did not create expected output video: {save_path}")
|
||||
relative_name = str(Path(output_subfolder) / output_file) if output_subfolder else output_file
|
||||
return (relative_name,)
|
||||
+147
@@ -0,0 +1,147 @@
|
||||
"""Talk-object input and combiner nodes (for multi-speaker audio-driven generation)."""
|
||||
|
||||
from ..config_builder import TalkObjectConfigBuilder
|
||||
from ..data_models import TalkObjectsConfig
|
||||
|
||||
|
||||
class TalkObjectInput:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"name": (
|
||||
"STRING",
|
||||
{"default": "person_1", "tooltip": "speaker name identifier"},
|
||||
),
|
||||
},
|
||||
"optional": {
|
||||
"audio": ("AUDIO", {"tooltip": "uploaded audio file"}),
|
||||
"mask": ("MASK", {"tooltip": "uploaded mask image (optional)"}),
|
||||
"save_to_input": (
|
||||
"BOOLEAN",
|
||||
{"default": True, "tooltip": "save to input folder"},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("TALK_OBJECT",)
|
||||
RETURN_NAMES = ("talk_object",)
|
||||
FUNCTION = "create_talk_object"
|
||||
CATEGORY = "LightX2V/Audio"
|
||||
|
||||
def create_talk_object(self, name, audio=None, mask=None, save_to_input=True):
|
||||
"""Create a talk object from input data."""
|
||||
builder = TalkObjectConfigBuilder()
|
||||
|
||||
talk_object = builder.build_from_input(name=name, audio=audio, mask=mask, save_to_input=save_to_input)
|
||||
|
||||
if talk_object:
|
||||
return (talk_object,)
|
||||
return (None,)
|
||||
|
||||
|
||||
class TalkObjectsCombiner:
|
||||
PREDEFINED_SLOTS = 16
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
inputs = {"required": {}, "optional": {}}
|
||||
|
||||
for i in range(cls.PREDEFINED_SLOTS):
|
||||
inputs["optional"][f"talk_object_{i + 1}"] = (
|
||||
"TALK_OBJECT",
|
||||
{"tooltip": f"talk object {i + 1}"},
|
||||
)
|
||||
|
||||
return inputs
|
||||
|
||||
RETURN_TYPES = ("TALK_OBJECTS_CONFIG",)
|
||||
RETURN_NAMES = ("talk_objects_config",)
|
||||
FUNCTION = "combine_talk_objects"
|
||||
CATEGORY = "LightX2V/Audio"
|
||||
|
||||
def combine_talk_objects(self, **kwargs):
|
||||
config = TalkObjectsConfig()
|
||||
|
||||
for i in range(self.PREDEFINED_SLOTS):
|
||||
talk_obj = kwargs.get(f"talk_object_{i + 1}")
|
||||
|
||||
if talk_obj is not None:
|
||||
config.add_object(talk_obj)
|
||||
|
||||
if not config.talk_objects:
|
||||
return (None,)
|
||||
|
||||
return (config,)
|
||||
|
||||
|
||||
class TalkObjectsFromJSON:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"json_config": (
|
||||
"STRING",
|
||||
{
|
||||
"multiline": True,
|
||||
"default": '[{"name": "person1", "audio": "/path/to/audio1.wav", "mask": "/path/to/mask1.png"}]',
|
||||
"tooltip": "JSON format talk objects configuration",
|
||||
},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("TALK_OBJECTS_CONFIG",)
|
||||
RETURN_NAMES = ("talk_objects_config",)
|
||||
FUNCTION = "parse_json_config"
|
||||
CATEGORY = "LightX2V/Audio"
|
||||
|
||||
def parse_json_config(self, json_config):
|
||||
builder = TalkObjectConfigBuilder()
|
||||
talk_objects_config = builder.build_from_json(json_config)
|
||||
return (talk_objects_config,)
|
||||
|
||||
|
||||
class TalkObjectsFromFiles:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"audio_files": (
|
||||
"STRING",
|
||||
{
|
||||
"multiline": True,
|
||||
"default": "audio1.wav\naudio2.wav",
|
||||
"tooltip": "audio file list (one per line)",
|
||||
},
|
||||
),
|
||||
},
|
||||
"optional": {
|
||||
"mask_files": (
|
||||
"STRING",
|
||||
{
|
||||
"multiline": True,
|
||||
"default": "mask1.png\nmask2.png",
|
||||
"tooltip": "mask file list (one per line, optional)",
|
||||
},
|
||||
),
|
||||
"names": (
|
||||
"STRING",
|
||||
{
|
||||
"multiline": True,
|
||||
"default": "person1\nperson2",
|
||||
"tooltip": "talk object name list (one per line, optional)",
|
||||
},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("TALK_OBJECTS_CONFIG",)
|
||||
RETURN_NAMES = ("talk_objects_config",)
|
||||
FUNCTION = "build_from_files"
|
||||
CATEGORY = "LightX2V/Audio"
|
||||
|
||||
def build_from_files(self, audio_files, mask_files="", names=""):
|
||||
builder = TalkObjectConfigBuilder()
|
||||
talk_objects_config = builder.build_from_files(audio_files, mask_files, names)
|
||||
return (talk_objects_config,)
|
||||
@@ -12,3 +12,17 @@ Repository = "https://github.com/ModelTC/ComfyUI-Lightx2vWrapper"
|
||||
PublisherId = "gaclove"
|
||||
DisplayName = "ComfyUI-Lightx2vWrapper"
|
||||
Icon = ""
|
||||
|
||||
[tool.ruff]
|
||||
target-version = "py311"
|
||||
line-length = 150
|
||||
indent-width = 4
|
||||
|
||||
[tool.ruff.lint]
|
||||
extend-select = ["I", "F401"]
|
||||
ignore = ["F"]
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
"**/__init__.py" = ["F401"]
|
||||
"**/lightx2v_kernel/*" = ["F401"]
|
||||
"**/{cookbook,docs}/*" = ["E402", "F401", "F811", "F841"]
|
||||
|
||||
Reference in New Issue
Block a user