nodes/ModelTC-ComfyUI-Lightx2vWrapper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: finalize LightX2V 0.6 wrapper migration

Browse Source
This commit is contained in:
gaclove
2026-10-23 14:26:36 +08:00
parent ecb81530f7
commit c13737d6d0
10 changed files with 315 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1 -1
View File
@@ -346,4 +346,4 @@ Temporary Items
CLAUDE.md
AGENTS.md
.gitnexus/
.claude/
.claude/
example_workflows/lightx2v_seko_talk_static.json
+150
View File
@@ -0,0 +1,150 @@
{
"11": {
"inputs": {
"audio": "audio_full_female_10s.wav",
"start_time": 0,
"duration": 0
},
"class_type": "VHS_LoadAudioUpload",
"_meta": {
"title": "Load Audio (Upload)🎥🅥🅗🅢"
}
},
"13": {
"inputs": {
"prompt": "A person speaks naturally to the camera, with clear lip movement and subtle facial expressions.",
"negative_prompt": "",
"inference_config": [
"14",
0
],
"image": [
"20",
0
],
"audio": [
"11",
0
],
"memory_config": [
"16",
0
],
"quantization_config": [
"17",
0
]
},
"class_type": "LightX2VConfigCombinerV2",
"_meta": {
"title": "LightX2V Config Combiner V2"
}
},
"14": {
"inputs": {
"model_cls": "seko_talk",
"model_name": "Wan2.1-R2V0909-Audio-14B-720P-fp8",
"task": "s2v",
"infer_steps": 4,
"seed": 4221706066,
"cfg_scale": 1,
"cfg_scale2": 1,
"sample_shift": 5,
"height": 1280,
"width": 720,
"duration": 5,
"attention_type": "sage_attn2",
"denoising_steps": "",
"resize_mode": "adaptive",
"fixed_area": "720p",
"segment_length": 81,
"prev_frame_length": 5,
"use_tiny_vae": false
},
"class_type": "LightX2VInferenceConfig",
"_meta": {
"title": "LightX2V Inference Config"
}
},
"15": {
"inputs": {
"prepared_config": [
"13",
0
]
},
"class_type": "LightX2VModularInferenceV2",
"_meta": {
"title": "LightX2V Modular Inference V2"
}
},
"16": {
"inputs": {
"enable_rotary_chunk": false,
"rotary_chunk_size": 100,
"clean_cuda_cache": true,
"cpu_offload": true,
"offload_granularity": "block",
"offload_ratio": 1.0,
"t5_cpu_offload": true,
"t5_offload_granularity": "model",
"audio_encoder_cpu_offload": true,
"audio_adapter_cpu_offload": true,
"vae_cpu_offload": true,
"use_tiling_vae": true,
"lazy_load": false,
"unload_after_inference": false
},
"class_type": "LightX2VMemoryOptimization",
"_meta": {
"title": "LightX2V Memory Optimization"
}
},
"17": {
"inputs": {
"dit_quant_scheme": "fp8-sgl",
"t5_quant_scheme": "fp8-sgl",
"clip_quant_scheme": "fp8-sgl",
"adapter_quant_scheme": "fp8-sgl"
},
"class_type": "LightX2VQuantization",
"_meta": {
"title": "LightX2V Quantization"
}
},
"18": {
"inputs": {
"frame_rate": 16,
"loop_count": 0,
"filename_prefix": "lightx2v_sekotalk/static",
"format": "video/h264-mp4",
"pix_fmt": "yuv420p",
"crf": 19,
"save_metadata": false,
"trim_to_audio": false,
"pingpong": false,
"save_output": true,
"images": [
"15",
0
],
"audio": [
"15",
1
]
},
"class_type": "VHS_VideoCombine",
"_meta": {
"title": "Video Combine 🎥🅥🅗🅢"
}
},
"20": {
"inputs": {
"image": "codex_image_20260922_135831_1323a530_B.jpg"
},
"class_type": "LoadImage",
"_meta": {
"title": "Load Image"
}
}
}
example_workflows/lightx2v_swiftvr_image.json
+1 -1
View File
@@ -202,4 +202,4 @@
"VHS_KeepIntermediate": true
},
"version": 0.4
}
}
nodes/_audio_shim.py
+13 -2
View File
@@ -41,6 +41,17 @@ _PATCH_SITES = (
)
class _AudioSentinel(str):
"""A path-shaped token that releases its registry lease when discarded."""
def __del__(self):
try:
release(str(self))
except Exception:
# Destructors can run during interpreter shutdown.
pass
def register(waveform: torch.Tensor, sample_rate: int) -> str:
"""Register a [C, T] float32 waveform; return a sentinel "path" for it."""
if waveform.dim() != 2:
@@ -50,12 +61,12 @@ def register(waveform: torch.Tensor, sample_rate: int) -> str:
stashed = waveform.detach().to(torch.float32).cpu().contiguous()
with _LOCK:
_REGISTRY[sentinel] = (stashed, int(sample_rate))
return sentinel
return _AudioSentinel(sentinel)
def release(sentinel: str) -> None:
with _LOCK:
_REGISTRY.pop(sentinel, None)
_REGISTRY.pop(str(sentinel), None)
def is_sentinel(path) -> bool:
nodes/_image_shim.py
+113
View File
@@ -0,0 +1,113 @@
"""Keep ComfyUI IMAGE inputs in memory while LightX2V reads them.
LightX2V's runner API names the request field ``image_path`` and accepts a
PIL image at the final read boundary. The wrapper used to serialize the first
ComfyUI frame to a temporary PNG only to have the runner decode it again. This
shim keeps the path-shaped request contract while replacing that disk
round-trip with a short-lived in-memory PIL image.
"""
from __future__ import annotations
import logging
import threading
import types
import uuid
from typing import Dict
import torch
from PIL import Image
SENTINEL_PREFIX = "<lightx2v-mem-image:"
SENTINEL_SUFFIX = ">"
_REGISTRY: Dict[str, Image.Image] = {}
_LOCK = threading.Lock()
class _ImageSentinel(str):
"""A path-shaped token that releases its registry lease when discarded.
The registry keeps a plain ``str`` key rather than this object, so the
object can be collected even while the registry still contains the image.
This covers failures between Combiner execution and ``generate()`` where
the normal inference ``finally`` block has not started yet.
"""
def __del__(self):
try:
release(str(self))
except Exception:
# Destructors can run during interpreter shutdown.
pass
def register(images: torch.Tensor) -> str:
"""Register the first ComfyUI IMAGE frame and return a path-shaped token."""
if not torch.is_tensor(images) or images.ndim != 4:
raise ValueError(f"IMAGE must be a [B, H, W, C] tensor; got {type(images).__name__} {getattr(images, 'shape', '')}")
if images.shape[-1] not in (1, 3, 4):
raise ValueError(f"IMAGE must have 1, 3, or 4 channels; got shape {tuple(images.shape)}")
frame = images[0].detach().cpu().float().clamp(0.0, 1.0)
frame_u8 = (frame.mul(255.0).round()).to(torch.uint8).numpy()
if frame_u8.shape[-1] == 1:
frame_u8 = frame_u8[..., 0]
elif frame_u8.shape[-1] == 4:
frame_u8 = frame_u8[..., :3]
token = f"{SENTINEL_PREFIX}{uuid.uuid4().hex}{SENTINEL_SUFFIX}"
image = Image.fromarray(frame_u8)
if image.mode != "RGB":
image = image.convert("RGB")
with _LOCK:
_REGISTRY[token] = image
return _ImageSentinel(token)
def release(sentinel: str | None) -> None:
if not is_sentinel(sentinel):
return
with _LOCK:
_REGISTRY.pop(str(sentinel), None)
def is_sentinel(path) -> bool:
return isinstance(path, str) and path.startswith(SENTINEL_PREFIX)
def _lookup(sentinel: str) -> Image.Image:
with _LOCK:
image = _REGISTRY.get(sentinel)
if image is None:
raise FileNotFoundError(f"Stale LightX2V in-memory image sentinel: {sentinel}")
return image
def install(runner) -> bool:
"""Patch one runner's image read boundary; return whether it was patched."""
if runner is None or not hasattr(runner, "read_image_input"):
return False
if hasattr(runner, "_lightx2v_original_read_image_input"):
return True
original = runner.read_image_input
def _read_image_input(self, image_path):
if is_sentinel(image_path):
image_path = _lookup(image_path)
return original(image_path)
runner._lightx2v_original_read_image_input = original
runner.read_image_input = types.MethodType(_read_image_input, runner)
logging.info("_image_shim installed on %s", type(runner).__name__)
return True
def install_on_runner_tree(runner) -> int:
"""Patch a regular runner or every clip runner inside an rs2v pipeline."""
targets = [runner]
clip_generators = getattr(runner, "clip_generators", None)
if isinstance(clip_generators, dict):
targets.extend(clip_generators.values())
return sum(1 for target in targets if install(target))
nodes/combiner.py
+12 -7
View File
@@ -168,19 +168,24 @@ class _BaseConfigCombiner:
if image is None:
raise ValueError("i2v or s2v or rs2v task requires input image")
image_np = (image[0].cpu().numpy() * 255).astype(np.uint8)
pil_image = Image.fromarray(image_np)
temp_path = self.temp_manager.create_temp_file(suffix=".png")
pil_image.save(temp_path)
config.image_path = temp_path
logging.info(f"Image saved to {temp_path}")
frame_count = int(image.shape[0]) if hasattr(image, "shape") and image.ndim > 0 else 0
if frame_count > 1:
logging.warning(
"LightX2V uses only the first IMAGE frame as the reference; received %s frames",
frame_count,
)
from ._image_shim import register
config.image_path = register(image)
logging.info("Routed ComfyUI IMAGE through the in-memory shim")
def _save_single_audio_if_needed(self, config, audio):
# Route ComfyUI AUDIO straight into the runner via the in-memory shim
# (no WAV temp file, no soundfile round-trip). The shim only intercepts
# this single-AUDIO path; V3 multi-talker padding still produces real
# files since its inputs are external paths/URLs, not ComfyUI tensors.
if audio is None or not hasattr(config, "model_cls") or "seko" not in config.model_cls:
if audio is None or config.task not in {"s2v", "rs2v"}:
return
from ._audio_shim import comfyui_audio_to_loader_pair, install, register
nodes/config.py
+3 -3
View File
@@ -42,7 +42,7 @@ class LightX2VInferenceConfig:
["t2v", "i2v", "s2v", "rs2v"],
{
"default": "i2v",
"tooltip": "Task type: text-to-video or image-to-video or reference_image and audio to video (shot)",
"tooltip": "Task: t2v text-to-video, i2v image-to-video, s2v image+audio-to-video, or rs2v reference-shot image+audio-to-video",
},
),
"infer_steps": (
@@ -136,7 +136,7 @@ class LightX2VInferenceConfig:
],
{
"default": "adaptive",
"tooltip": "Adaptive resize input image to target aspect ratio",
"tooltip": "Resize policy: adaptive preserves the target area, fixed_shape uses the exact target size, and fixed-area/min-side modes preserve aspect ratio",
},
),
"fixed_area": (
@@ -161,7 +161,7 @@ class LightX2VInferenceConfig:
"default": 5,
"min": 0,
"max": 16,
"tooltip": "Previous frame overlap for sekotalk models",
"tooltip": "Previous-frame overlap for rs2v continuation; s2v uses the model's own initial reference and usually ignores this value",
},
),
"use_tiny_vae": (
nodes/inference.py
+19 -11
View File
@@ -100,9 +100,7 @@ class LightX2VModularInferenceV2:
int(config.get("target_height", config.get("height", 480))),
int(config.get("target_width", config.get("width", 832))),
]
startup_data["num_frames"] = int(
config.get("target_video_length", config.get("video_length", config.get("num_frames", 81)))
)
startup_data["num_frames"] = int(config.get("target_video_length", config.get("video_length", config.get("num_frames", 81))))
formatted = _modern_build_startup_config(startup_data)
else:
# set_args2config strips any key that's part of an InputInfo
@@ -143,9 +141,7 @@ class LightX2VModularInferenceV2:
int(config.get("target_height", config.get("height", 480))),
int(config.get("target_width", config.get("width", 832))),
]
startup_data["num_frames"] = int(
config.get("target_video_length", config.get("video_length", config.get("num_frames", 81)))
)
startup_data["num_frames"] = int(config.get("target_video_length", config.get("video_length", config.get("num_frames", 81))))
return _modern_build_startup_config(startup_data)
@staticmethod
@@ -168,9 +164,7 @@ class LightX2VModularInferenceV2:
int(config.get("target_width", config.get("width", 832))),
]
elif field == "num_frames":
request[field] = int(
config.get("target_video_length", config.get("video_length", config.get("num_frames", 81)))
)
request[field] = int(config.get("target_video_length", config.get("video_length", config.get("num_frames", 81))))
elif field in config:
request[field] = config[field]
@@ -183,6 +177,14 @@ class LightX2VModularInferenceV2:
"""Run inference with prepared configuration."""
config = prepared_config
from ._image_shim import install_on_runner_tree as _install_image_shim
from ._image_shim import is_sentinel as _is_image_sentinel
from ._image_shim import release as _release_image_sentinel
_image_sentinel = config.get("image_path") if isinstance(config, dict) else getattr(config, "image_path", None)
if not _is_image_sentinel(_image_sentinel):
_image_sentinel = None
# Combiner may have stashed the input AUDIO in the in-memory shim and
# set audio_path to a sentinel — release it on exit so the tensor isn't
# retained across runs (one ComfyUI graph tick = one sentinel).
@@ -199,7 +201,7 @@ class LightX2VModularInferenceV2:
current_runner = getattr(self.__class__, "_current_runner", None)
current_config_hash = getattr(self.__class__, "_current_config_hash", None)
needs_reinit = current_runner is None or current_config_hash != config_hash or getattr(config, "lazy_load", False)
needs_reinit = current_runner is None or current_config_hash != config_hash or config.get("lazy_load", False)
logging.info(f"Needs reinit: {needs_reinit}, old config hash: {current_config_hash}, new config hash: {config_hash}")
if needs_reinit:
@@ -240,6 +242,10 @@ class LightX2VModularInferenceV2:
current_runner = self.__class__._current_runner
if _image_sentinel is not None:
if _install_image_shim(current_runner) == 0:
raise RuntimeError("LightX2V IMAGE input shim could not find a runner image reader")
if hasattr(current_runner, "set_progress_callback"):
current_runner.set_progress_callback(update_progress)
@@ -279,7 +285,7 @@ class LightX2VModularInferenceV2:
if images.dtype != torch.float32:
images = images.float()
if getattr(config, "unload_after_inference", False):
if config.get("unload_after_inference", False):
current_runner = None # drop local ref so __del__ can run
self._release_runner()
else:
@@ -292,5 +298,7 @@ class LightX2VModularInferenceV2:
logging.error(f"Error during inference: {e}")
raise
finally:
if _image_sentinel is not None:
_release_image_sentinel(_image_sentinel)
if _audio_sentinel is not None:
_release_audio_sentinel(_audio_sentinel)
nodes/seedvr.py
+1 -1
View File
@@ -701,7 +701,7 @@ class LightX2VOutputVideoPreview:
"AUDIO",
{"tooltip": "Optional audio from Load Video/Get Video Components to merge into the output MP4."},
),
}
},
}
RETURN_TYPES = ("STRING",)
nodes/swiftvr.py
+2 -6
View File
@@ -81,10 +81,7 @@ def _resolve_output_size(source_width: int, source_height: int, target_short_edg
scale = target_short_edge / source_short_edge
if scale > _MAX_SR_RATIO:
raise ValueError(
f"SwiftVR scale {scale:.3f}x exceeds the supported maximum {_MAX_SR_RATIO:.1f}x; "
f"lower target_short_edge"
)
raise ValueError(f"SwiftVR scale {scale:.3f}x exceeds the supported maximum {_MAX_SR_RATIO:.1f}x; lower target_short_edge")
if source_width <= source_height:
output_width = target_short_edge
output_height = int(round(source_height * scale))
@@ -97,8 +94,7 @@ def _resolve_output_size(source_width: int, source_height: int, target_short_edg
output_height = max(2, (output_height + 1) // 2 * 2)
if max(output_width, output_height) > _MAX_OUTPUT_DIMENSION:
raise ValueError(
f"SwiftVR output {output_width}x{output_height} exceeds the maximum supported dimension "
f"{_MAX_OUTPUT_DIMENSION}; lower target_short_edge"
f"SwiftVR output {output_width}x{output_height} exceeds the maximum supported dimension {_MAX_OUTPUT_DIMENSION}; lower target_short_edge"
)
return output_height, output_width, scale