From c13737d6d049a30bcdd93079d7710744329274bd Mon Sep 17 00:00:00 2001 From: gaclove Date: Fri, 23 Oct 2026 14:26:36 +0800 Subject: [PATCH] fix: finalize LightX2V 0.6 wrapper migration --- .gitignore | 2 +- .../lightx2v_seko_talk_static.json | 150 ++++++++++++++++++ example_workflows/lightx2v_swiftvr_image.json | 2 +- nodes/_audio_shim.py | 15 +- nodes/_image_shim.py | 113 +++++++++++++ nodes/combiner.py | 19 ++- nodes/config.py | 6 +- nodes/inference.py | 30 ++-- nodes/seedvr.py | 2 +- nodes/swiftvr.py | 8 +- 10 files changed, 315 insertions(+), 32 deletions(-) create mode 100644 example_workflows/lightx2v_seko_talk_static.json create mode 100644 nodes/_image_shim.py diff --git a/.gitignore b/.gitignore index 0bbb58e..c44cdb7 100644 --- a/.gitignore +++ b/.gitignore @@ -346,4 +346,4 @@ Temporary Items CLAUDE.md AGENTS.md .gitnexus/ -.claude/ \ No newline at end of file +.claude/ diff --git a/example_workflows/lightx2v_seko_talk_static.json b/example_workflows/lightx2v_seko_talk_static.json new file mode 100644 index 0000000..02f254f --- /dev/null +++ b/example_workflows/lightx2v_seko_talk_static.json @@ -0,0 +1,150 @@ +{ + "11": { + "inputs": { + "audio": "audio_full_female_10s.wav", + "start_time": 0, + "duration": 0 + }, + "class_type": "VHS_LoadAudioUpload", + "_meta": { + "title": "Load Audio (Upload)🎥🅥🅗🅢" + } + }, + "13": { + "inputs": { + "prompt": "A person speaks naturally to the camera, with clear lip movement and subtle facial expressions.", + "negative_prompt": "", + "inference_config": [ + "14", + 0 + ], + "image": [ + "20", + 0 + ], + "audio": [ + "11", + 0 + ], + "memory_config": [ + "16", + 0 + ], + "quantization_config": [ + "17", + 0 + ] + }, + "class_type": "LightX2VConfigCombinerV2", + "_meta": { + "title": "LightX2V Config Combiner V2" + } + }, + "14": { + "inputs": { + "model_cls": "seko_talk", + "model_name": "Wan2.1-R2V0909-Audio-14B-720P-fp8", + "task": "s2v", + "infer_steps": 4, + "seed": 4221706066, + "cfg_scale": 1, + "cfg_scale2": 1, + "sample_shift": 5, + "height": 1280, + "width": 720, + "duration": 5, + "attention_type": "sage_attn2", + "denoising_steps": "", + "resize_mode": "adaptive", + "fixed_area": "720p", + "segment_length": 81, + "prev_frame_length": 5, + "use_tiny_vae": false + }, + "class_type": "LightX2VInferenceConfig", + "_meta": { + "title": "LightX2V Inference Config" + } + }, + "15": { + "inputs": { + "prepared_config": [ + "13", + 0 + ] + }, + "class_type": "LightX2VModularInferenceV2", + "_meta": { + "title": "LightX2V Modular Inference V2" + } + }, + "16": { + "inputs": { + "enable_rotary_chunk": false, + "rotary_chunk_size": 100, + "clean_cuda_cache": true, + "cpu_offload": true, + "offload_granularity": "block", + "offload_ratio": 1.0, + "t5_cpu_offload": true, + "t5_offload_granularity": "model", + "audio_encoder_cpu_offload": true, + "audio_adapter_cpu_offload": true, + "vae_cpu_offload": true, + "use_tiling_vae": true, + "lazy_load": false, + "unload_after_inference": false + }, + "class_type": "LightX2VMemoryOptimization", + "_meta": { + "title": "LightX2V Memory Optimization" + } + }, + "17": { + "inputs": { + "dit_quant_scheme": "fp8-sgl", + "t5_quant_scheme": "fp8-sgl", + "clip_quant_scheme": "fp8-sgl", + "adapter_quant_scheme": "fp8-sgl" + }, + "class_type": "LightX2VQuantization", + "_meta": { + "title": "LightX2V Quantization" + } + }, + "18": { + "inputs": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "lightx2v_sekotalk/static", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": false, + "trim_to_audio": false, + "pingpong": false, + "save_output": true, + "images": [ + "15", + 0 + ], + "audio": [ + "15", + 1 + ] + }, + "class_type": "VHS_VideoCombine", + "_meta": { + "title": "Video Combine 🎥🅥🅗🅢" + } + }, + "20": { + "inputs": { + "image": "codex_image_20260922_135831_1323a530_B.jpg" + }, + "class_type": "LoadImage", + "_meta": { + "title": "Load Image" + } + } +} diff --git a/example_workflows/lightx2v_swiftvr_image.json b/example_workflows/lightx2v_swiftvr_image.json index b627b4d..5bec0cf 100644 --- a/example_workflows/lightx2v_swiftvr_image.json +++ b/example_workflows/lightx2v_swiftvr_image.json @@ -202,4 +202,4 @@ "VHS_KeepIntermediate": true }, "version": 0.4 -} \ No newline at end of file +} diff --git a/nodes/_audio_shim.py b/nodes/_audio_shim.py index 8db60fd..aa073fc 100644 --- a/nodes/_audio_shim.py +++ b/nodes/_audio_shim.py @@ -41,6 +41,17 @@ _PATCH_SITES = ( ) +class _AudioSentinel(str): + """A path-shaped token that releases its registry lease when discarded.""" + + def __del__(self): + try: + release(str(self)) + except Exception: + # Destructors can run during interpreter shutdown. + pass + + def register(waveform: torch.Tensor, sample_rate: int) -> str: """Register a [C, T] float32 waveform; return a sentinel "path" for it.""" if waveform.dim() != 2: @@ -50,12 +61,12 @@ def register(waveform: torch.Tensor, sample_rate: int) -> str: stashed = waveform.detach().to(torch.float32).cpu().contiguous() with _LOCK: _REGISTRY[sentinel] = (stashed, int(sample_rate)) - return sentinel + return _AudioSentinel(sentinel) def release(sentinel: str) -> None: with _LOCK: - _REGISTRY.pop(sentinel, None) + _REGISTRY.pop(str(sentinel), None) def is_sentinel(path) -> bool: diff --git a/nodes/_image_shim.py b/nodes/_image_shim.py new file mode 100644 index 0000000..25dd7fe --- /dev/null +++ b/nodes/_image_shim.py @@ -0,0 +1,113 @@ +"""Keep ComfyUI IMAGE inputs in memory while LightX2V reads them. + +LightX2V's runner API names the request field ``image_path`` and accepts a +PIL image at the final read boundary. The wrapper used to serialize the first +ComfyUI frame to a temporary PNG only to have the runner decode it again. This +shim keeps the path-shaped request contract while replacing that disk +round-trip with a short-lived in-memory PIL image. +""" + +from __future__ import annotations + +import logging +import threading +import types +import uuid +from typing import Dict + +import torch +from PIL import Image + +SENTINEL_PREFIX = " str: + """Register the first ComfyUI IMAGE frame and return a path-shaped token.""" + if not torch.is_tensor(images) or images.ndim != 4: + raise ValueError(f"IMAGE must be a [B, H, W, C] tensor; got {type(images).__name__} {getattr(images, 'shape', '')}") + if images.shape[-1] not in (1, 3, 4): + raise ValueError(f"IMAGE must have 1, 3, or 4 channels; got shape {tuple(images.shape)}") + + frame = images[0].detach().cpu().float().clamp(0.0, 1.0) + frame_u8 = (frame.mul(255.0).round()).to(torch.uint8).numpy() + if frame_u8.shape[-1] == 1: + frame_u8 = frame_u8[..., 0] + elif frame_u8.shape[-1] == 4: + frame_u8 = frame_u8[..., :3] + + token = f"{SENTINEL_PREFIX}{uuid.uuid4().hex}{SENTINEL_SUFFIX}" + image = Image.fromarray(frame_u8) + if image.mode != "RGB": + image = image.convert("RGB") + with _LOCK: + _REGISTRY[token] = image + return _ImageSentinel(token) + + +def release(sentinel: str | None) -> None: + if not is_sentinel(sentinel): + return + with _LOCK: + _REGISTRY.pop(str(sentinel), None) + + +def is_sentinel(path) -> bool: + return isinstance(path, str) and path.startswith(SENTINEL_PREFIX) + + +def _lookup(sentinel: str) -> Image.Image: + with _LOCK: + image = _REGISTRY.get(sentinel) + if image is None: + raise FileNotFoundError(f"Stale LightX2V in-memory image sentinel: {sentinel}") + return image + + +def install(runner) -> bool: + """Patch one runner's image read boundary; return whether it was patched.""" + if runner is None or not hasattr(runner, "read_image_input"): + return False + if hasattr(runner, "_lightx2v_original_read_image_input"): + return True + + original = runner.read_image_input + + def _read_image_input(self, image_path): + if is_sentinel(image_path): + image_path = _lookup(image_path) + return original(image_path) + + runner._lightx2v_original_read_image_input = original + runner.read_image_input = types.MethodType(_read_image_input, runner) + logging.info("_image_shim installed on %s", type(runner).__name__) + return True + + +def install_on_runner_tree(runner) -> int: + """Patch a regular runner or every clip runner inside an rs2v pipeline.""" + targets = [runner] + clip_generators = getattr(runner, "clip_generators", None) + if isinstance(clip_generators, dict): + targets.extend(clip_generators.values()) + return sum(1 for target in targets if install(target)) diff --git a/nodes/combiner.py b/nodes/combiner.py index f1b64a6..2091c11 100644 --- a/nodes/combiner.py +++ b/nodes/combiner.py @@ -168,19 +168,24 @@ class _BaseConfigCombiner: if image is None: raise ValueError("i2v or s2v or rs2v task requires input image") - image_np = (image[0].cpu().numpy() * 255).astype(np.uint8) - pil_image = Image.fromarray(image_np) - temp_path = self.temp_manager.create_temp_file(suffix=".png") - pil_image.save(temp_path) - config.image_path = temp_path - logging.info(f"Image saved to {temp_path}") + frame_count = int(image.shape[0]) if hasattr(image, "shape") and image.ndim > 0 else 0 + if frame_count > 1: + logging.warning( + "LightX2V uses only the first IMAGE frame as the reference; received %s frames", + frame_count, + ) + + from ._image_shim import register + + config.image_path = register(image) + logging.info("Routed ComfyUI IMAGE through the in-memory shim") def _save_single_audio_if_needed(self, config, audio): # Route ComfyUI AUDIO straight into the runner via the in-memory shim # (no WAV temp file, no soundfile round-trip). The shim only intercepts # this single-AUDIO path; V3 multi-talker padding still produces real # files since its inputs are external paths/URLs, not ComfyUI tensors. - if audio is None or not hasattr(config, "model_cls") or "seko" not in config.model_cls: + if audio is None or config.task not in {"s2v", "rs2v"}: return from ._audio_shim import comfyui_audio_to_loader_pair, install, register diff --git a/nodes/config.py b/nodes/config.py index 4d85ef3..5cbecb7 100644 --- a/nodes/config.py +++ b/nodes/config.py @@ -42,7 +42,7 @@ class LightX2VInferenceConfig: ["t2v", "i2v", "s2v", "rs2v"], { "default": "i2v", - "tooltip": "Task type: text-to-video or image-to-video or reference_image and audio to video (shot)", + "tooltip": "Task: t2v text-to-video, i2v image-to-video, s2v image+audio-to-video, or rs2v reference-shot image+audio-to-video", }, ), "infer_steps": ( @@ -136,7 +136,7 @@ class LightX2VInferenceConfig: ], { "default": "adaptive", - "tooltip": "Adaptive resize input image to target aspect ratio", + "tooltip": "Resize policy: adaptive preserves the target area, fixed_shape uses the exact target size, and fixed-area/min-side modes preserve aspect ratio", }, ), "fixed_area": ( @@ -161,7 +161,7 @@ class LightX2VInferenceConfig: "default": 5, "min": 0, "max": 16, - "tooltip": "Previous frame overlap for sekotalk models", + "tooltip": "Previous-frame overlap for rs2v continuation; s2v uses the model's own initial reference and usually ignores this value", }, ), "use_tiny_vae": ( diff --git a/nodes/inference.py b/nodes/inference.py index d079ff8..633d4bd 100644 --- a/nodes/inference.py +++ b/nodes/inference.py @@ -100,9 +100,7 @@ class LightX2VModularInferenceV2: int(config.get("target_height", config.get("height", 480))), int(config.get("target_width", config.get("width", 832))), ] - startup_data["num_frames"] = int( - config.get("target_video_length", config.get("video_length", config.get("num_frames", 81))) - ) + startup_data["num_frames"] = int(config.get("target_video_length", config.get("video_length", config.get("num_frames", 81)))) formatted = _modern_build_startup_config(startup_data) else: # set_args2config strips any key that's part of an InputInfo @@ -143,9 +141,7 @@ class LightX2VModularInferenceV2: int(config.get("target_height", config.get("height", 480))), int(config.get("target_width", config.get("width", 832))), ] - startup_data["num_frames"] = int( - config.get("target_video_length", config.get("video_length", config.get("num_frames", 81))) - ) + startup_data["num_frames"] = int(config.get("target_video_length", config.get("video_length", config.get("num_frames", 81)))) return _modern_build_startup_config(startup_data) @staticmethod @@ -168,9 +164,7 @@ class LightX2VModularInferenceV2: int(config.get("target_width", config.get("width", 832))), ] elif field == "num_frames": - request[field] = int( - config.get("target_video_length", config.get("video_length", config.get("num_frames", 81))) - ) + request[field] = int(config.get("target_video_length", config.get("video_length", config.get("num_frames", 81)))) elif field in config: request[field] = config[field] @@ -183,6 +177,14 @@ class LightX2VModularInferenceV2: """Run inference with prepared configuration.""" config = prepared_config + from ._image_shim import install_on_runner_tree as _install_image_shim + from ._image_shim import is_sentinel as _is_image_sentinel + from ._image_shim import release as _release_image_sentinel + + _image_sentinel = config.get("image_path") if isinstance(config, dict) else getattr(config, "image_path", None) + if not _is_image_sentinel(_image_sentinel): + _image_sentinel = None + # Combiner may have stashed the input AUDIO in the in-memory shim and # set audio_path to a sentinel — release it on exit so the tensor isn't # retained across runs (one ComfyUI graph tick = one sentinel). @@ -199,7 +201,7 @@ class LightX2VModularInferenceV2: current_runner = getattr(self.__class__, "_current_runner", None) current_config_hash = getattr(self.__class__, "_current_config_hash", None) - needs_reinit = current_runner is None or current_config_hash != config_hash or getattr(config, "lazy_load", False) + needs_reinit = current_runner is None or current_config_hash != config_hash or config.get("lazy_load", False) logging.info(f"Needs reinit: {needs_reinit}, old config hash: {current_config_hash}, new config hash: {config_hash}") if needs_reinit: @@ -240,6 +242,10 @@ class LightX2VModularInferenceV2: current_runner = self.__class__._current_runner + if _image_sentinel is not None: + if _install_image_shim(current_runner) == 0: + raise RuntimeError("LightX2V IMAGE input shim could not find a runner image reader") + if hasattr(current_runner, "set_progress_callback"): current_runner.set_progress_callback(update_progress) @@ -279,7 +285,7 @@ class LightX2VModularInferenceV2: if images.dtype != torch.float32: images = images.float() - if getattr(config, "unload_after_inference", False): + if config.get("unload_after_inference", False): current_runner = None # drop local ref so __del__ can run self._release_runner() else: @@ -292,5 +298,7 @@ class LightX2VModularInferenceV2: logging.error(f"Error during inference: {e}") raise finally: + if _image_sentinel is not None: + _release_image_sentinel(_image_sentinel) if _audio_sentinel is not None: _release_audio_sentinel(_audio_sentinel) diff --git a/nodes/seedvr.py b/nodes/seedvr.py index 256a3ac..25a3eeb 100644 --- a/nodes/seedvr.py +++ b/nodes/seedvr.py @@ -701,7 +701,7 @@ class LightX2VOutputVideoPreview: "AUDIO", {"tooltip": "Optional audio from Load Video/Get Video Components to merge into the output MP4."}, ), - } + }, } RETURN_TYPES = ("STRING",) diff --git a/nodes/swiftvr.py b/nodes/swiftvr.py index a394aa1..f7e6774 100644 --- a/nodes/swiftvr.py +++ b/nodes/swiftvr.py @@ -81,10 +81,7 @@ def _resolve_output_size(source_width: int, source_height: int, target_short_edg scale = target_short_edge / source_short_edge if scale > _MAX_SR_RATIO: - raise ValueError( - f"SwiftVR scale {scale:.3f}x exceeds the supported maximum {_MAX_SR_RATIO:.1f}x; " - f"lower target_short_edge" - ) + raise ValueError(f"SwiftVR scale {scale:.3f}x exceeds the supported maximum {_MAX_SR_RATIO:.1f}x; lower target_short_edge") if source_width <= source_height: output_width = target_short_edge output_height = int(round(source_height * scale)) @@ -97,8 +94,7 @@ def _resolve_output_size(source_width: int, source_height: int, target_short_edg output_height = max(2, (output_height + 1) // 2 * 2) if max(output_width, output_height) > _MAX_OUTPUT_DIMENSION: raise ValueError( - f"SwiftVR output {output_width}x{output_height} exceeds the maximum supported dimension " - f"{_MAX_OUTPUT_DIMENSION}; lower target_short_edge" + f"SwiftVR output {output_width}x{output_height} exceeds the maximum supported dimension {_MAX_OUTPUT_DIMENSION}; lower target_short_edge" ) return output_height, output_width, scale