From e57e93a91d75913aacf4897a229dde0c7c52d99e Mon Sep 17 00:00:00 2001 From: gaclove Date: Thu, 20 Nov 2025 16:26:44 +0800 Subject: [PATCH 1/8] refactor: change denoising_step_list type from List[int] to List[float] in InferenceConfig for improved flexibility --- config_builder.py | 2 +- data_models.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config_builder.py b/config_builder.py index 97d557c..2352102 100644 --- a/config_builder.py +++ b/config_builder.py @@ -140,7 +140,7 @@ class InferenceConfigBuilder: steps_str = optional_params["denoising_steps"] if steps_str and steps_str.strip(): try: - steps_list = [int(s.strip()) for s in steps_str.split(",")] + steps_list = [float(s.strip()) for s in steps_str.split(",")] config.denoising_step_list = steps_list config.infer_steps = len(steps_list) except ValueError: diff --git a/data_models.py b/data_models.py index 072b85b..2022f8b 100644 --- a/data_models.py +++ b/data_models.py @@ -53,7 +53,7 @@ class InferenceConfig: use_31_block: bool = True # Optional parameters - denoising_step_list: Optional[List[int]] = None + denoising_step_list: Optional[List[float]] = None resize_mode: str = "adaptive" fixed_area: str = "720p" segment_length: int = 81 From 50f037a5b4881f5a1c67bb887050f8a2c93372c9 Mon Sep 17 00:00:00 2001 From: gaclove Date: Thu, 20 Nov 2025 18:03:53 +0000 Subject: [PATCH 2/8] feat: add pre_frame --- nodes.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/nodes.py b/nodes.py index 12d5b77..7952e77 100644 --- a/nodes.py +++ b/nodes.py @@ -771,8 +771,8 @@ class LightX2VModularInference: }, } - RETURN_TYPES = ("IMAGE", "AUDIO") - RETURN_NAMES = ("images", "audio") + RETURN_TYPES = ("IMAGE", "AUDIO", "PREV_FRAMES") + RETURN_NAMES = ("images", "audio", "prev_frames") FUNCTION = "generate" CATEGORY = "LightX2V/Inference" @@ -918,6 +918,7 @@ class LightX2VModularInference: result_dict = current_runner.run_pipeline(input_info) images = result_dict.get("video", None) audio = result_dict.get("audio", None) + for_next_section = result_dict.get("for_next_section", None) if images is not None and images.numel() > 0: images = images.cpu() @@ -933,7 +934,7 @@ class LightX2VModularInference: torch.cuda.empty_cache() gc.collect() - return (images, audio) + return (images, audio, for_next_section) except Exception as e: logging.error(f"Error during inference: {e}") @@ -992,6 +993,7 @@ class LightX2VConfigCombinerV2: "AUDIO", {"tooltip": "Input audio for audio-driven generation for s2v task"}, ), + "prev_frames": ("PREV_FRAMES", {"tooltip": "Previous frames for s2v task"}), }, } @@ -1012,6 +1014,7 @@ class LightX2VConfigCombinerV2: talk_objects_config=None, image=None, audio=None, + prev_frames=None, ): """Combine configurations and prepare data for inference.""" @@ -1058,6 +1061,13 @@ class LightX2VConfigCombinerV2: config.audio_path = temp_path logging.info(f"Audio saved to {temp_path}") + # Handle previous frames input + if prev_frames is not None: + temp_path = self.temp_manager.create_temp_file(suffix=".pt") + torch.save(prev_frames, temp_path) + config.prev_section_info_path = temp_path + logging.info(f"Previous frames saved to {temp_path}") + # Handle talk objects if hasattr(config, "talk_objects") and config.talk_objects: talk_objects = config.talk_objects @@ -1720,8 +1730,8 @@ class LightX2VModularInferenceV2: }, } - RETURN_TYPES = ("IMAGE", "AUDIO") - RETURN_NAMES = ("images", "audio") + RETURN_TYPES = ("IMAGE", "AUDIO", "PREV_FRAMES") + RETURN_NAMES = ("images", "audio", "prev_frames") FUNCTION = "generate" CATEGORY = "LightX2V/InferenceV2" @@ -1765,6 +1775,7 @@ class LightX2VModularInferenceV2: config["return_result_tensor"] = True config["save_result_path"] = "" config["negative_prompt"] = config.get("negative_prompt", "") + config["prev_section_info_path"] = config.get("prev_section_info_path", "") input_info = set_input_info(config) current_runner.set_config(config) @@ -1772,6 +1783,7 @@ class LightX2VModularInferenceV2: images = result_dict.get("video", None) audio = result_dict.get("audio", None) + prev_frames = result_dict.get("for_next_section", None) if images is not None and images.numel() > 0: images = images.cpu() @@ -1787,7 +1799,7 @@ class LightX2VModularInferenceV2: torch.cuda.empty_cache() gc.collect() - return (images, audio) + return (images, audio, prev_frames) except Exception as e: logging.error(f"Error during inference: {e}") From e6ed13773b6a8e0e3ca7356723193e30c39290c8 Mon Sep 17 00:00:00 2001 From: gaclove Date: Wed, 26 Nov 2025 12:11:01 +0000 Subject: [PATCH 3/8] feat: add first image input handling in LightX2VConfigCombinerV2 for s2v task --- nodes.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/nodes.py b/nodes.py index 7952e77..b7350b7 100644 --- a/nodes.py +++ b/nodes.py @@ -994,6 +994,7 @@ class LightX2VConfigCombinerV2: {"tooltip": "Input audio for audio-driven generation for s2v task"}, ), "prev_frames": ("PREV_FRAMES", {"tooltip": "Previous frames for s2v task"}), + "frist_image": ("IMAGE", {"tooltip": "First image for s2v task"}), }, } @@ -1015,6 +1016,7 @@ class LightX2VConfigCombinerV2: image=None, audio=None, prev_frames=None, + frist_image=None, ): """Combine configurations and prepare data for inference.""" @@ -1061,6 +1063,15 @@ class LightX2VConfigCombinerV2: config.audio_path = temp_path logging.info(f"Audio saved to {temp_path}") + # Handle first image input + if frist_image is not None: + frist_image_np = (frist_image[0].cpu().numpy() * 255).astype(np.uint8) + pil_image = Image.fromarray(frist_image_np) + temp_path = self.temp_manager.create_temp_file(suffix=".png") + pil_image.save(temp_path) + config.frist_image_path = temp_path + logging.info(f"First image saved to {temp_path}") + # Handle previous frames input if prev_frames is not None: temp_path = self.temp_manager.create_temp_file(suffix=".pt") From 04f7fd09fdb8597e128a9208eed71f9d680ed2ec Mon Sep 17 00:00:00 2001 From: gaclove Date: Thu, 27 Nov 2025 10:26:28 +0000 Subject: [PATCH 4/8] feat: add rope_type configuration to LightX2VDefaultConfig --- bridge.py | 1 + lightx2v | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/bridge.py b/bridge.py index 5a0fe70..1f589e0 100644 --- a/bridge.py +++ b/bridge.py @@ -161,6 +161,7 @@ class LightX2VDefaultConfig: "return_video": True, "talk_objects": None, "boundary_step_index": 2, + "rope_type": "torch", } diff --git a/lightx2v b/lightx2v index cf0332c..b85f25c 160000 --- a/lightx2v +++ b/lightx2v @@ -1 +1 @@ -Subproject commit cf0332ca8334191e37258cf0f859681f82282006 +Subproject commit b85f25c76057aa21458cadd872ff98c29b7abf13 From 39ea1a8128c7deb0f6a79448bfb608f874c88ee9 Mon Sep 17 00:00:00 2001 From: gaclove Date: Fri, 28 Nov 2025 05:47:54 +0000 Subject: [PATCH 5/8] feat: upgrade lightx2v --- __init__.py | 1 + lightx2v | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/__init__.py b/__init__.py index d9f99c0..9b7eb3e 100644 --- a/__init__.py +++ b/__init__.py @@ -6,6 +6,7 @@ os.environ["TOKENIZERS_PARALLELISM"] = "false" os.environ["ENABLE_GRAPH_MODE"] = "false" os.environ["ENABLE_PROFILING_DEBUG"] = "true" # os.environ["SENSITIVE_LAYER_DTYPE"] = "FP32" +os.environ["PROFILING_DEBUG_LEVEL"] = "2" os.environ["DTYPE"] = "BF16" current_path = Path(__file__).parent.absolute() diff --git a/lightx2v b/lightx2v index b85f25c..0807e60 160000 --- a/lightx2v +++ b/lightx2v @@ -1 +1 @@ -Subproject commit b85f25c76057aa21458cadd872ff98c29b7abf13 +Subproject commit 0807e60830d9d572456a999cf54ac3546765a043 From 12889efc55e4d2f52872c971b80955b6320ef135 Mon Sep 17 00:00:00 2001 From: gaclove Date: Mon, 1 Dec 2025 13:23:19 +0000 Subject: [PATCH 6/8] feat: refactor LightX2VConfigCombiner classes for improved configuration handling and introduce ExtractLastNFrames utility for video processing --- nodes.py | 629 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 338 insertions(+), 291 deletions(-) diff --git a/nodes.py b/nodes.py index b7350b7..d233015 100644 --- a/nodes.py +++ b/nodes.py @@ -945,8 +945,8 @@ class LightX2VModularInference: pass -class LightX2VConfigCombinerV2: - """Config combiner that also handles data preparation (image/audio/prompts).""" +class LightX2VConfigCombinerBase: + """Base class for config combiners with common functionality.""" def __init__(self): self.config_builder = ConfigBuilder() @@ -956,6 +956,129 @@ class LightX2VConfigCombinerV2: self.resolver = ComfyUIFileResolver() self.http_downloader = HTTPFileDownloader() + def _build_base_config( + self, + inference_config, + prompt, + negative_prompt, + teacache_config=None, + quantization_config=None, + memory_config=None, + lora_chain=None, + talk_objects_config=None, + ): + """Build base config from input configurations.""" + inf_config = InferenceConfig(**inference_config) if isinstance(inference_config, dict) else inference_config + tea_config = TeaCacheConfig(**teacache_config) if teacache_config and isinstance(teacache_config, dict) else teacache_config + quant_config = ( + QuantizationConfig(**quantization_config) if quantization_config and isinstance(quantization_config, dict) else quantization_config + ) + mem_config = MemoryOptimizationConfig(**memory_config) if memory_config and isinstance(memory_config, dict) else memory_config + + config = self.config_builder.combine_configs( + inference_config=inf_config, + teacache_config=tea_config, + quantization_config=quant_config, + memory_config=mem_config, + lora_chain=lora_chain, + talk_objects_config=talk_objects_config, + ) + + config.prompt = prompt + config.negative_prompt = negative_prompt + return config + + def _handle_image_input(self, config, image): + """Handle image input for i2v or s2v task.""" + if config.task in ["i2v", "s2v"] and image is None: + raise ValueError("i2v or s2v task requires input image") + + if config.task in ["i2v", "s2v"] and image is not None: + image_np = (image[0].cpu().numpy() * 255).astype(np.uint8) + pil_image = Image.fromarray(image_np) + temp_path = self.temp_manager.create_temp_file(suffix=".png") + pil_image.save(temp_path) + config.image_path = temp_path + logging.info(f"Image saved to {temp_path}") + + def _handle_audio_input(self, config, audio): + """Handle audio input for seko models.""" + if audio is not None and hasattr(config, "model_cls") and "seko" in config.model_cls: + temp_path = self.temp_manager.create_temp_file(suffix=".wav") + self.audio_handler.save(audio, temp_path) + config.audio_path = temp_path + logging.info(f"Audio saved to {temp_path}") + + def _resolve_path(self, path, prefix): + """Resolve a path: download if URL, resolve if relative.""" + if self.http_downloader.is_url(path): + try: + downloaded_path = self.http_downloader.download_if_url(path, prefix=prefix) + logging.info(f"Downloaded {prefix} from URL: {path} -> {downloaded_path}") + return downloaded_path, True + except Exception as e: + logging.error(f"Failed to download {prefix} from {path}: {e}") + return path, False + elif not os.path.isabs(path) and not path.startswith("/tmp"): + resolved = self.resolver.resolve_input_path(path) + logging.info(f"Resolved {prefix} path: {path} -> {resolved}") + return resolved, True + return path, True + + def _extract_talk_objects(self, config): + """Extract talk objects from config.""" + if not hasattr(config, "talk_objects") or not config.talk_objects: + return [] + + result = [] + for talk_obj in config.talk_objects: + obj = {} + if "audio" in talk_obj: + obj["audio"] = talk_obj["audio"] + if "mask" in talk_obj: + obj["mask"] = talk_obj["mask"] + if "audio" in obj: + result.append(obj) + return result + + def _resolve_talk_object_paths(self, talk_objects): + """Resolve paths in talk objects (audio and mask).""" + for obj in talk_objects: + if "audio" in obj and obj["audio"]: + resolved_path, success = self._resolve_path(obj["audio"], "audio") + if not success: + continue + obj["audio"] = resolved_path + if not os.path.exists(obj["audio"]): + logging.warning(f"Audio file not found: {obj['audio']}") + + if "mask" in obj and obj["mask"]: + resolved_path, _ = self._resolve_path(obj["mask"], "mask") + obj["mask"] = resolved_path + if not os.path.exists(obj["mask"]): + logging.warning(f"Mask file not found: {obj['mask']}") + + return talk_objects + + def _save_talk_objects_config(self, config, processed_talk_objects): + """Save processed talk objects to config.""" + if not processed_talk_objects: + return + + if len(processed_talk_objects) == 1 and not processed_talk_objects[0].get("mask", "").strip(): + config.audio_path = processed_talk_objects[0]["audio"] + logging.info(f"Convert Processed 1 talk object to audio path: {config.audio_path}") + else: + temp_dir = self.temp_manager.create_temp_dir() + with open(os.path.join(temp_dir, "config.json"), "w") as f: + json.dump({"talk_objects": processed_talk_objects}, f) + config.audio_path = temp_dir + logging.info(f"Processed {len(processed_talk_objects)} talk objects") + + +class LightX2VConfigCombinerV2(LightX2VConfigCombinerBase): + """Config combiner that also handles data preparation (image/audio/prompts).""" + @classmethod def INPUT_TYPES(cls): return { @@ -1019,51 +1142,21 @@ class LightX2VConfigCombinerV2: frist_image=None, ): """Combine configurations and prepare data for inference.""" - - # Convert dict configs back to objects if needed - inf_config = InferenceConfig(**inference_config) if isinstance(inference_config, dict) else inference_config - tea_config = TeaCacheConfig(**teacache_config) if teacache_config and isinstance(teacache_config, dict) else teacache_config - quant_config = ( - QuantizationConfig(**quantization_config) if quantization_config and isinstance(quantization_config, dict) else quantization_config - ) - mem_config = MemoryOptimizationConfig(**memory_config) if memory_config and isinstance(memory_config, dict) else memory_config - - # Build combined config - config = self.config_builder.combine_configs( - inference_config=inf_config, - teacache_config=tea_config, - quantization_config=quant_config, - memory_config=mem_config, - lora_chain=lora_chain, - talk_objects_config=talk_objects_config, + config = self._build_base_config( + inference_config, + prompt, + negative_prompt, + teacache_config, + quantization_config, + memory_config, + lora_chain, + talk_objects_config, ) - # Add prompts to config - config.prompt = prompt - config.negative_prompt = negative_prompt + self._handle_image_input(config, image) + self._handle_audio_input(config, audio) - # Validate task requirements - if config.task in ["i2v", "s2v"] and image is None: - raise ValueError("i2v or s2v task requires input image") - - # Handle image input - if config.task in ["i2v", "s2v"] and image is not None: - image_np = (image[0].cpu().numpy() * 255).astype(np.uint8) - pil_image = Image.fromarray(image_np) - - temp_path = self.temp_manager.create_temp_file(suffix=".png") - pil_image.save(temp_path) - config.image_path = temp_path - logging.info(f"Image saved to {temp_path}") - - # Handle audio input for seko models - if audio is not None and hasattr(config, "model_cls") and "seko" in config.model_cls: - temp_path = self.temp_manager.create_temp_file(suffix=".wav") - self.audio_handler.save(audio, temp_path) - config.audio_path = temp_path - logging.info(f"Audio saved to {temp_path}") - - # Handle first image input + # Handle first image input (V2 specific) if frist_image is not None: frist_image_np = (frist_image[0].cpu().numpy() * 255).astype(np.uint8) pil_image = Image.fromarray(frist_image_np) @@ -1072,100 +1165,26 @@ class LightX2VConfigCombinerV2: config.frist_image_path = temp_path logging.info(f"First image saved to {temp_path}") - # Handle previous frames input + # Handle previous frames input (V2 specific) if prev_frames is not None: temp_path = self.temp_manager.create_temp_file(suffix=".pt") - torch.save(prev_frames, temp_path) + prev_section_info = {"prev_video": prev_frames, "prev_latent": None} + torch.save(prev_section_info, temp_path) config.prev_section_info_path = temp_path - logging.info(f"Previous frames saved to {temp_path}") + logging.info(f"Previous frames saved to {temp_path}, shape: {prev_frames.shape}") # Handle talk objects - if hasattr(config, "talk_objects") and config.talk_objects: - talk_objects = config.talk_objects - processed_talk_objects = [] - - for talk_obj in talk_objects: - processed_obj = {} - - if "audio" in talk_obj: - processed_obj["audio"] = talk_obj["audio"] - - if "mask" in talk_obj: - processed_obj["mask"] = talk_obj["mask"] - - if "audio" in processed_obj: - processed_talk_objects.append(processed_obj) - - # Resolve paths and download URLs - for obj in processed_talk_objects: - if "audio" in obj and obj["audio"]: - audio_path = obj["audio"] - - # Check if it's a URL and download if needed - if self.http_downloader.is_url(audio_path): - try: - downloaded_path = self.http_downloader.download_if_url(audio_path, prefix="audio") - obj["audio"] = downloaded_path - logging.info(f"Downloaded audio from URL: {audio_path} -> {downloaded_path}") - except Exception as e: - logging.error(f"Failed to download audio from {audio_path}: {e}") - continue - # Handle relative paths - elif not os.path.isabs(audio_path) and not audio_path.startswith("/tmp"): - obj["audio"] = self.resolver.resolve_input_path(audio_path) - logging.info(f"Resolved audio path: {audio_path} -> {obj['audio']}") - - # Check if file exists - if not os.path.exists(obj["audio"]): - logging.warning(f"Audio file not found: {obj['audio']}") - - if "mask" in obj and obj["mask"]: - mask_path = obj["mask"] - - # Check if it's a URL and download if needed - if self.http_downloader.is_url(mask_path): - try: - downloaded_path = self.http_downloader.download_if_url(mask_path, prefix="mask") - obj["mask"] = downloaded_path - logging.info(f"Downloaded mask from URL: {mask_path} -> {downloaded_path}") - except Exception as e: - logging.error(f"Failed to download mask from {mask_path}: {e}") - # Don't skip the object if mask download fails (mask is optional) - # Handle relative paths - elif not os.path.isabs(mask_path) and not mask_path.startswith("/tmp"): - obj["mask"] = self.resolver.resolve_input_path(mask_path) - logging.info(f"Resolved mask path: {mask_path} -> {obj['mask']}") - - # Check if file exists - if not os.path.exists(obj["mask"]): - logging.warning(f"Mask file not found: {obj['mask']}") - - if processed_talk_objects: - if len(processed_talk_objects) == 1 and not processed_talk_objects[0].get("mask", "").strip(): - config.audio_path = processed_talk_objects[0]["audio"] - logging.info(f"Convert Processed 1 talk object to audio path: {config.audio_path}") - else: - temp_dir = self.temp_manager.create_temp_dir() - with open(os.path.join(temp_dir, "config.json"), "w") as f: - json.dump({"talk_objects": processed_talk_objects}, f) - config.audio_path = temp_dir - logging.info(f"Processed {len(processed_talk_objects)} talk objects") + talk_objects = self._extract_talk_objects(config) + if talk_objects: + processed = self._resolve_talk_object_paths(talk_objects) + self._save_talk_objects_config(config, processed) logging.info("lightx2v prepared config: " + json.dumps(config, indent=2, ensure_ascii=False)) - return (config,) -class LightX2VConfigCombinerV3: - """Config combiner that also handles data preparation (image/audio/prompts).""" - - def __init__(self): - self.config_builder = ConfigBuilder() - self.temp_manager = TempFileManager() - self.image_handler = ImageFileHandler() - self.audio_handler = AudioFileHandler() - self.resolver = ComfyUIFileResolver() - self.http_downloader = HTTPFileDownloader() +class LightX2VConfigCombinerV3(LightX2VConfigCombinerBase): + """Config combiner with advanced talk objects processing (audio extension, background generation).""" @staticmethod def extend_mp3(input_path: str, output_path: str, duration: float) -> bool: @@ -1527,6 +1546,82 @@ class LightX2VConfigCombinerV3: FUNCTION = "prepare_config" CATEGORY = "LightX2V/ConfigV2" + def _resolve_talk_objects_with_duration(self, talk_objects): + """Resolve paths and get audio duration for each talk object.""" + max_duration = None + for obj in talk_objects: + if "audio" in obj and obj["audio"]: + resolved_path, success = self._resolve_path(obj["audio"], "audio") + if not success: + continue + obj["audio"] = resolved_path + if not os.path.exists(obj["audio"]): + logging.warning(f"Audio file not found: {obj['audio']}") + duration = self.get_audio_duration(obj["audio"]) + obj["duration"] = duration + if max_duration is None or duration > max_duration: + max_duration = duration + + if "mask" in obj and obj["mask"]: + resolved_path, _ = self._resolve_path(obj["mask"], "mask") + obj["mask"] = resolved_path + if not os.path.exists(obj["mask"]): + logging.warning(f"Mask file not found: {obj['mask']}") + + return talk_objects, max_duration + + def _process_multi_speaker_talk_objects(self, src_talk_objects, max_duration): + """Process multiple talk objects: extend audio and generate background.""" + processed_talk_objects = [] + mask_img_paths = [] + extend_count = 0 + + for obj in src_talk_objects: + dst_obj = {} + src_audio_path = obj["audio"] + src_audio_duration = obj["duration"] + + if max_duration - src_audio_duration > 0.1: + dst_audio_path = self.temp_manager.create_temp_file(suffix=".mp3") + self.extend_mp3(src_audio_path, dst_audio_path, max_duration) + extend_count += 1 + dst_obj["audio"] = dst_audio_path + else: + dst_obj["audio"] = src_audio_path + + src_mask = obj.get("mask", None) + if src_mask: + dst_obj["mask"] = src_mask + mask_img_paths.append(src_mask) + processed_talk_objects.append(dst_obj) + + logging.info(f"Extended {extend_count} audio files") + + # Generate background mask and audio + bg_mask_io = self.generate_background_mask(mask_img_paths) + bg_mask_path = self.temp_manager.create_temp_file(suffix=".jpg") + with open(bg_mask_path, "wb") as f: + f.write(bg_mask_io.getvalue()) + + bg_noise_data = self.generate_white_noise( + duration=max_duration, + framerate=16000, + n_channels=1, + rms=0.00232, + std_dev=0.00232, + ) + wav_io = io.BytesIO() + self.save_wav_file(audio_data=bg_noise_data, output_path=wav_io, framerate=16000, sample_width=2) + bg_audio_path = self.temp_manager.create_temp_file(suffix=".wav") + with open(bg_audio_path, "wb") as f: + f.write(wav_io.getvalue()) + + bg_obj = {"audio": bg_audio_path, "mask": bg_mask_path} + processed_talk_objects.append(bg_obj) + logging.info(f"Generated background mask and audio: {bg_mask_path}, {bg_audio_path}") + + return processed_talk_objects + def prepare_config( self, inference_config, @@ -1541,178 +1636,33 @@ class LightX2VConfigCombinerV3: audio=None, ): """Combine configurations and prepare data for inference.""" - - # Convert dict configs back to objects if needed - inf_config = InferenceConfig(**inference_config) if isinstance(inference_config, dict) else inference_config - tea_config = TeaCacheConfig(**teacache_config) if teacache_config and isinstance(teacache_config, dict) else teacache_config - quant_config = ( - QuantizationConfig(**quantization_config) if quantization_config and isinstance(quantization_config, dict) else quantization_config - ) - mem_config = MemoryOptimizationConfig(**memory_config) if memory_config and isinstance(memory_config, dict) else memory_config - - # Build combined config - config = self.config_builder.combine_configs( - inference_config=inf_config, - teacache_config=tea_config, - quantization_config=quant_config, - memory_config=mem_config, - lora_chain=lora_chain, - talk_objects_config=talk_objects_config, + config = self._build_base_config( + inference_config, + prompt, + negative_prompt, + teacache_config, + quantization_config, + memory_config, + lora_chain, + talk_objects_config, ) - # Add prompts to config - config.prompt = prompt - config.negative_prompt = negative_prompt + self._handle_image_input(config, image) + self._handle_audio_input(config, audio) - # Validate task requirements - if config.task in ["i2v", "s2v"] and image is None: - raise ValueError("i2v or s2v task requires input image") - - # Handle image input - if config.task in ["i2v", "s2v"] and image is not None: - image_np = (image[0].cpu().numpy() * 255).astype(np.uint8) - pil_image = Image.fromarray(image_np) - - temp_path = self.temp_manager.create_temp_file(suffix=".png") - pil_image.save(temp_path) - config.image_path = temp_path - logging.info(f"Image saved to {temp_path}") - - # Handle audio input for seko models - if audio is not None and hasattr(config, "model_cls") and "seko" in config.model_cls: - temp_path = self.temp_manager.create_temp_file(suffix=".wav") - self.audio_handler.save(audio, temp_path) - config.audio_path = temp_path - logging.info(f"Audio saved to {temp_path}") - - # Handle talk objects - if hasattr(config, "talk_objects") and config.talk_objects: - talk_objects = config.talk_objects - src_talk_objects = [] - - for talk_obj in talk_objects: - src_obj = {} - - if "audio" in talk_obj: - src_obj["audio"] = talk_obj["audio"] - - if "mask" in talk_obj: - src_obj["mask"] = talk_obj["mask"] - - if "audio" in src_obj: - src_talk_objects.append(src_obj) - - # Resolve paths and download URLs, - # record the max duration of the src talk objects - max_src_duration = None - for obj in src_talk_objects: - if "audio" in obj and obj["audio"]: - audio_path = obj["audio"] - - # Check if it's a URL and download if needed - if self.http_downloader.is_url(audio_path): - try: - downloaded_path = self.http_downloader.download_if_url(audio_path, prefix="audio") - obj["audio"] = downloaded_path - logging.info(f"Downloaded audio from URL: {audio_path} -> {downloaded_path}") - except Exception as e: - logging.error(f"Failed to download audio from {audio_path}: {e}") - continue - # Handle relative paths - elif not os.path.isabs(audio_path) and not audio_path.startswith("/tmp"): - obj["audio"] = self.resolver.resolve_input_path(audio_path) - logging.info(f"Resolved audio path: {audio_path} -> {obj['audio']}") - - # Check if file exists - if not os.path.exists(obj["audio"]): - logging.warning(f"Audio file not found: {obj['audio']}") - duration = self.get_audio_duration(obj["audio"]) - obj["duration"] = duration - if max_src_duration is None or duration > max_src_duration: - max_src_duration = duration - - if "mask" in obj and obj["mask"]: - mask_path = obj["mask"] - - # Check if it's a URL and download if needed - if self.http_downloader.is_url(mask_path): - try: - downloaded_path = self.http_downloader.download_if_url(mask_path, prefix="mask") - obj["mask"] = downloaded_path - logging.info(f"Downloaded mask from URL: {mask_path} -> {downloaded_path}") - except Exception as e: - logging.error(f"Failed to download mask from {mask_path}: {e}") - # Don't skip the object if mask download fails (mask is optional) - # Handle relative paths - elif not os.path.isabs(mask_path) and not mask_path.startswith("/tmp"): - obj["mask"] = self.resolver.resolve_input_path(mask_path) - logging.info(f"Resolved mask path: {mask_path} -> {obj['mask']}") - - # Check if file exists - if not os.path.exists(obj["mask"]): - logging.warning(f"Mask file not found: {obj['mask']}") + # Handle talk objects with advanced processing (V3 specific) + src_talk_objects = self._extract_talk_objects(config) + if src_talk_objects: + src_talk_objects, max_duration = self._resolve_talk_objects_with_duration(src_talk_objects) if len(src_talk_objects) > 1: - # extend audio duration to the max duration of the src talk objects - processed_talk_objects: list[dict[str, str]] = list() - mask_img_paths = list() - extend_count = 0 - for obj in src_talk_objects: - dst_obj = dict() - src_audio_path = obj["audio"] - src_audio_duration = obj["duration"] - if max_src_duration - src_audio_duration > 0.1: - dst_audio_path = self.temp_manager.create_temp_file(suffix=".mp3") - self.extend_mp3(src_audio_path, dst_audio_path, max_src_duration) - extend_count += 1 - dst_obj["audio"] = dst_audio_path - else: - dst_obj["audio"] = src_audio_path - src_mask = obj.get("mask", None) - if src_mask: - dst_obj["mask"] = src_mask - mask_img_paths.append(src_mask) - processed_talk_objects.append(dst_obj) - logging.info(f"Extended {extend_count} audio files") - # generate background mask and audio - bg_mask_io = self.generate_background_mask(mask_img_paths) - bg_mask_path = self.temp_manager.create_temp_file(suffix=".jpg") - with open(bg_mask_path, "wb") as f: - f.write(bg_mask_io.getvalue()) - bg_noise_data = self.generate_white_noise( - duration=max_src_duration, - framerate=16000, - n_channels=1, - rms=0.00232, - std_dev=0.00232, - ) - wav_io = io.BytesIO() - self.save_wav_file(audio_data=bg_noise_data, output_path=wav_io, framerate=16000, sample_width=2) - bg_audio_path = self.temp_manager.create_temp_file(suffix=".wav") - with open(bg_audio_path, "wb") as f: - f.write(wav_io.getvalue()) - bg_obj = dict( - audio=bg_audio_path, - mask=bg_mask_path, - ) - processed_talk_objects.append(bg_obj) - logging.info(f"Generated background mask and audio: {bg_mask_path}, {bg_audio_path}") + processed = self._process_multi_speaker_talk_objects(src_talk_objects, max_duration) else: - processed_talk_objects = src_talk_objects + processed = src_talk_objects - if processed_talk_objects: - if len(processed_talk_objects) == 1 and not processed_talk_objects[0].get("mask", "").strip(): - config.audio_path = processed_talk_objects[0]["audio"] - logging.info(f"Convert Processed 1 talk object to audio path: {config.audio_path}") - else: - temp_dir = self.temp_manager.create_temp_dir() - with open(os.path.join(temp_dir, "config.json"), "w") as f: - json.dump({"talk_objects": processed_talk_objects}, f) - config.audio_path = temp_dir - logging.info(f"Processed {len(processed_talk_objects)} talk objects") + self._save_talk_objects_config(config, processed) logging.info("lightx2v prepared config: " + json.dumps(config, indent=2, ensure_ascii=False)) - return (config,) @@ -1821,6 +1771,101 @@ class LightX2VModularInferenceV2: pass +class ExtractLastNFrames: + """Extract the last n frames from I2V output video and convert to prev_frames format""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "video": ("IMAGE", {"tooltip": "Video frames from I2V output"}), + "num_frames": ("INT", {"default": 5, "min": 1, "max": 100, "tooltip": "Number of last frames to extract"}), + }, + } + + RETURN_TYPES = ("PREV_FRAMES", "IMAGE") + RETURN_NAMES = ("prev_frames", "preview_images") + FUNCTION = "extract_frames" + CATEGORY = "LightX2V/Utils" + + def extract_frames(self, video, num_frames): + """ + Extract the last n frames from video and convert to prev_frames format + + Args: + video: Video tensor from I2V output, can be one of the following formats: + - (batch, height, width, channels) - ComfyUI standard IMAGE format (single frame) + - (frames, height, width, channels) - Video frame sequence format + - (batch, channels, frames, height, width) - Video tensor format + num_frames: Number of frames to extract + + Returns: + prev_frames: Tensor with shape (1, 3, num_frames, height, width) for prev_frames input + preview_images: IMAGE tensor with shape (frames, height, width, 3) for visual preview + """ + if video is None or video.numel() == 0: + raise ValueError("Input video cannot be empty") + + dims = len(video.shape) + + if dims == 4: + if video.shape[-1] in [3, 4]: + frames = video.shape[0] + channels = video.shape[-1] + + video = video.permute(0, 3, 1, 2) # (frames, channels, H, W) + video = video.permute(1, 0, 2, 3).unsqueeze(0) # (1, channels, frames, H, W) + + num_frames = min(num_frames, frames) + prev_frames = video[:, :, -num_frames:, :, :] + + if channels != 3: + if channels == 1: + prev_frames = prev_frames.repeat(1, 3, 1, 1, 1) + elif channels == 4: + prev_frames = prev_frames[:, :3, :, :, :] + else: + prev_frames = prev_frames[:, :3, :, :, :] + else: + raise ValueError(f"Unsupported 4D tensor format: {video.shape}, expected last dimension to be 3 or 4 (RGB/RGBA)") + elif dims == 5: + batch, channels, frames, _height, _width = video.shape + num_frames = min(num_frames, frames) + prev_frames = video[:, :, -num_frames:, :, :] # (batch, channels, num_frames, H, W) + + if batch > 1: + prev_frames = prev_frames[0:1, :, :, :, :] + + if channels != 3: + if channels == 1: + prev_frames = prev_frames.repeat(1, 3, 1, 1, 1) + else: + prev_frames = prev_frames[:, :3, :, :, :] + else: + raise ValueError(f"Unsupported tensor dimension: {dims}, shape: {video.shape}") + + if prev_frames.dtype != torch.float32: + prev_frames = prev_frames.float() + + if prev_frames.max() > 1.0: + prev_frames = prev_frames / 255.0 + + # Convert to [-1, 1] range (VAE decoded format) + prev_frames_for_save = prev_frames * 2.0 - 1.0 # [0, 1] -> [-1, 1] + prev_frames_for_save = torch.clamp(prev_frames_for_save, -1.0, 1.0) + + # Convert to IMAGE format for visualization: (1, 3, frames, H, W) -> (frames, H, W, 3) + preview_images = prev_frames.squeeze(0) # (3, frames, H, W) + preview_images = preview_images.permute(1, 2, 3, 0) # (frames, H, W, 3) + + logging.info( + f"Extracted last {num_frames} frames, prev_frames shape: {prev_frames_for_save.shape}, preview_images shape: {preview_images.shape}" + ) + logging.info(f"prev_frames value range: [{prev_frames_for_save.min():.3f}, {prev_frames_for_save.max():.3f}]") + + return (prev_frames_for_save, preview_images) + + NODE_CLASS_MAPPINGS = { "LightX2VInferenceConfig": LightX2VInferenceConfig, "LightX2VTeaCache": LightX2VTeaCache, @@ -1836,6 +1881,7 @@ NODE_CLASS_MAPPINGS = { "LightX2VTalkObjectsCombiner": TalkObjectsCombiner, "LightX2VTalkObjectsFromJSON": TalkObjectsFromJSON, "LightX2VTalkObjectsFromFiles": TalkObjectsFromFiles, + "LightX2VExtractLastNFrames": ExtractLastNFrames, } NODE_DISPLAY_NAME_MAPPINGS = { @@ -1853,4 +1899,5 @@ NODE_DISPLAY_NAME_MAPPINGS = { "LightX2VTalkObjectsCombiner": "LightX2V Talk Objects Combiner", "LightX2VTalkObjectsFromFiles": "LightX2V Talk Objects From Files", "LightX2VTalkObjectsFromJSON": "LightX2V Talk Objects From JSON (API)", + "LightX2VExtractLastNFrames": "LightX2V Extract Last N Frames", } From 56a89b18c67bea0ab3b5dd35ee3cb507ab3be5c9 Mon Sep 17 00:00:00 2001 From: gaclove Date: Thu, 11 Dec 2025 11:12:46 +0000 Subject: [PATCH 7/8] feat: add f2v --- bridge.py | 1 + config_builder.py | 4 ++++ data_models.py | 1 + nodes.py | 8 ++++++++ 4 files changed, 14 insertions(+) diff --git a/bridge.py b/bridge.py index 1f589e0..dacc1ee 100644 --- a/bridge.py +++ b/bridge.py @@ -317,6 +317,7 @@ class ModularConfigManager: "use_31_block": "use_31_block", "prev_frame_length": "prev_frame_length", "fixed_area": "fixed_area", + "f2v_process": "f2v_process", } self._update_from_config(updates, config, basic_mappings) diff --git a/config_builder.py b/config_builder.py index 2352102..5b2781b 100644 --- a/config_builder.py +++ b/config_builder.py @@ -166,6 +166,9 @@ class InferenceConfigBuilder: if "prev_frame_length" in optional_params: config.prev_frame_length = optional_params["prev_frame_length"] + if "f2v_process" in optional_params: + config.f2v_process = optional_params["f2v_process"] + class TalkObjectConfigBuilder: """Builder for talk object configurations.""" @@ -338,6 +341,7 @@ class ConfigBuilder: "adapter_quant_scheme": getattr(config, "adapter_quant_scheme", None), "adapter_quantized": getattr(config, "adapter_quantized", False), "feature_caching": getattr(config, "feature_caching", None), + "f2v_process": getattr(config, "f2v_process", False), } config_str = json.dumps(relevant_configs, sort_keys=True) diff --git a/data_models.py b/data_models.py index 2022f8b..128a800 100644 --- a/data_models.py +++ b/data_models.py @@ -65,6 +65,7 @@ class InferenceConfig: negative_prompt: str = "" image_path: Optional[str] = None audio_path: Optional[str] = None + f2v_process: bool = False def to_dict(self) -> Dict[str, Any]: """Convert to dictionary, excluding None values.""" diff --git a/nodes.py b/nodes.py index d233015..f84a766 100644 --- a/nodes.py +++ b/nodes.py @@ -199,6 +199,13 @@ class LightX2VInferenceConfig: "tooltip": "Use lightweight VAE to accelerate decoding", }, ), + "f2v_process": ( + "BOOLEAN", + { + "default": False, + "tooltip": "Use f2v process", + }, + ), }, } @@ -227,6 +234,7 @@ class LightX2VInferenceConfig: segment_length=81, prev_frame_length=5, use_tiny_vae=False, + f2v_process=False, ): """Create basic inference configuration.""" builder = InferenceConfigBuilder() From ac23ad2e8b368e3cb8b0bf614ae18e30d092bd7c Mon Sep 17 00:00:00 2001 From: gaclove Date: Thu, 11 Dec 2025 14:15:53 +0000 Subject: [PATCH 8/8] feat: integrate f2v_process into LightX2VInferenceConfig and update config handling in InferenceConfigBuilder --- config_builder.py | 6 ++---- lightx2v | 2 +- nodes.py | 1 + 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/config_builder.py b/config_builder.py index 5b2781b..73e83a6 100644 --- a/config_builder.py +++ b/config_builder.py @@ -153,6 +153,7 @@ class InferenceConfigBuilder: "segment_length", "prev_frame_length", "use_tiny_vae", + "f2v_process", ]: if param in optional_params: setattr(config, param, optional_params[param]) @@ -166,9 +167,6 @@ class InferenceConfigBuilder: if "prev_frame_length" in optional_params: config.prev_frame_length = optional_params["prev_frame_length"] - if "f2v_process" in optional_params: - config.f2v_process = optional_params["f2v_process"] - class TalkObjectConfigBuilder: """Builder for talk object configurations.""" @@ -341,7 +339,7 @@ class ConfigBuilder: "adapter_quant_scheme": getattr(config, "adapter_quant_scheme", None), "adapter_quantized": getattr(config, "adapter_quantized", False), "feature_caching": getattr(config, "feature_caching", None), - "f2v_process": getattr(config, "f2v_process", False), + # "f2v_process": getattr(config, "f2v_process", False), } config_str = json.dumps(relevant_configs, sort_keys=True) diff --git a/lightx2v b/lightx2v index 0807e60..9992296 160000 --- a/lightx2v +++ b/lightx2v @@ -1 +1 @@ -Subproject commit 0807e60830d9d572456a999cf54ac3546765a043 +Subproject commit 9992296c12328829fc85397759361beeefdf7a25 diff --git a/nodes.py b/nodes.py index f84a766..8d6c038 100644 --- a/nodes.py +++ b/nodes.py @@ -258,6 +258,7 @@ class LightX2VInferenceConfig: segment_length=segment_length, prev_frame_length=prev_frame_length, use_tiny_vae=use_tiny_vae, + f2v_process=f2v_process, ) return (config.to_dict(),)