diff --git a/hyvideo/diffusion/pipelines/pipeline_hunyuan_video.py b/hyvideo/diffusion/pipelines/pipeline_hunyuan_video.py index 357fcd3..214322b 100644 --- a/hyvideo/diffusion/pipelines/pipeline_hunyuan_video.py +++ b/hyvideo/diffusion/pipelines/pipeline_hunyuan_video.py @@ -427,6 +427,7 @@ class HunyuanVideoPipeline(DiffusionPipeline): context_options: Optional[Dict[str, Any]] = None, feta_args: Optional[Dict] = None, leapfusion_img2vid: Optional[bool] = False, + image_cond_latents: Optional[torch.Tensor] = None, **kwargs, ): r""" @@ -576,6 +577,19 @@ class HunyuanVideoPipeline(DiffusionPipeline): enable_enhance(feta_args["single_blocks"], feta_args["double_blocks"]) else: disable_enhance() + + if image_cond_latents is not None: + padding_shape = ( + batch_size, + 16, + latent_video_length - 1, + int(height) // 8, + int(width) // 8, + ) + latent_padding = torch.zeros(padding_shape, device=device, dtype=self.base_dtype) + image_latents = torch.cat([image_cond_latents, latent_padding], dim=2) + print("image_cond_latents shape:", image_cond_latents.shape) + print("image_latents shape:", image_latents.shape) # context windows @@ -614,7 +628,8 @@ class HunyuanVideoPipeline(DiffusionPipeline): logger.info("Single input latent frame detected, LeapFusion img2vid enabled") original_latents = latents # 5. Prepare latent variables - num_channels_latents = self.transformer.config.in_channels + #num_channels_latents = self.transformer.config.in_channels + num_channels_latents = 16 latents, timesteps = self.prepare_latents( batch_size * num_videos_per_prompt, num_channels_latents, @@ -706,6 +721,14 @@ class HunyuanVideoPipeline(DiffusionPipeline): if leapfusion_img2vid: latent_model_input[:, :, [0,], :, :] = original_latents[:, :, [0,], :, :].to(latent_model_input) + if image_cond_latents is not None: + latent_image_input = ( + torch.cat([image_latents] * 2) if self.do_classifier_free_guidance else image_latents + ) + print("latent_image_input", latent_image_input.shape) + print("latent_model_input", latent_model_input.shape) + latent_model_input = torch.cat([latent_model_input, latent_image_input], dim=1) + if embedded_guidance_scale is not None and not cfg_enabled: guidance_expand = ( torch.tensor( @@ -811,7 +834,7 @@ class HunyuanVideoPipeline(DiffusionPipeline): if leapfusion_img2vid: callback_latent = (latent_model_input[:, :, 1:, :, :] - noise_pred[:, :, 1:, :, :] * t / 1000).detach()[0].permute(1,0,2,3) else: - callback_latent = (latent_model_input - noise_pred * t / 1000).detach()[0].permute(1,0,2,3) + callback_latent = (latent_model_input[:, :16, :, :, :] - noise_pred * t / 1000).detach()[0].permute(1,0,2,3) callback( i, callback_latent, diff --git a/nodes.py b/nodes.py index 5564860..8258ae0 100644 --- a/nodes.py +++ b/nodes.py @@ -326,7 +326,9 @@ class HyVideoModelLoader: model_path = folder_paths.get_full_path_or_raise("diffusion_models", model) sd = load_torch_file(model_path, device=transformer_load_device, safe_load=True) - in_channels = out_channels = 16 + in_channels = sd["img_in.proj.weight"].shape[1] + + out_channels = 16 factor_kwargs = {"device": transformer_load_device, "dtype": base_dtype} HUNYUAN_VIDEO_CONFIG = { "mm_double_blocks_depth": 20, @@ -382,6 +384,7 @@ class HyVideoModelLoader: dtype = base_dtype params_to_keep = {"norm", "bias", "time_in", "vector_in", "guidance_in", "txt_in", "img_in"} for name, param in transformer.named_parameters(): + #print("Assigning Parameter name: ", name) dtype_to_use = base_dtype if any(keyword in name for keyword in params_to_keep) else dtype set_module_tensor_to_device(transformer, name, device=transformer_load_device, dtype=dtype_to_use, value=sd[name]) @@ -1143,6 +1146,7 @@ class HyVideoSampler: }, "optional": { "samples": ("LATENT", {"tooltip": "init Latents to use for video2video process"} ), + "image_cond_latents": ("LATENT", {"tooltip": "init Latents to use for image2video process"} ), "denoise_strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}), "stg_args": ("STGARGS", ), "context_options": ("HYVIDCONTEXT", ), @@ -1161,7 +1165,7 @@ class HyVideoSampler: CATEGORY = "HunyuanVideoWrapper" def process(self, model, hyvid_embeds, flow_shift, steps, embedded_guidance_scale, seed, width, height, num_frames, - samples=None, denoise_strength=1.0, force_offload=True, stg_args=None, context_options=None, feta_args=None, teacache_args=None, scheduler=None): + samples=None, denoise_strength=1.0, force_offload=True, stg_args=None, context_options=None, feta_args=None, teacache_args=None, scheduler=None, image_cond_latents=None): model = model.model device = mm.get_torch_device() @@ -1298,7 +1302,8 @@ class HyVideoSampler: stg_end_percent=stg_args["stg_end_percent"] if stg_args is not None else 1.0, context_options=context_options, feta_args=feta_args, - leapfusion_img2vid = leapfusion_img2vid + leapfusion_img2vid = leapfusion_img2vid, + image_cond_latents = image_cond_latents["samples"] * VAE_SCALING_FACTOR if image_cond_latents is not None else None, ) print_memory(device)