diff --git a/hyvideo/diffusion/pipelines/pipeline_hunyuan_video.py b/hyvideo/diffusion/pipelines/pipeline_hunyuan_video.py index 5e03083..43e7006 100644 --- a/hyvideo/diffusion/pipelines/pipeline_hunyuan_video.py +++ b/hyvideo/diffusion/pipelines/pipeline_hunyuan_video.py @@ -237,7 +237,8 @@ class HunyuanVideoPipeline(DiffusionPipeline): denoise_strength=1.0, freenoise=False, context_size=None, - context_overlap=None + context_overlap=None, + leapfusion_img2vid=False ): shape = ( batch_size, @@ -286,8 +287,12 @@ class HunyuanVideoPipeline(DiffusionPipeline): # apply shuffled indexes #print("place_idx:", place_idx, "delta:", delta, "list_idx:", list_idx) noise[:, :, place_idx:place_idx + delta, :, :] = noise[:, :, list_idx, :, :] + if latents is None: latents = noise + elif leapfusion_img2vid: + noise[:, :, [0,], :, :] = latents[:, :, [0,], :, :] + latents = noise.to(device) elif denoise_strength < 1.0: latents = latents.to(device) timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, denoise_strength, device) @@ -421,6 +426,7 @@ class HunyuanVideoPipeline(DiffusionPipeline): stg_end_percent: Optional[float] = 1.0, context_options: Optional[Dict[str, Any]] = None, feta_args: Optional[Dict] = None, + leapfusion_img2vid: Optional[bool] = False, **kwargs, ): r""" @@ -604,7 +610,9 @@ class HunyuanVideoPipeline(DiffusionPipeline): freqs_cos = freqs_cos.to(device) freqs_sin = freqs_sin.to(device) - + if leapfusion_img2vid: + logger.info("Single input latent frame detected, LeapFusion img2vid enabled") + original_latents = latents # 5. Prepare latent variables num_channels_latents = self.transformer.config.in_channels latents, timesteps = self.prepare_latents( @@ -621,7 +629,8 @@ class HunyuanVideoPipeline(DiffusionPipeline): denoise_strength=denoise_strength, freenoise=freenoise, context_size=context_frames, - context_overlap=context_overlap + context_overlap=context_overlap, + leapfusion_img2vid=leapfusion_img2vid ) # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline @@ -693,6 +702,10 @@ class HunyuanVideoPipeline(DiffusionPipeline): latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) t_expand = t.repeat(latent_model_input.shape[0]) + + if leapfusion_img2vid: + latent_model_input[:, :, [0,], :, :] = original_latents[:, :, [0,], :, :].to(latent_model_input) + if embedded_guidance_scale is not None and not cfg_enabled: guidance_expand = ( torch.tensor( @@ -808,5 +821,6 @@ class HunyuanVideoPipeline(DiffusionPipeline): # Offload all models #self.maybe_free_model_hooks() - + if leapfusion_img2vid: + latents[:, :, [0,], :, :] = original_latents[:, :, [0,], :, :].to(latent_model_input) return latents \ No newline at end of file diff --git a/nodes.py b/nodes.py index f3f0383..e71ae57 100644 --- a/nodes.py +++ b/nodes.py @@ -1156,6 +1156,9 @@ class HyVideoSampler: cfg = 1.0 cfg_start_percent = 0.0 cfg_end_percent = 1.0 + + if embedded_guidance_scale == 0.0: + embedded_guidance_scale = None generator = torch.Generator(device=torch.device("cpu")).manual_seed(seed) @@ -1259,6 +1262,7 @@ class HyVideoSampler: stg_end_percent=stg_args["stg_end_percent"] if stg_args is not None else 1.0, context_options=context_options, feta_args=feta_args, + leapfusion_img2vid = True if samples["samples"].shape[2] == 1 else False, ) print_memory(device)