Support leapfusion img2vid

https://github.com/AeroScripts/leapfusion-hunyuan-image2video

hyvideo/diffusion/pipelines/pipeline_hunyuan_video.py

@@ -237,7 +237,8 @@ class HunyuanVideoPipeline(DiffusionPipeline):
         denoise_strength=1.0,
         freenoise=False, 
         context_size=None, 
-        context_overlap=None
+        context_overlap=None,
+        leapfusion_img2vid=False
     ):
         shape = (
             batch_size,
@@ -286,8 +287,12 @@ class HunyuanVideoPipeline(DiffusionPipeline):
                 # apply shuffled indexes
                 #print("place_idx:", place_idx, "delta:", delta, "list_idx:", list_idx)
                 noise[:, :, place_idx:place_idx + delta, :, :] = noise[:, :, list_idx, :, :]
+
         if latents is None:
             latents = noise
+        elif leapfusion_img2vid:
+            noise[:, :, [0,], :, :] = latents[:, :, [0,], :, :]
+            latents = noise.to(device)
         elif denoise_strength < 1.0:
             latents = latents.to(device)
             timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, denoise_strength, device)
@@ -421,6 +426,7 @@ class HunyuanVideoPipeline(DiffusionPipeline):
         stg_end_percent: Optional[float] = 1.0,
         context_options: Optional[Dict[str, Any]] = None,
         feta_args: Optional[Dict] = None,
+        leapfusion_img2vid: Optional[bool] = False,
         **kwargs,
     ):
         r"""
@@ -604,7 +610,9 @@ class HunyuanVideoPipeline(DiffusionPipeline):
             freqs_cos = freqs_cos.to(device)
             freqs_sin = freqs_sin.to(device)
         
-
+        if leapfusion_img2vid:
+            logger.info("Single input latent frame detected, LeapFusion img2vid enabled")
+            original_latents = latents
         # 5. Prepare latent variables
         num_channels_latents = self.transformer.config.in_channels
         latents, timesteps = self.prepare_latents(
@@ -621,7 +629,8 @@ class HunyuanVideoPipeline(DiffusionPipeline):
             denoise_strength=denoise_strength,
             freenoise=freenoise,
             context_size=context_frames,
-            context_overlap=context_overlap
+            context_overlap=context_overlap,
+            leapfusion_img2vid=leapfusion_img2vid
         )
 
         # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
@@ -693,6 +702,10 @@ class HunyuanVideoPipeline(DiffusionPipeline):
                 latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)
 
                 t_expand = t.repeat(latent_model_input.shape[0])
+
+                if leapfusion_img2vid:
+                    latent_model_input[:, :, [0,], :, :] = original_latents[:, :, [0,], :, :].to(latent_model_input)
+
                 if embedded_guidance_scale is not None and not cfg_enabled:
                     guidance_expand = (
                         torch.tensor(
@@ -808,5 +821,6 @@ class HunyuanVideoPipeline(DiffusionPipeline):
 
         # Offload all models
         #self.maybe_free_model_hooks()
-
+        if leapfusion_img2vid:
+                latents[:, :, [0,], :, :] = original_latents[:, :, [0,], :, :].to(latent_model_input)
         return latents

nodes.py

@@ -1156,6 +1156,9 @@ class HyVideoSampler:
             cfg = 1.0
             cfg_start_percent = 0.0
             cfg_end_percent = 1.0
+        
+        if embedded_guidance_scale == 0.0:
+            embedded_guidance_scale = None
 
         generator = torch.Generator(device=torch.device("cpu")).manual_seed(seed)
 
@@ -1259,6 +1262,7 @@ class HyVideoSampler:
             stg_end_percent=stg_args["stg_end_percent"] if stg_args is not None else 1.0,
             context_options=context_options,
             feta_args=feta_args,
+            leapfusion_img2vid = True if samples["samples"].shape[2] == 1 else False,
         )
 
         print_memory(device)