testing for skyreel i2v

might not work proper yet

https://huggingface.co/Kijai/SkyReels-V1-Hunyuan_comfy/blob/main/skyreels_hunyuan_i2v_bf16.safetensors

hyvideo/diffusion/pipelines/pipeline_hunyuan_video.py

@@ -427,6 +427,7 @@ class HunyuanVideoPipeline(DiffusionPipeline):
         context_options: Optional[Dict[str, Any]] = None,
         feta_args: Optional[Dict] = None,
         leapfusion_img2vid: Optional[bool] = False,
+        image_cond_latents: Optional[torch.Tensor] = None,
         **kwargs,
     ):
         r"""
@@ -576,6 +577,19 @@ class HunyuanVideoPipeline(DiffusionPipeline):
             enable_enhance(feta_args["single_blocks"], feta_args["double_blocks"])
         else:
             disable_enhance()
+
+        if image_cond_latents is not None:
+            padding_shape = (
+                batch_size,
+                16,
+                latent_video_length - 1,
+                int(height) // 8,
+                int(width) // 8,
+            )
+            latent_padding = torch.zeros(padding_shape, device=device, dtype=self.base_dtype)
+            image_latents = torch.cat([image_cond_latents, latent_padding], dim=2)
+            print("image_cond_latents shape:", image_cond_latents.shape)
+            print("image_latents shape:", image_latents.shape)
         
 
         #  context windows
@@ -614,7 +628,8 @@ class HunyuanVideoPipeline(DiffusionPipeline):
             logger.info("Single input latent frame detected, LeapFusion img2vid enabled")
             original_latents = latents
         # 5. Prepare latent variables
-        num_channels_latents = self.transformer.config.in_channels
+        #num_channels_latents = self.transformer.config.in_channels
+        num_channels_latents = 16
         latents, timesteps = self.prepare_latents(
             batch_size * num_videos_per_prompt,
             num_channels_latents,
@@ -706,6 +721,14 @@ class HunyuanVideoPipeline(DiffusionPipeline):
                 if leapfusion_img2vid:
                     latent_model_input[:, :, [0,], :, :] = original_latents[:, :, [0,], :, :].to(latent_model_input)
 
+                if image_cond_latents is not None:
+                    latent_image_input = (
+                        torch.cat([image_latents] * 2) if self.do_classifier_free_guidance else image_latents
+                    )
+                    print("latent_image_input", latent_image_input.shape)
+                    print("latent_model_input", latent_model_input.shape)
+                    latent_model_input = torch.cat([latent_model_input, latent_image_input], dim=1)
+
                 if embedded_guidance_scale is not None and not cfg_enabled:
                     guidance_expand = (
                         torch.tensor(
@@ -811,7 +834,7 @@ class HunyuanVideoPipeline(DiffusionPipeline):
                         if leapfusion_img2vid:
                             callback_latent = (latent_model_input[:, :, 1:, :, :] - noise_pred[:, :, 1:, :, :] * t / 1000).detach()[0].permute(1,0,2,3)
                         else:
-                            callback_latent = (latent_model_input - noise_pred * t / 1000).detach()[0].permute(1,0,2,3)
+                            callback_latent = (latent_model_input[:, :16, :, :, :] - noise_pred * t / 1000).detach()[0].permute(1,0,2,3)
                         callback(
                             i, 
                             callback_latent,

nodes.py

@@ -326,7 +326,9 @@ class HyVideoModelLoader:
         model_path = folder_paths.get_full_path_or_raise("diffusion_models", model)
         sd = load_torch_file(model_path, device=transformer_load_device, safe_load=True)
 
-        in_channels = out_channels = 16
+        in_channels = sd["img_in.proj.weight"].shape[1]
+
+        out_channels = 16
         factor_kwargs = {"device": transformer_load_device, "dtype": base_dtype}
         HUNYUAN_VIDEO_CONFIG = {
             "mm_double_blocks_depth": 20,
@@ -382,6 +384,7 @@ class HyVideoModelLoader:
                 dtype = base_dtype
             params_to_keep = {"norm", "bias", "time_in", "vector_in", "guidance_in", "txt_in", "img_in"}
             for name, param in transformer.named_parameters():
+                #print("Assigning Parameter name: ", name)
                 dtype_to_use = base_dtype if any(keyword in name for keyword in params_to_keep) else dtype
                 set_module_tensor_to_device(transformer, name, device=transformer_load_device, dtype=dtype_to_use, value=sd[name])
 
@@ -1143,6 +1146,7 @@ class HyVideoSampler:
             },
             "optional": {
                 "samples": ("LATENT", {"tooltip": "init Latents to use for video2video process"} ),
+                "image_cond_latents": ("LATENT", {"tooltip": "init Latents to use for image2video process"} ),
                 "denoise_strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}),
                 "stg_args": ("STGARGS", ),
                 "context_options": ("HYVIDCONTEXT", ),
@@ -1161,7 +1165,7 @@ class HyVideoSampler:
     CATEGORY = "HunyuanVideoWrapper"
 
     def process(self, model, hyvid_embeds, flow_shift, steps, embedded_guidance_scale, seed, width, height, num_frames, 
-                samples=None, denoise_strength=1.0, force_offload=True, stg_args=None, context_options=None, feta_args=None, teacache_args=None, scheduler=None):
+                samples=None, denoise_strength=1.0, force_offload=True, stg_args=None, context_options=None, feta_args=None, teacache_args=None, scheduler=None, image_cond_latents=None):
         model = model.model
 
         device = mm.get_torch_device()
@@ -1298,7 +1302,8 @@ class HyVideoSampler:
             stg_end_percent=stg_args["stg_end_percent"] if stg_args is not None else 1.0,
             context_options=context_options,
             feta_args=feta_args,
-            leapfusion_img2vid = leapfusion_img2vid
+            leapfusion_img2vid = leapfusion_img2vid,
+            image_cond_latents = image_cond_latents["samples"] * VAE_SCALING_FACTOR if image_cond_latents is not None else None,
         )
 
         print_memory(device)