ckpt

examples/inference/basic/basic_hy15.py

@@ -2,18 +2,18 @@ from fastvideo import VideoGenerator
 import json
 # from fastvideo.configs.sample import SamplingParam
 
-OUTPUT_PATH = "video_samples_hy15"
+OUTPUT_PATH = "video_samples_hy15_step_distilled"
 def main():
     # FastVideo will automatically use the optimal default arguments for the
     # model.
     # If a local path is provided, FastVideo will make a best effort
     # attempt to identify the optimal arguments.
     generator = VideoGenerator.from_pretrained(
-        "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v",
+        "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v_step_distilled",
         # FastVideo will automatically handle distributed setup
         num_gpus=1,
         use_fsdp_inference=False, # set to True if GPU is out of memory
-        dit_cpu_offload=True,
+        dit_cpu_offload=False,
         vae_cpu_offload=True,
         text_encoder_cpu_offload=True,
         pin_cpu_memory=True, # set to false if low CPU RAM or hit obscure "CUDA error: Invalid argument"
@@ -26,7 +26,7 @@ def main():
         "natural light filtering through the petals. Mid-shot, warm and cheerful tones."
     )
 
-    video = generator.generate_video(prompt, output_path=OUTPUT_PATH, save_video=True, negative_prompt="", num_frames=81, fps=16)
+    video = generator.generate_video(prompt, output_path=OUTPUT_PATH, save_video=True, negative_prompt="")
 
     prompt2 = (
         "A majestic lion strides across the golden savanna, its powerful frame "
@@ -35,7 +35,7 @@ def main():
         "embodying the raw energy of the wild. Low angle, steady tracking shot, "
         "cinematic.")
 
-    video2 = generator.generate_video(prompt2, output_path=OUTPUT_PATH, save_video=True, negative_prompt="", num_frames=81, fps=16)
+    video2 = generator.generate_video(prompt2, output_path=OUTPUT_PATH, save_video=True, negative_prompt="")
 
 
 if __name__ == "__main__":

examples/inference/basic/basic_hy15_sr.py

@@ -9,7 +9,7 @@ def main():
     # If a local path is provided, FastVideo will make a best effort
     # attempt to identify the optimal arguments.
     generator = VideoGenerator.from_pretrained(
-        "weizhou03/HunyuanVideo-1.5-Diffusers-1080p",
+        "/mnt/weka/home/hao.zhang/wei/HunyuanVideo-1.5-Diffusers-1080p",
         # FastVideo will automatically handle distributed setup
         num_gpus=1,
         use_fsdp_inference=False, # set to True if GPU is out of memory

fastvideo/configs/pipelines/hunyuan15.py

@@ -131,6 +131,9 @@ class Hunyuan15T2V480PConfig(PipelineConfig):
         self.vae_config.load_encoder = False
         self.vae_config.load_decoder = True
 
+@dataclass
+class Hunyuan15I2V480PStepDistilledConfig(Hunyuan15T2V480PConfig):
+    flow_shift: int = 7
 
 @dataclass
 class Hunyuan15T2V720PConfig(Hunyuan15T2V480PConfig):
@@ -153,7 +156,7 @@ class Hunyuan15SR1080PConfig(Hunyuan15T2V720PConfig):
     """Base configuration for HunYuan pipeline architecture."""
 
     # HunyuanConfig-specific parameters with defaults
-    flow_shift: int = 9
+    flow_shift: int = 7
     flow_shift_sr: int = 2
     upsampler_config: UpsamplerConfig = field(
         default_factory=SRTo1080pUpsamplerConfig)

fastvideo/configs/sample/hunyuan15.py

@@ -27,6 +27,13 @@ class Hunyuan15_480P_SamplingParam(SamplingParam):
         self.sigmas = list(
             np.linspace(1.0, 0.0, self.num_inference_steps + 1)[:-1])
 
+@dataclass
+class Hunyuan15_480P_StepDistilled_I2V_SamplingParam(Hunyuan15_480P_SamplingParam):
+    num_inference_steps: int = 12
+
+    height: int = 720
+    width: int = 1280
+    guidance_scale: float = 1.0
 
 @dataclass
 class Hunyuan15_720P_SamplingParam(Hunyuan15_480P_SamplingParam):
@@ -44,7 +51,7 @@ class Hunyuan15_SR_1080P_SamplingParam(Hunyuan15_720P_SamplingParam):
     height_sr: int = 1072
     width_sr: int = 1920
 
-    num_inference_steps: int = 50
+    num_inference_steps: int = 12
     num_inference_steps_sr: int = 8
 
     guidance_scale: float = 1.0

fastvideo/models/loader/component_loader.py

@@ -559,7 +559,8 @@ class VAELoader(ComponentLoader):
     def load(self, model_path: str, fastvideo_args: FastVideoArgs):
         """Load the VAE based on the model path, and inference args."""
         config = get_diffusers_config(model=model_path)
-        class_name = config.get("_class_name")
+        class_name = config.pop("_class_name")
+        config.pop("_name_or_path", None)
         assert class_name is not None, (
             "Model config does not contain a _class_name attribute. Only diffusers format is supported."
         )
@@ -732,6 +733,7 @@ class TransformerLoader(ComponentLoader):
         config = get_diffusers_config(model=model_path)
         hf_config = deepcopy(config)
         cls_name = config.pop("_class_name")
+        config.pop("_name_or_path", None)
         if cls_name is None:
             raise ValueError(
                 "Model config does not contain a _class_name attribute. "
@@ -746,7 +748,7 @@ class TransformerLoader(ComponentLoader):
         fastvideo_args.model_paths["transformer"] = model_path
 
         # Config from Diffusers supersedes fastvideo's model config
-        dit_config = fastvideo_args.pipeline_config.dit_config
+        dit_config = deepcopy(fastvideo_args.pipeline_config.dit_config)
         dit_config.update_model_arch(config)
 
         model_cls, _ = ModelRegistry.resolve_model_cls(cls_name)

fastvideo/pipelines/composed_pipeline_base.py

@@ -293,6 +293,7 @@ class ComposedPipelineBase(ABC):
         # remove keys that are not pipeline modules
         model_index.pop("_class_name")
         model_index.pop("_diffusers_version")
+        model_index.pop("_name_or_path", None)
         model_index.pop("workload_type", None)
         if "boundary_ratio" in model_index and model_index[
                 "boundary_ratio"] is not None:

fastvideo/pipelines/stages/denoising.py

@@ -102,6 +102,7 @@ class DenoisingStage(PipelineStage):
         """
         pipeline = self.pipeline() if self.pipeline else None
         if not fastvideo_args.model_loaded["transformer"]:
+            raise RuntimeError("Transformer is not loaded")
             loader = TransformerLoader()
             self.transformer = loader.load(
                 fastvideo_args.model_paths["transformer"], fastvideo_args)
@@ -315,6 +316,23 @@ class DenoisingStage(PipelineStage):
                 else:
                     t_expand = t.repeat(latent_model_input.shape[0])
 
+                use_meanflow = getattr(self.transformer.config, "use_meanflow", False)
+                if use_meanflow:
+                    if i == len(timesteps) - 1:
+                        timesteps_r = torch.tensor([0.0], device=get_local_torch_device())
+                    else:
+                        timesteps_r = timesteps[i + 1]
+                    timesteps_r = timesteps_r.repeat(latent_model_input.shape[0])
+                else:
+                    timesteps_r = None
+
+                timesteps_r_kwarg = self.prepare_extra_func_kwargs(
+                    self.transformer.forward,
+                    {
+                        "timestep_r": timesteps_r,
+                    },
+                )
+
                 latent_model_input = self.scheduler.scale_model_input(
                     latent_model_input, t)
 
@@ -406,6 +424,7 @@ class DenoisingStage(PipelineStage):
                             **image_kwargs,
                             **pos_cond_kwargs,
                             **action_kwargs,
+                            **timesteps_r_kwarg,
                         )
 
                     if batch.do_classifier_free_guidance:
@@ -423,6 +442,7 @@ class DenoisingStage(PipelineStage):
                                 **image_kwargs,
                                 **neg_cond_kwargs,
                                 **action_kwargs,
+                                **timesteps_r_kwarg,
                             )
 
                         noise_pred_text = noise_pred