From d826ee613d04d68e9e5e90daaca9668fcdf76304 Mon Sep 17 00:00:00 2001 From: JerryZhou54 Date: Thu, 22 Jan 2026 06:25:54 +0000 Subject: [PATCH] ckpt --- examples/inference/basic/basic_hy15.py | 10 +++++----- examples/inference/basic/basic_hy15_sr.py | 2 +- fastvideo/configs/pipelines/hunyuan15.py | 5 ++++- fastvideo/configs/sample/hunyuan15.py | 9 ++++++++- fastvideo/models/loader/component_loader.py | 6 ++++-- fastvideo/pipelines/composed_pipeline_base.py | 1 + fastvideo/pipelines/stages/denoising.py | 20 +++++++++++++++++++ 7 files changed, 43 insertions(+), 10 deletions(-) diff --git a/examples/inference/basic/basic_hy15.py b/examples/inference/basic/basic_hy15.py index 39d8772d3..eac65afe9 100644 --- a/examples/inference/basic/basic_hy15.py +++ b/examples/inference/basic/basic_hy15.py @@ -2,18 +2,18 @@ from fastvideo import VideoGenerator import json # from fastvideo.configs.sample import SamplingParam -OUTPUT_PATH = "video_samples_hy15" +OUTPUT_PATH = "video_samples_hy15_step_distilled" def main(): # FastVideo will automatically use the optimal default arguments for the # model. # If a local path is provided, FastVideo will make a best effort # attempt to identify the optimal arguments. generator = VideoGenerator.from_pretrained( - "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", + "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v_step_distilled", # FastVideo will automatically handle distributed setup num_gpus=1, use_fsdp_inference=False, # set to True if GPU is out of memory - dit_cpu_offload=True, + dit_cpu_offload=False, vae_cpu_offload=True, text_encoder_cpu_offload=True, pin_cpu_memory=True, # set to false if low CPU RAM or hit obscure "CUDA error: Invalid argument" @@ -26,7 +26,7 @@ def main(): "natural light filtering through the petals. Mid-shot, warm and cheerful tones." ) - video = generator.generate_video(prompt, output_path=OUTPUT_PATH, save_video=True, negative_prompt="", num_frames=81, fps=16) + video = generator.generate_video(prompt, output_path=OUTPUT_PATH, save_video=True, negative_prompt="") prompt2 = ( "A majestic lion strides across the golden savanna, its powerful frame " @@ -35,7 +35,7 @@ def main(): "embodying the raw energy of the wild. Low angle, steady tracking shot, " "cinematic.") - video2 = generator.generate_video(prompt2, output_path=OUTPUT_PATH, save_video=True, negative_prompt="", num_frames=81, fps=16) + video2 = generator.generate_video(prompt2, output_path=OUTPUT_PATH, save_video=True, negative_prompt="") if __name__ == "__main__": diff --git a/examples/inference/basic/basic_hy15_sr.py b/examples/inference/basic/basic_hy15_sr.py index 49cd81124..a70a9fb46 100644 --- a/examples/inference/basic/basic_hy15_sr.py +++ b/examples/inference/basic/basic_hy15_sr.py @@ -9,7 +9,7 @@ def main(): # If a local path is provided, FastVideo will make a best effort # attempt to identify the optimal arguments. generator = VideoGenerator.from_pretrained( - "weizhou03/HunyuanVideo-1.5-Diffusers-1080p", + "/mnt/weka/home/hao.zhang/wei/HunyuanVideo-1.5-Diffusers-1080p", # FastVideo will automatically handle distributed setup num_gpus=1, use_fsdp_inference=False, # set to True if GPU is out of memory diff --git a/fastvideo/configs/pipelines/hunyuan15.py b/fastvideo/configs/pipelines/hunyuan15.py index 7074d3219..5b80912f5 100644 --- a/fastvideo/configs/pipelines/hunyuan15.py +++ b/fastvideo/configs/pipelines/hunyuan15.py @@ -131,6 +131,9 @@ class Hunyuan15T2V480PConfig(PipelineConfig): self.vae_config.load_encoder = False self.vae_config.load_decoder = True +@dataclass +class Hunyuan15I2V480PStepDistilledConfig(Hunyuan15T2V480PConfig): + flow_shift: int = 7 @dataclass class Hunyuan15T2V720PConfig(Hunyuan15T2V480PConfig): @@ -153,7 +156,7 @@ class Hunyuan15SR1080PConfig(Hunyuan15T2V720PConfig): """Base configuration for HunYuan pipeline architecture.""" # HunyuanConfig-specific parameters with defaults - flow_shift: int = 9 + flow_shift: int = 7 flow_shift_sr: int = 2 upsampler_config: UpsamplerConfig = field( default_factory=SRTo1080pUpsamplerConfig) diff --git a/fastvideo/configs/sample/hunyuan15.py b/fastvideo/configs/sample/hunyuan15.py index 6fefb17a4..8e0fbcf72 100644 --- a/fastvideo/configs/sample/hunyuan15.py +++ b/fastvideo/configs/sample/hunyuan15.py @@ -27,6 +27,13 @@ class Hunyuan15_480P_SamplingParam(SamplingParam): self.sigmas = list( np.linspace(1.0, 0.0, self.num_inference_steps + 1)[:-1]) +@dataclass +class Hunyuan15_480P_StepDistilled_I2V_SamplingParam(Hunyuan15_480P_SamplingParam): + num_inference_steps: int = 12 + + height: int = 720 + width: int = 1280 + guidance_scale: float = 1.0 @dataclass class Hunyuan15_720P_SamplingParam(Hunyuan15_480P_SamplingParam): @@ -44,7 +51,7 @@ class Hunyuan15_SR_1080P_SamplingParam(Hunyuan15_720P_SamplingParam): height_sr: int = 1072 width_sr: int = 1920 - num_inference_steps: int = 50 + num_inference_steps: int = 12 num_inference_steps_sr: int = 8 guidance_scale: float = 1.0 diff --git a/fastvideo/models/loader/component_loader.py b/fastvideo/models/loader/component_loader.py index 011ae1973..697113a3c 100644 --- a/fastvideo/models/loader/component_loader.py +++ b/fastvideo/models/loader/component_loader.py @@ -559,7 +559,8 @@ class VAELoader(ComponentLoader): def load(self, model_path: str, fastvideo_args: FastVideoArgs): """Load the VAE based on the model path, and inference args.""" config = get_diffusers_config(model=model_path) - class_name = config.get("_class_name") + class_name = config.pop("_class_name") + config.pop("_name_or_path", None) assert class_name is not None, ( "Model config does not contain a _class_name attribute. Only diffusers format is supported." ) @@ -732,6 +733,7 @@ class TransformerLoader(ComponentLoader): config = get_diffusers_config(model=model_path) hf_config = deepcopy(config) cls_name = config.pop("_class_name") + config.pop("_name_or_path", None) if cls_name is None: raise ValueError( "Model config does not contain a _class_name attribute. " @@ -746,7 +748,7 @@ class TransformerLoader(ComponentLoader): fastvideo_args.model_paths["transformer"] = model_path # Config from Diffusers supersedes fastvideo's model config - dit_config = fastvideo_args.pipeline_config.dit_config + dit_config = deepcopy(fastvideo_args.pipeline_config.dit_config) dit_config.update_model_arch(config) model_cls, _ = ModelRegistry.resolve_model_cls(cls_name) diff --git a/fastvideo/pipelines/composed_pipeline_base.py b/fastvideo/pipelines/composed_pipeline_base.py index f8d99ce10..d9a7f6419 100644 --- a/fastvideo/pipelines/composed_pipeline_base.py +++ b/fastvideo/pipelines/composed_pipeline_base.py @@ -293,6 +293,7 @@ class ComposedPipelineBase(ABC): # remove keys that are not pipeline modules model_index.pop("_class_name") model_index.pop("_diffusers_version") + model_index.pop("_name_or_path", None) model_index.pop("workload_type", None) if "boundary_ratio" in model_index and model_index[ "boundary_ratio"] is not None: diff --git a/fastvideo/pipelines/stages/denoising.py b/fastvideo/pipelines/stages/denoising.py index c09254365..9e2d9f4ff 100644 --- a/fastvideo/pipelines/stages/denoising.py +++ b/fastvideo/pipelines/stages/denoising.py @@ -102,6 +102,7 @@ class DenoisingStage(PipelineStage): """ pipeline = self.pipeline() if self.pipeline else None if not fastvideo_args.model_loaded["transformer"]: + raise RuntimeError("Transformer is not loaded") loader = TransformerLoader() self.transformer = loader.load( fastvideo_args.model_paths["transformer"], fastvideo_args) @@ -315,6 +316,23 @@ class DenoisingStage(PipelineStage): else: t_expand = t.repeat(latent_model_input.shape[0]) + use_meanflow = getattr(self.transformer.config, "use_meanflow", False) + if use_meanflow: + if i == len(timesteps) - 1: + timesteps_r = torch.tensor([0.0], device=get_local_torch_device()) + else: + timesteps_r = timesteps[i + 1] + timesteps_r = timesteps_r.repeat(latent_model_input.shape[0]) + else: + timesteps_r = None + + timesteps_r_kwarg = self.prepare_extra_func_kwargs( + self.transformer.forward, + { + "timestep_r": timesteps_r, + }, + ) + latent_model_input = self.scheduler.scale_model_input( latent_model_input, t) @@ -406,6 +424,7 @@ class DenoisingStage(PipelineStage): **image_kwargs, **pos_cond_kwargs, **action_kwargs, + **timesteps_r_kwarg, ) if batch.do_classifier_free_guidance: @@ -423,6 +442,7 @@ class DenoisingStage(PipelineStage): **image_kwargs, **neg_cond_kwargs, **action_kwargs, + **timesteps_r_kwarg, ) noise_pred_text = noise_pred