From fb3bfecd18101b566760592d4387cb6961ef88e1 Mon Sep 17 00:00:00 2001 From: JerryZhou54 Date: Thu, 18 Dec 2025 04:55:20 +0000 Subject: [PATCH] Pipeline working --- examples/inference/basic/basic_hy15.py | 31 ++++++++++++ fastvideo/configs/models/dits/base.py | 2 + .../configs/models/dits/hunyuanvideo15.py | 2 +- fastvideo/configs/models/encoders/t5.py | 1 + fastvideo/configs/pipelines/__init__.py | 3 +- fastvideo/configs/pipelines/hunyuan15.py | 25 +++++----- fastvideo/configs/pipelines/registry.py | 10 +++- fastvideo/configs/sample/base.py | 1 + fastvideo/configs/sample/hunyuan15.py | 20 ++++++++ fastvideo/configs/sample/registry.py | 16 ++++-- fastvideo/models/dits/hunyuanvideo15.py | 19 ++++--- fastvideo/models/loader/component_loader.py | 5 +- .../basic/hunyuan15/hunyuan15_pipeline.py | 8 ++- fastvideo/pipelines/pipeline_registry.py | 1 + fastvideo/pipelines/stages/__init__.py | 3 +- fastvideo/pipelines/stages/denoising.py | 6 +++ fastvideo/pipelines/stages/image_encoding.py | 23 +++++++++ fastvideo/pipelines/stages/text_encoding.py | 49 ++++++++++++++----- .../tests/transformers/test_hunyuanvideo15.py | 19 +++---- .../transformers/test_ori_hunyuanvideo15.py | 10 ++-- 20 files changed, 198 insertions(+), 56 deletions(-) create mode 100644 examples/inference/basic/basic_hy15.py create mode 100644 fastvideo/configs/sample/hunyuan15.py diff --git a/examples/inference/basic/basic_hy15.py b/examples/inference/basic/basic_hy15.py new file mode 100644 index 000000000..6477c3538 --- /dev/null +++ b/examples/inference/basic/basic_hy15.py @@ -0,0 +1,31 @@ +from fastvideo import VideoGenerator +import json +# from fastvideo.configs.sample import SamplingParam + +OUTPUT_PATH = "video_samples_hy15" +def main(): + # FastVideo will automatically use the optimal default arguments for the + # model. + # If a local path is provided, FastVideo will make a best effort + # attempt to identify the optimal arguments. + generator = VideoGenerator.from_pretrained( + "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", + # FastVideo will automatically handle distributed setup + num_gpus=1, + use_fsdp_inference=True, + dit_cpu_offload=True, + vae_cpu_offload=True, + text_encoder_cpu_offload=True, + pin_cpu_memory=True, # set to false if low CPU RAM or hit obscure "CUDA error: Invalid argument" + # image_encoder_cpu_offload=False, + ) + + with open("data/validation_64.json", "r") as f: + data = json.load(f)["data"] + for item in data[1:]: + prompt = item["caption"] + video = generator.generate_video(prompt, output_path=OUTPUT_PATH, save_video=True, negative_prompt="", num_frames=81, fps=16) + + +if __name__ == "__main__": + main() diff --git a/fastvideo/configs/models/dits/base.py b/fastvideo/configs/models/dits/base.py index 148eb711b..818e0bee3 100644 --- a/fastvideo/configs/models/dits/base.py +++ b/fastvideo/configs/models/dits/base.py @@ -23,6 +23,8 @@ class DiTArchConfig(ArchConfig): hidden_size: int = 0 num_attention_heads: int = 0 num_channels_latents: int = 0 + in_channels: int = 0 + out_channels: int = 0 exclude_lora_layers: list[str] = field(default_factory=list) boundary_ratio: float | None = None diff --git a/fastvideo/configs/models/dits/hunyuanvideo15.py b/fastvideo/configs/models/dits/hunyuanvideo15.py index 9677163cc..666065b6f 100644 --- a/fastvideo/configs/models/dits/hunyuanvideo15.py +++ b/fastvideo/configs/models/dits/hunyuanvideo15.py @@ -150,7 +150,7 @@ class HunyuanVideo15ArchConfig(DiTArchConfig): def __post_init__(self): super().__post_init__() self.hidden_size: int = self.attention_head_dim * self.num_attention_heads - self.num_channels_latents: int = self.in_channels + self.num_channels_latents: int = self.out_channels @dataclass diff --git a/fastvideo/configs/models/encoders/t5.py b/fastvideo/configs/models/encoders/t5.py index 0a06b36eb..3fbe75cb0 100644 --- a/fastvideo/configs/models/encoders/t5.py +++ b/fastvideo/configs/models/encoders/t5.py @@ -70,6 +70,7 @@ class T5ArchConfig(TextEncoderArchConfig): "return_attention_mask": True, "return_tensors": "pt", } + self.hidden_size = self.d_model @dataclass diff --git a/fastvideo/configs/pipelines/__init__.py b/fastvideo/configs/pipelines/__init__.py index 8bbb0a60c..41432d1a5 100644 --- a/fastvideo/configs/pipelines/__init__.py +++ b/fastvideo/configs/pipelines/__init__.py @@ -2,6 +2,7 @@ from fastvideo.configs.pipelines.base import (PipelineConfig, SlidingTileAttnConfig) from fastvideo.configs.pipelines.cosmos import CosmosConfig from fastvideo.configs.pipelines.hunyuan import FastHunyuanConfig, HunyuanConfig +from fastvideo.configs.pipelines.hunyuan15 import Hunyuan15T2V480PConfig from fastvideo.configs.pipelines.registry import ( get_pipeline_config_cls_from_name) from fastvideo.configs.pipelines.stepvideo import StepVideoT2VConfig @@ -10,7 +11,7 @@ from fastvideo.configs.pipelines.wan import (SelfForcingWanT2V480PConfig, WanT2V480PConfig, WanT2V720PConfig) __all__ = [ - "HunyuanConfig", "FastHunyuanConfig", "PipelineConfig", + "HunyuanConfig", "FastHunyuanConfig", "PipelineConfig", "Hunyuan15T2V480PConfig", "SlidingTileAttnConfig", "WanT2V480PConfig", "WanI2V480PConfig", "WanT2V720PConfig", "WanI2V720PConfig", "StepVideoT2VConfig", "SelfForcingWanT2V480PConfig", "CosmosConfig", diff --git a/fastvideo/configs/pipelines/hunyuan15.py b/fastvideo/configs/pipelines/hunyuan15.py index c677a2bda..be97051f4 100644 --- a/fastvideo/configs/pipelines/hunyuan15.py +++ b/fastvideo/configs/pipelines/hunyuan15.py @@ -13,12 +13,14 @@ from fastvideo.configs.models.encoders import (BaseEncoderOutput, from fastvideo.configs.models.vaes import Hunyuan15VAEConfig from fastvideo.configs.pipelines.base import PipelineConfig +PROMPT_TEMPLATE_TOKEN_LENGTH = 108 + PROMPT_TEMPLATE_ENCODE_VIDEO = "You are a helpful assistant. Describe the video by detailing the following aspects: \ -1. The main content and theme of the video. \ -2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects. \ -3. Actions, events, behaviors temporal relationships, physical movement changes of the objects. \ -4. background environment, light, style and atmosphere. \ -5. camera angles, movements, and transitions used in the video." + 1. The main content and theme of the video. \ + 2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects. \ + 3. Actions, events, behaviors temporal relationships, physical movement changes of the objects. \ + 4. background environment, light, style and atmosphere. \ + 5. camera angles, movements, and transitions used in the video." def extract_glyph_texts(prompt: str) -> List[str]: """ @@ -60,7 +62,6 @@ def format_text_input(prompt: str, system_message: str) -> List[Dict[str, Any]]: def qwen_preprocess_text(prompt: str) -> str: - prompt = [prompt] if isinstance(prompt, str) else prompt prompt = format_text_input(prompt, PROMPT_TEMPLATE_ENCODE_VIDEO) return prompt @@ -68,8 +69,8 @@ def qwen_preprocess_text(prompt: str) -> str: def qwen_postprocess_text(outputs: BaseEncoderOutput, mask: torch.tensor) -> tuple[torch.tensor, torch.tensor]: assert outputs.hidden_states is not None output = outputs.hidden_states[-3] - output = output[:, 108:] - mask = mask[:, 108:] + output = output[:, PROMPT_TEMPLATE_TOKEN_LENGTH:] + mask = mask[:, PROMPT_TEMPLATE_TOKEN_LENGTH:] return output, mask @@ -97,7 +98,7 @@ class Hunyuan15T2V480PConfig(PipelineConfig): # Text encoding stage text_encoder_configs: tuple[EncoderConfig, ...] = field( - default_factory=lambda: (T5Config(), Qwen2_5_VLConfig())) + default_factory=lambda: (Qwen2_5_VLConfig(), T5Config())) preprocess_text_funcs: tuple[Callable[[str], str], ...] = field( default_factory=lambda: (qwen_preprocess_text, byt5_preprocess_text)) postprocess_text_funcs: tuple[ @@ -110,9 +111,11 @@ class Hunyuan15T2V480PConfig(PipelineConfig): vae_precision: str = "fp16" text_encoder_precisions: tuple[str, ...] = field( default_factory=lambda: ("bf16", "fp32")) - text_encoder_crop_start: int = 108 + text_encoder_crop_start: int = PROMPT_TEMPLATE_TOKEN_LENGTH text_encoder_max_lengths: tuple[int, ...] = field( - default_factory=lambda: (1000 + 108, 256)) + default_factory=lambda: (1000 + PROMPT_TEMPLATE_TOKEN_LENGTH, 256)) + + vae_tiling: bool = True def __post_init__(self): self.vae_config.load_encoder = False diff --git a/fastvideo/configs/pipelines/registry.py b/fastvideo/configs/pipelines/registry.py index d3de0b872..383d4cba1 100644 --- a/fastvideo/configs/pipelines/registry.py +++ b/fastvideo/configs/pipelines/registry.py @@ -7,6 +7,7 @@ from collections.abc import Callable from fastvideo.configs.pipelines.base import PipelineConfig from fastvideo.configs.pipelines.cosmos import CosmosConfig from fastvideo.configs.pipelines.hunyuan import FastHunyuanConfig, HunyuanConfig +from fastvideo.configs.pipelines.hunyuan15 import Hunyuan15T2V480PConfig from fastvideo.configs.pipelines.stepvideo import StepVideoT2VConfig # isort: off @@ -27,6 +28,7 @@ logger = init_logger(__name__) PIPE_NAME_TO_CONFIG: dict[str, type[PipelineConfig]] = { "FastVideo/FastHunyuan-diffusers": FastHunyuanConfig, "hunyuanvideo-community/HunyuanVideo": HunyuanConfig, + "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v": Hunyuan15T2V480PConfig, "Wan-AI/Wan2.1-T2V-1.3B-Diffusers": WanT2V480PConfig, "weizhou03/Wan2.1-Fun-1.3B-InP-Diffusers": WanI2V480PConfig, "IRMChen/Wan2.1-Fun-1.3B-Control-Diffusers": WANV2VConfig, @@ -56,6 +58,8 @@ PIPE_NAME_TO_CONFIG: dict[str, type[PipelineConfig]] = { PIPELINE_DETECTOR: dict[str, Callable[[str], bool]] = { "hunyuan": lambda id: "hunyuan" in id.lower(), + "hunyuan15": + lambda id: "hunyuan15" in id.lower(), "matrixgame": lambda id: "matrix-game" in id.lower() or "matrixgame" in id.lower(), "wanpipeline": @@ -77,7 +81,10 @@ PIPELINE_DETECTOR: dict[str, Callable[[str], bool]] = { PIPELINE_FALLBACK_CONFIG: dict[str, type[PipelineConfig]] = { "hunyuan": HunyuanConfig, # Base Hunyuan config as fallback for any Hunyuan variant - "matrixgame": MatrixGameI2V480PConfig, + "matrixgame": + MatrixGameI2V480PConfig, + "hunyuan15": + Hunyuan15T2V480PConfig, # Base Hunyuan15 config as fallback for any Hunyuan15 variant "wanpipeline": WanT2V480PConfig, # Base Wan config as fallback for any Wan variant "wanimagetovideo": WanI2V480PConfig, @@ -123,6 +130,7 @@ def get_pipeline_config_cls_from_name( # First try exact match for specific weights if pipeline_name_or_path in PIPE_NAME_TO_CONFIG: pipeline_config_cls = PIPE_NAME_TO_CONFIG[pipeline_name_or_path] + return pipeline_config_cls # Try partial matches (for local paths that might include the weight ID) for registered_id, config_class in PIPE_NAME_TO_CONFIG.items(): diff --git a/fastvideo/configs/sample/base.py b/fastvideo/configs/sample/base.py index 8d62f95a1..fcf292869 100644 --- a/fastvideo/configs/sample/base.py +++ b/fastvideo/configs/sample/base.py @@ -50,6 +50,7 @@ class SamplingParam: guidance_scale: float = 1.0 guidance_rescale: float = 0.0 boundary_ratio: float | None = None + sigmas: list[float] | None = None # TeaCache parameters enable_teacache: bool = False diff --git a/fastvideo/configs/sample/hunyuan15.py b/fastvideo/configs/sample/hunyuan15.py new file mode 100644 index 000000000..4807a53f6 --- /dev/null +++ b/fastvideo/configs/sample/hunyuan15.py @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: Apache-2.0 +import numpy as np +from dataclasses import dataclass, field + +from fastvideo.configs.sample.base import SamplingParam + + +@dataclass +class Hunyuan15_480P_SamplingParam(SamplingParam): + num_inference_steps: int = 50 + + num_frames: int = 121 + height: int = 480 + width: int = 848 + fps: int = 24 + + guidance_scale: float = 6.0 + prompt_attention_mask: list = field(default_factory=list) + negative_attention_mask: list = field(default_factory=list) + sigmas: list[float] | None = field(default_factory=lambda: list(np.linspace(1.0, 0.0, 50 + 1)[:-1])) \ No newline at end of file diff --git a/fastvideo/configs/sample/registry.py b/fastvideo/configs/sample/registry.py index 58969f2f0..96bf0af77 100644 --- a/fastvideo/configs/sample/registry.py +++ b/fastvideo/configs/sample/registry.py @@ -5,6 +5,7 @@ from typing import Any from fastvideo.configs.sample.hunyuan import (FastHunyuanSamplingParam, HunyuanSamplingParam) +from fastvideo.configs.sample.hunyuan15 import Hunyuan15_480P_SamplingParam from fastvideo.configs.sample.stepvideo import StepVideoT2VSamplingParam from fastvideo.configs.sample.cosmos import Cosmos_Predict2_2B_Video2World_SamplingParam @@ -33,9 +34,14 @@ from fastvideo.utils import (maybe_download_model_index, logger = init_logger(__name__) # Registry maps specific model weights to their config classes SAMPLING_PARAM_REGISTRY: dict[str, Any] = { - "FastVideo/FastHunyuan-diffusers": FastHunyuanSamplingParam, - "hunyuanvideo-community/HunyuanVideo": HunyuanSamplingParam, - "FastVideo/stepvideo-t2v-diffusers": StepVideoT2VSamplingParam, + "FastVideo/FastHunyuan-diffusers": + FastHunyuanSamplingParam, + "hunyuanvideo-community/HunyuanVideo": + HunyuanSamplingParam, + "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v": + Hunyuan15_480P_SamplingParam, + "FastVideo/stepvideo-t2v-diffusers": + StepVideoT2VSamplingParam, # Wan2.1 "Wan-AI/Wan2.1-T2V-1.3B-Diffusers": WanT2V_1_3B_SamplingParam, @@ -86,6 +92,8 @@ SAMPLING_PARAM_REGISTRY: dict[str, Any] = { SAMPLING_PARAM_DETECTOR: dict[str, Callable[[str], bool]] = { "hunyuan": lambda id: "hunyuan" in id.lower(), + "hunyuan15": + lambda id: "hunyuan15" in id.lower(), "wanpipeline": lambda id: "wanpipeline" in id.lower(), "wanimagetovideo": @@ -105,6 +113,8 @@ SAMPLING_PARAM_DETECTOR: dict[str, Callable[[str], bool]] = { SAMPLING_FALLBACK_PARAM: dict[str, Any] = { "hunyuan": HunyuanSamplingParam, # Base Hunyuan config as fallback for any Hunyuan variant + "hunyuan15": + Hunyuan15_480P_SamplingParam, # Base Hunyuan15 config as fallback for any Hunyuan15 variant "wanpipeline": WanT2V_1_3B_SamplingParam, # Base Wan config as fallback for any Wan variant "wanimagetovideo": WanI2V_14B_480P_SamplingParam, diff --git a/fastvideo/models/dits/hunyuanvideo15.py b/fastvideo/models/dits/hunyuanvideo15.py index 934816e5c..2ce6b7cf0 100644 --- a/fastvideo/models/dits/hunyuanvideo15.py +++ b/fastvideo/models/dits/hunyuanvideo15.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, List import torch import torch.nn as nn @@ -465,15 +465,18 @@ class HunyuanVideo15Transformer3DModel(CachableDiT): def forward( self, hidden_states: torch.Tensor, + encoder_hidden_states: List[torch.Tensor], timestep: torch.LongTensor, - encoder_hidden_states: torch.Tensor, - encoder_attention_mask: torch.Tensor, + encoder_hidden_states_image: List[torch.Tensor], + encoder_attention_mask: List[torch.Tensor], + guidance: Optional[torch.Tensor] = None, timestep_r: Optional[torch.LongTensor] = None, - encoder_hidden_states_2: Optional[torch.Tensor] = None, - encoder_attention_mask_2: Optional[torch.Tensor] = None, - image_embeds: Optional[torch.Tensor] = None, attention_kwargs: Optional[Dict[str, Any]] = None, ): + encoder_hidden_states_image = encoder_hidden_states_image[0] + encoder_hidden_states, encoder_hidden_states_2 = encoder_hidden_states + encoder_attention_mask, encoder_attention_mask_2 = encoder_attention_mask + batch_size, num_channels, num_frames, height, width = hidden_states.shape p_t, p_h, p_w = self.config.patch_size_t, self.config.patch_size, self.config.patch_size post_patch_num_frames = num_frames // p_t @@ -511,8 +514,8 @@ class HunyuanVideo15Transformer3DModel(CachableDiT): encoder_hidden_states_2 = encoder_hidden_states_2 + encoder_hidden_states_2_cond_emb # image embed - encoder_hidden_states_3 = self.image_embedder(image_embeds) - is_t2v = torch.all(image_embeds == 0) + encoder_hidden_states_3 = self.image_embedder(encoder_hidden_states_image) + is_t2v = torch.all(encoder_hidden_states_image == 0) if is_t2v: encoder_hidden_states_3 = encoder_hidden_states_3 * 0.0 encoder_attention_mask_3 = torch.zeros( diff --git a/fastvideo/models/loader/component_loader.py b/fastvideo/models/loader/component_loader.py index 7f781e407..778cc4cc1 100644 --- a/fastvideo/models/loader/component_loader.py +++ b/fastvideo/models/loader/component_loader.py @@ -249,6 +249,7 @@ class TextEncoderLoader(ComponentLoader): dtype: str = "fp16"): use_cpu_offload = fastvideo_args.text_encoder_cpu_offload and len( getattr(model_config, "_fsdp_shard_conditions", [])) > 0 + logger.info("Text encoder use_cpu_offload: %s", use_cpu_offload) from fastvideo.platforms import current_platform @@ -480,8 +481,8 @@ class TransformerLoader(ComponentLoader): fastvideo_args.pipeline_config.dit_precision] # Load the model using FSDP loader - logger.info("Loading model from %s, default_dtype: %s", cls_name, - default_dtype) + logger.info("Loading model from %s, default_dtype: %s, cpu_offload: %s", cls_name, + default_dtype, fastvideo_args.dit_cpu_offload) assert fastvideo_args.hsdp_shard_dim is not None model = maybe_load_fsdp_model( model_cls=model_cls, diff --git a/fastvideo/pipelines/basic/hunyuan15/hunyuan15_pipeline.py b/fastvideo/pipelines/basic/hunyuan15/hunyuan15_pipeline.py index 916b2ac33..62d6b8aca 100644 --- a/fastvideo/pipelines/basic/hunyuan15/hunyuan15_pipeline.py +++ b/fastvideo/pipelines/basic/hunyuan15/hunyuan15_pipeline.py @@ -13,7 +13,8 @@ from fastvideo.pipelines.stages import (ConditioningStage, DecodingStage, DenoisingStage, InputValidationStage, LatentPreparationStage, TextEncodingStage, - TimestepPreparationStage) + TimestepPreparationStage, + Hy15ImageEncodingStage) # TODO(will): move PRECISION_TO_TYPE to better place @@ -56,6 +57,11 @@ class HunyuanVideo15Pipeline(ComposedPipelineBase): stage=LatentPreparationStage( scheduler=self.get_module("scheduler"), transformer=self.get_module("transformer"))) + + self.add_stage(stage_name="image_encoding_stage", + stage=Hy15ImageEncodingStage( + image_encoder=None, + image_processor=None)) self.add_stage(stage_name="denoising_stage", stage=DenoisingStage( diff --git a/fastvideo/pipelines/pipeline_registry.py b/fastvideo/pipelines/pipeline_registry.py index 64b7699a5..0e41765c8 100644 --- a/fastvideo/pipelines/pipeline_registry.py +++ b/fastvideo/pipelines/pipeline_registry.py @@ -25,6 +25,7 @@ _PIPELINE_NAME_TO_ARCHITECTURE_NAME: dict[str, str] = { "WanCausalDMDPipeline": "wan", "StepVideoPipeline": "stepvideo", "HunyuanVideoPipeline": "hunyuan", + "HunyuanVideo15Pipeline": "hunyuan15", "Cosmos2VideoToWorldPipeline": "cosmos", "MatrixGamePipeline": "matrixgame", "MatrixGameCausalDMDPipeline": "matrixgame", diff --git a/fastvideo/pipelines/stages/__init__.py b/fastvideo/pipelines/stages/__init__.py index 998d6eb09..cbe24da7b 100644 --- a/fastvideo/pipelines/stages/__init__.py +++ b/fastvideo/pipelines/stages/__init__.py @@ -16,7 +16,7 @@ from fastvideo.pipelines.stages.denoising import (CosmosDenoisingStage, from fastvideo.pipelines.stages.encoding import EncodingStage from fastvideo.pipelines.stages.image_encoding import ( ImageEncodingStage, MatrixGameImageEncodingStage, RefImageEncodingStage, - ImageVAEEncodingStage, VideoVAEEncodingStage) + ImageVAEEncodingStage, VideoVAEEncodingStage, Hy15ImageEncodingStage) from fastvideo.pipelines.stages.input_validation import InputValidationStage from fastvideo.pipelines.stages.latent_preparation import ( CosmosLatentPreparationStage, LatentPreparationStage) @@ -44,6 +44,7 @@ __all__ = [ "DecodingStage", "ImageEncodingStage", "MatrixGameImageEncodingStage", + "Hy15ImageEncodingStage", "RefImageEncodingStage", "ImageVAEEncodingStage", "VideoVAEEncodingStage", diff --git a/fastvideo/pipelines/stages/denoising.py b/fastvideo/pipelines/stages/denoising.py index 4ab4e1b76..084ed36b6 100644 --- a/fastvideo/pipelines/stages/denoising.py +++ b/fastvideo/pipelines/stages/denoising.py @@ -3,6 +3,7 @@ Denoising stage for diffusion pipelines. """ +import gc import inspect import weakref from collections.abc import Iterable @@ -474,6 +475,11 @@ class DenoisingStage(PipelineStage): logger.info("Memory after deallocating transformer: %s", torch.mps.current_allocated_memory()) + if fastvideo_args.dit_cpu_offload: + self.transformer = self.transformer.to('cpu') + gc.collect() + torch.cuda.empty_cache() + return batch def prepare_extra_func_kwargs(self, func, kwargs) -> dict[str, Any]: diff --git a/fastvideo/pipelines/stages/image_encoding.py b/fastvideo/pipelines/stages/image_encoding.py index 66d8c7db6..743bf2af3 100644 --- a/fastvideo/pipelines/stages/image_encoding.py +++ b/fastvideo/pipelines/stages/image_encoding.py @@ -99,6 +99,29 @@ class ImageEncodingStage(PipelineStage): V.list_of_tensors_dims(3)) return result +class Hy15ImageEncodingStage(ImageEncodingStage): + """ + Stage for encoding image prompts into embeddings for HunyuanVideo1.5 models. + """ + def verify_input(self, batch: ForwardBatch, + fastvideo_args: FastVideoArgs) -> VerificationResult: + """Verify image encoding stage inputs.""" + return VerificationResult() + + def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch: + """ + Encode the prompt into image encoder hidden states. + """ + if batch.pil_image is None: + batch.image_embeds = [ + torch.zeros(1, 729, 1152, device=get_local_torch_device()) + ] + + raw_latent_shape = list(batch.raw_latent_shape) + raw_latent_shape[1] = 1 + batch.video_latent = torch.zeros(tuple(raw_latent_shape), device=get_local_torch_device()) + return batch + class MatrixGameImageEncodingStage(ImageEncodingStage): CLIP_MEAN = [0.48145466, 0.4578275, 0.40821073] diff --git a/fastvideo/pipelines/stages/text_encoding.py b/fastvideo/pipelines/stages/text_encoding.py index c073986c1..19c7c19b2 100644 --- a/fastvideo/pipelines/stages/text_encoding.py +++ b/fastvideo/pipelines/stages/text_encoding.py @@ -69,12 +69,17 @@ class TextEncodingStage(PipelineStage): encoder_index=all_indices, return_attention_mask=True, ) + logger.info(f"prompt: {prompt_text}") + logger.info(f"pos text 1 sum: {prompt_embeds_list[0].sum()}, pos text 2 sum: {prompt_embeds_list[1].sum()}") + logger.info(f"pos text 1 mask sum: {prompt_masks_list[0].sum()}, pos text 2 mask sum: {prompt_masks_list[1].sum()}") for pe in prompt_embeds_list: batch.prompt_embeds.append(pe) if batch.prompt_attention_mask is not None: for am in prompt_masks_list: batch.prompt_attention_mask.append(am) + else: + raise ValueError("prompt_attention_mask is required") # Encode negative prompt if CFG is enabled if batch.do_classifier_free_guidance: @@ -85,7 +90,9 @@ class TextEncodingStage(PipelineStage): encoder_index=all_indices, return_attention_mask=True, ) - + logger.info(f"neg text: {batch.negative_prompt}") + logger.info(f"neg text 1 sum: {neg_embeds_list[0].sum()}, neg text 2 sum: {neg_embeds_list[1].sum()}") + logger.info(f"neg text 1 mask sum: {neg_masks_list[0].sum()}, neg text 2 mask sum: {neg_masks_list[1].sum()}") assert batch.negative_prompt_embeds is not None for ne in neg_embeds_list: batch.negative_prompt_embeds.append(ne) @@ -100,9 +107,9 @@ class TextEncodingStage(PipelineStage): """Verify text encoding stage inputs.""" result = VerificationResult() result.add_check("prompt", batch.prompt, V.string_or_list_strings) - result.add_check( - "negative_prompt", batch.negative_prompt, lambda x: not batch. - do_classifier_free_guidance or V.string_not_empty(x)) + # result.add_check( + # "negative_prompt", batch.negative_prompt, lambda x: not batch. + # do_classifier_free_guidance or V.string_not_empty(x)) result.add_check("do_classifier_free_guidance", batch.do_classifier_free_guidance, V.bool_value) result.add_check("prompt_embeds", batch.prompt_embeds, V.is_list) @@ -203,14 +210,6 @@ class TextEncodingStage(PipelineStage): preprocess_func = preprocess_funcs[i] postprocess_func = postprocess_funcs[i] - processed_texts: list[str] = [] - for prompt_str in texts: - processed_text = preprocess_func(prompt_str) - if processed_text is not None: - processed_texts.append(processed_text) - else: - processed_texts.append(prompt_str) - tok_kwargs = dict(encoder_config.tokenizer_kwargs) if max_length is not None: tok_kwargs["max_length"] = max_length @@ -222,7 +221,24 @@ class TextEncodingStage(PipelineStage): if padding is not None: tok_kwargs["padding"] = padding + processed_texts: list[str] = [] + for prompt_str in texts: + processed_text = preprocess_func(prompt_str) + if processed_text is not None: + processed_texts.append(processed_text) + else: + # Assuming batch_size = 1 + prompt_embeds = torch.zeros( + (1, tok_kwargs["max_length"], encoder_config.hidden_size), device=target_device + ) + attention_mask = torch.zeros((1, tok_kwargs["max_length"]), device=target_device, dtype=torch.int64) + embeds_list.append(prompt_embeds) + attn_masks_list.append(attention_mask) + return self.return_embeds(embeds_list, attn_masks_list, return_type, return_attention_mask, indices) + if encoder_config.is_chat_model: + logger.info(f"processed_texts: {processed_texts}") + logger.info(f"tok_kwargs: {tok_kwargs}") text_inputs = tokenizer.apply_chat_template(processed_texts, **tok_kwargs).to(target_device) else: text_inputs = tokenizer(processed_texts, @@ -249,6 +265,15 @@ class TextEncodingStage(PipelineStage): if return_attention_mask: attn_masks_list.append(attention_mask) + return self.return_embeds(embeds_list, attn_masks_list, return_type, return_attention_mask, indices) + + def return_embeds(self, + embeds_list: list[torch.Tensor], + attn_masks_list: list[torch.Tensor], + return_type: str = "list", + return_attention_mask: bool = False, + indices: list[int] = None, + ) -> tuple[list[torch.Tensor], list[torch.Tensor]]: # Shape results according to return_type if return_type == "list": if return_attention_mask: diff --git a/fastvideo/tests/transformers/test_hunyuanvideo15.py b/fastvideo/tests/transformers/test_hunyuanvideo15.py index 49322ba37..0fe1e7118 100644 --- a/fastvideo/tests/transformers/test_hunyuanvideo15.py +++ b/fastvideo/tests/transformers/test_hunyuanvideo15.py @@ -75,6 +75,7 @@ def test_hunyuanvideo15_transformer(): # Create identical inputs for both models batch_size = 1 seq_len = 30 + seq_len_2 = 70 # Video latents [B, C, T, H, W] hidden_states = torch.randn(batch_size, @@ -96,18 +97,20 @@ def test_hunyuanvideo15_transformer(): seq_len + 1, device=device, dtype=torch.bool) - # encoder_attention_mask[:, seq_len//2:] = False + encoder_hidden_states[:, 15:] = 0 + encoder_attention_mask[:, 15:] = False encoder_hidden_states_2 = torch.randn(batch_size, - seq_len + 1, + seq_len_2 + 1, 1472, device=device, dtype=precision) encoder_attention_mask_2 = torch.ones(batch_size, - seq_len + 1, + seq_len_2 + 1, device=device, dtype=torch.bool) - # encoder_attention_mask_2[:, seq_len//2:] = False + encoder_hidden_states_2[:, 39:] = 0 + encoder_attention_mask_2[:, 39:] = False image_embeds = torch.zeros(batch_size, 729, 1152, dtype=precision, device=device) @@ -136,11 +139,9 @@ def test_hunyuanvideo15_transformer(): forward_batch=forward_batch, ): output2 = model2(hidden_states=hidden_states, - encoder_hidden_states=encoder_hidden_states, - encoder_attention_mask=encoder_attention_mask, - encoder_hidden_states_2=encoder_hidden_states_2, - encoder_attention_mask_2=encoder_attention_mask_2, - image_embeds=image_embeds, + encoder_hidden_states=[encoder_hidden_states, encoder_hidden_states_2], + encoder_attention_mask=[encoder_attention_mask, encoder_attention_mask_2], + encoder_hidden_states_image=[image_embeds], timestep=timestep) # Check if outputs have the same shape diff --git a/fastvideo/tests/transformers/test_ori_hunyuanvideo15.py b/fastvideo/tests/transformers/test_ori_hunyuanvideo15.py index 0b0d6f5bb..f782a5c12 100644 --- a/fastvideo/tests/transformers/test_ori_hunyuanvideo15.py +++ b/fastvideo/tests/transformers/test_ori_hunyuanvideo15.py @@ -43,7 +43,7 @@ def test_hunyuanvideo15_transformer(): model2 = loader.load(TRANSFORMER_PATH, args).to(dtype=precision) model1 = HunyuanVideo_1_5_DiffusionTransformer.from_pretrained( - "/mnt/weka/home/hao.zhang/wei/hy1.5_models/transformer/480p_t2v", device=device, + "/mnt/fast-disks/hao_lab/wei/hy1.5_models/transformer/480p_t2v", device=device, torch_dtype=precision).to(device, dtype=precision).requires_grad_(False) # model1 = HunyuanVideo15Transformer3DModel.from_pretrained( # TRANSFORMER_PATH, device=device, @@ -148,11 +148,9 @@ def test_hunyuanvideo15_transformer(): forward_batch=forward_batch, ): output2 = model2(hidden_states=hidden_states, - encoder_hidden_states=encoder_hidden_states, - encoder_attention_mask=encoder_attention_mask, - encoder_hidden_states_2=encoder_hidden_states_2, - encoder_attention_mask_2=encoder_attention_mask_2, - image_embeds=image_embeds, + encoder_hidden_states=[encoder_hidden_states, encoder_hidden_states_2], + encoder_attention_mask=[encoder_attention_mask, encoder_attention_mask_2], + encoder_hidden_states_image=[image_embeds], timestep=timestep) # Check if outputs have the same shape