From 8039b29624be79eae6f02ea601142f1a94cdf0af Mon Sep 17 00:00:00 2001 From: Evelyn Zhang Date: Tue, 30 Dec 2025 01:36:05 +0000 Subject: [PATCH] add hunyuan15 preprocessing --- .../hunyuan15_t2v/download_dataset.sh | 3 + .../preprocess_hunyuan_data_t2v.sh | 25 ++++++++ fastvideo/models/vaes/hunyuan15vae.py | 6 +- .../preprocess/hunyuan15/__init__.py | 0 .../hunyuan15/hunyuan_preprocess_pipelins.py | 59 +++++++++++++++++++ fastvideo/workflow/preprocess/components.py | 10 +++- 6 files changed, 100 insertions(+), 3 deletions(-) create mode 100644 examples/training/finetune/hunyuan15_t2v/download_dataset.sh create mode 100755 examples/training/finetune/hunyuan15_t2v/preprocess_hunyuan_data_t2v.sh create mode 100644 fastvideo/pipelines/preprocess/hunyuan15/__init__.py create mode 100644 fastvideo/pipelines/preprocess/hunyuan15/hunyuan_preprocess_pipelins.py diff --git a/examples/training/finetune/hunyuan15_t2v/download_dataset.sh b/examples/training/finetune/hunyuan15_t2v/download_dataset.sh new file mode 100644 index 000000000..5743cd19c --- /dev/null +++ b/examples/training/finetune/hunyuan15_t2v/download_dataset.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +python scripts/huggingface/download_hf.py --repo_id "wlsaidhi/crush-smol-merged" --local_dir "data/crush-smol" --repo_type "dataset" diff --git a/examples/training/finetune/hunyuan15_t2v/preprocess_hunyuan_data_t2v.sh b/examples/training/finetune/hunyuan15_t2v/preprocess_hunyuan_data_t2v.sh new file mode 100755 index 000000000..3860570da --- /dev/null +++ b/examples/training/finetune/hunyuan15_t2v/preprocess_hunyuan_data_t2v.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +GPU_NUM=1 # 2,4,8 +MODEL_PATH="hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v" +DATASET_PATH="data/crush-smol" +OUTPUT_DIR="data/crush-smol_processed_t2v_hunyuan15/" + +torchrun --nproc_per_node=$GPU_NUM --master_port=29513 \ + -m fastvideo.pipelines.preprocess.v1_preprocessing_new \ + --model_path $MODEL_PATH \ + --mode preprocess \ + --workload_type t2v \ + --preprocess.video_loader_type torchvision \ + --preprocess.dataset_type merged \ + --preprocess.dataset_path $DATASET_PATH \ + --preprocess.dataset_output_dir $OUTPUT_DIR \ + --preprocess.preprocess_video_batch_size 2 \ + --preprocess.dataloader_num_workers 0 \ + --preprocess.max_height 480 \ + --preprocess.max_width 832 \ + --preprocess.num_frames 61 \ + --preprocess.train_fps 15 \ + --preprocess.samples_per_file 8 \ + --preprocess.flush_frequency 8 \ + --preprocess.video_length_tolerance_range 5 diff --git a/fastvideo/models/vaes/hunyuan15vae.py b/fastvideo/models/vaes/hunyuan15vae.py index 66201bd09..ecc212ea4 100644 --- a/fastvideo/models/vaes/hunyuan15vae.py +++ b/fastvideo/models/vaes/hunyuan15vae.py @@ -114,11 +114,14 @@ class HunyuanVideo15AttnBlock(nn.Module): """ seq_len = n_frame * n_hw mask = torch.full((seq_len, seq_len), float("-inf"), dtype=dtype, device=device) + for i in range(seq_len): i_frame = i // n_hw mask[i, : (i_frame + 1) * n_hw] = 0 if batch_size is not None: - mask = mask.unsqueeze(0).expand(batch_size, -1, -1) + # mask = mask.unsqueeze(0).expand(batch_size, -1, -1) + mask = mask.unsqueeze(0).unsqueeze(1).expand(batch_size, 1, -1, -1) + return mask def forward(self, x: torch.Tensor) -> torch.Tensor: @@ -131,6 +134,7 @@ class HunyuanVideo15AttnBlock(nn.Module): value = self.to_v(x) batch_size, channels, frames, height, width = query.shape + query = query.reshape(batch_size, channels, frames * height * width).permute(0, 2, 1).unsqueeze(1).contiguous() key = key.reshape(batch_size, channels, frames * height * width).permute(0, 2, 1).unsqueeze(1).contiguous() diff --git a/fastvideo/pipelines/preprocess/hunyuan15/__init__.py b/fastvideo/pipelines/preprocess/hunyuan15/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/fastvideo/pipelines/preprocess/hunyuan15/hunyuan_preprocess_pipelins.py b/fastvideo/pipelines/preprocess/hunyuan15/hunyuan_preprocess_pipelins.py new file mode 100644 index 000000000..9eb987210 --- /dev/null +++ b/fastvideo/pipelines/preprocess/hunyuan15/hunyuan_preprocess_pipelins.py @@ -0,0 +1,59 @@ +from fastvideo.fastvideo_args import FastVideoArgs +from fastvideo.pipelines.composed_pipeline_base import ComposedPipelineBase +from fastvideo.pipelines.preprocess.preprocess_stages import ( + TextTransformStage, VideoTransformStage) +from fastvideo.pipelines.stages import (EncodingStage, ImageEncodingStage, + TextEncodingStage) +from fastvideo.pipelines.stages.image_encoding import ImageVAEEncodingStage + + +class PreprocessPipelineT2V(ComposedPipelineBase): + _required_config_modules = [ + "text_encoder", "tokenizer", "text_encoder_2", "tokenizer_2", "vae" + ] + + def create_pipeline_stages(self, fastvideo_args: FastVideoArgs): + assert fastvideo_args.preprocess_config is not None + + self.add_stage( + stage_name="text_transform_stage", + stage=TextTransformStage( + cfg_uncondition_drop_rate=fastvideo_args.preprocess_config.training_cfg_rate, + seed=fastvideo_args.preprocess_config.seed, + ) + ) + + text_encoders = [ + self.get_module("text_encoder"), + self.get_module("text_encoder_2") + ] + tokenizers = [ + self.get_module("tokenizer"), + self.get_module("tokenizer_2") + ] + + self.add_stage( + stage_name="prompt_encoding_stage", + stage=TextEncodingStage( + text_encoders=text_encoders, + tokenizers=tokenizers, + ) + ) + + self.add_stage( + stage_name="video_transform_stage", + stage=VideoTransformStage( + train_fps=fastvideo_args.preprocess_config.train_fps, + num_frames=fastvideo_args.preprocess_config.num_frames, + max_height=fastvideo_args.preprocess_config.max_height, + max_width=fastvideo_args.preprocess_config.max_width, + do_temporal_sample=fastvideo_args.preprocess_config.do_temporal_sample, + ) + ) + + self.add_stage( + stage_name="video_encoding_stage", + stage=EncodingStage(vae=self.get_module("vae")) + ) + +EntryClass = [PreprocessPipelineT2V] diff --git a/fastvideo/workflow/preprocess/components.py b/fastvideo/workflow/preprocess/components.py index 0ba03d7db..8df51a817 100644 --- a/fastvideo/workflow/preprocess/components.py +++ b/fastvideo/workflow/preprocess/components.py @@ -200,8 +200,14 @@ class ParquetDatasetSaver: value = getattr(batch, key.name) if isinstance(value, list): for idx in range(len(value)): - if isinstance(value[idx], torch.Tensor): - value[idx] = value[idx].cpu().numpy() + # if isinstance(value[idx], torch.Tensor): + # value[idx] = value[idx].cpu().numpy() + t = value[idx] + if isinstance(t, torch.Tensor): + if t.dtype == torch.bfloat16: + t = t.float() # or t.to(torch.float16) if you want smaller + value[idx] = t.cpu().numpy() + elif isinstance(value, torch.Tensor): value = value.cpu().numpy() setattr(batch, key.name, value)