From fb5efebdab2d47ed465d4062ecc6b194204ff173 Mon Sep 17 00:00:00 2001 From: Kyle Hu <160547796+KyleNeverGivesUp@users.noreply.github.com> Date: Wed, 7 Oct 2026 02:19:22 -0700 Subject: [PATCH] [feat] Add HunyuanVideo 1.5 T2V training support (#1662) Co-authored-by: William Lin Co-authored-by: SolitaryThinker Co-authored-by: Claude Opus 5.5 (1M context) --- docs/training/hunyuan15_overfit.md | 197 ++++++ .../configs/fine_tuning/hunyuan15/t2v.yaml | 78 +++ .../train/configs/overfit_hunyuan15_t2v.yaml | 95 +++ fastvideo/dataset/dataloader/schema.py | 11 + fastvideo/dataset/utils.py | 41 +- .../test_hunyuan15_dual_text_schema.py | 123 ++++ .../fixtures/hunyuan15_t2v_finetune_min.yaml | 50 ++ .../train/fixtures/hunyuan15_t2v_min.yaml | 20 + .../train/methods/grad_norm_regression.py | 2 +- .../train/methods/test_hunyuan15_finetune.py | 145 +++++ .../train/models/test_hunyuan15_smoke.py | 149 +++++ .../tests/train/models/test_load_hunyuan15.py | 153 +++++ .../models/test_wan_override_contract.py | 5 +- fastvideo/train/models/hunyuan15/__init__.py | 5 + fastvideo/train/models/hunyuan15/hunyuan15.py | 581 ++++++++++++++++++ fastvideo/train/models/wan/wan.py | 16 +- fastvideo/training/training_utils.py | 4 + mkdocs.yml | 1 + 18 files changed, 1663 insertions(+), 13 deletions(-) create mode 100644 docs/training/hunyuan15_overfit.md create mode 100644 examples/train/configs/fine_tuning/hunyuan15/t2v.yaml create mode 100644 examples/train/configs/overfit_hunyuan15_t2v.yaml create mode 100644 fastvideo/tests/dataset/test_hunyuan15_dual_text_schema.py create mode 100644 fastvideo/tests/train/fixtures/hunyuan15_t2v_finetune_min.yaml create mode 100644 fastvideo/tests/train/fixtures/hunyuan15_t2v_min.yaml create mode 100644 fastvideo/tests/train/methods/test_hunyuan15_finetune.py create mode 100644 fastvideo/tests/train/models/test_hunyuan15_smoke.py create mode 100644 fastvideo/tests/train/models/test_load_hunyuan15.py create mode 100644 fastvideo/train/models/hunyuan15/__init__.py create mode 100644 fastvideo/train/models/hunyuan15/hunyuan15.py diff --git a/docs/training/hunyuan15_overfit.md b/docs/training/hunyuan15_overfit.md new file mode 100644 index 000000000..bfe314d42 --- /dev/null +++ b/docs/training/hunyuan15_overfit.md @@ -0,0 +1,197 @@ +# HunyuanVideo 1.5 T2V Training + +This guide walks through training HunyuanVideo 1.5 (480p, text-to-video) in +FastVideo's modular training stack: preprocessing one clip, running a +single-GPU overfit to verify the pipeline end to end, and scaling out to +full-parameter multi-GPU fine-tuning. + +The preprocessing step below relies on the HY1.5 data-side pipeline — the +dual text-embedding parquet schema, its collate path, and +`preprocess_hunyuan15_overfit.py` — which ships separately. Training itself +only needs the parquet that step produces. + +## What you need + +| | | +|---|---| +| GPU | 48GB or more for the LoRA overfit; multi-GPU for full-parameter runs | +| Disk | ~40GB for the checkpoint (DiT 17GB + Qwen2.5-VL 16GB + ByT5/VAE) | +| Checkpoint | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v` | + +Install as documented in the repository README: + +```bash +UV_TORCH_BACKEND=cu126 uv pip install -e ".[dev]" # cu130 on CUDA 13 +python -c "import torch; print(torch.__version__, torch.cuda.is_available())" +``` + +`fastvideo-kernel` pulls in a CUTLASS submodule. If the build fails with +`cutlass/cutlass.h: No such file or directory`, run +`git submodule update --init --recursive` — or skip the kernel package +entirely, since this guide uses the PyTorch SDPA attention backend. + +## 1. Prepare a clip + +The preprocess script reads a caption manifest plus a `videos/` +subdirectory: + +``` +data/hunyuan15_overfit/ +├── videos2caption.json +└── videos/ + └── robot_pouring.mp4 +``` + +The repository already ships a suitable clip (5.8s, 16fps, 1280x704): + +```bash +mkdir -p data/hunyuan15_overfit/videos +cp assets/videos/robot_pouring.mp4 data/hunyuan15_overfit/videos/ + +cat > data/hunyuan15_overfit/videos2caption.json <<'EOF' +[ + { + "path": "robot_pouring.mp4", + "cap": ["A robotic arm carefully pours liquid from a bottle into a glass on a kitchen counter, steady camera, soft indoor lighting."] + } +] +EOF +``` + +Clips must supply at least 81 frames at 16fps (≈5.1s) and be a single +continuous shot; the script resizes to 480x832. + +## 2. Preprocess + +```bash +CUDA_VISIBLE_DEVICES=0 python fastvideo/pipelines/preprocess/preprocess_hunyuan15_overfit.py +``` + +This encodes the video with the HY1.5 VAE and the caption with both text +encoders, then writes a parquet shard plus a validation prompt file. +Expected output: + +``` +[Qwen 1/1] (29, 3584) # caption tokens x Qwen hidden size +[ByT5 1/1] (0, 1472) # zero-length: the caption has no quoted glyph text +[VAE 1/1] (32, 21, 30, 52) # 32 channels, 21 latent frames, 480/16 x 832/16 +Wrote 1 records to data/hunyuan15_overfit_preprocessed/data_00000.parquet +``` + +A `(0, 1472)` ByT5 shape is normal and expected: ByT5 only receives the +glyph text extracted from quotes in the caption, so most captions produce +zero tokens. The training plugin trims and forwards that zero-length +stream unchanged. + +Note that the parquet stores raw `latent_dist.mode()` outputs — the +training side multiplies by the VAE `scaling_factor` (1.03682) in +`normalize_dit_input("hunyuan15", ...)`. + +## 3. Overfit on a single GPU + +```bash +export WANDB_MODE=offline # or `wandb login` for cloud tracking +bash examples/train/run.sh examples/train/configs/overfit_hunyuan15_t2v.yaml +``` + +The config trains LoRA adapters on the attention projections, which keeps +the run at roughly 30GB. Loss is logged through the configured tracker, +not to stdout. + +For a quick "does it run at all" check, cap the step count: + +```bash +bash examples/train/run.sh examples/train/configs/overfit_hunyuan15_t2v.yaml \ + --training.loop.max_train_steps 50 +``` + +**Run long jobs under `tmux` or `screen`.** A dropped SSH session takes the +training process with it, and a checkpoint interrupted mid-write is not +resumable (the DCP `.metadata` file is written last). + +### Reading the loss + +Flow-matching loss is noisy: every step samples a random timestep, and the +denoising task is far easier at low noise levels than at high ones. Two +adjacent steps can differ by an order of magnitude with no bearing on +whether the model is learning. Compare segment means rather than +individual steps: + +```bash +RUN=$(ls -dt outputs/hunyuan15_overfit/tracker/wandb/offline-run-* | head -1) +strings -n 1 $RUN/run-*.wandb | grep -A 2 -x "total_loss" \ + | grep -E "^[0-9]+\.[0-9]" | uniq > /tmp/losses.txt + +python3 -c " +import re +vals = [float(m.group(1)) for line in open('/tmp/losses.txt') + if (m := re.match(r'^([0-9]+\.[0-9]+)', line))] +n = len(vals) +for i in range(6): + seg = vals[i * n // 6:(i + 1) * n // 6] + print(f'segment {i + 1}/6: {sum(seg) / len(seg):.4f}') +" +``` + +A steadily falling segment mean means the data → forward → loss → backward +→ optimizer chain is numerically sound. A flat series, or NaNs, points at +a real problem — stop and investigate rather than burning GPU hours. + +## 4. Scale out + +`examples/train/configs/fine_tuning/hunyuan15/t2v.yaml` runs +full-parameter training across 8 GPUs with FSDP. Full-parameter training +does not fit one 80GB card: + +It reads `data/hunyuan15_preprocessed` (and its `validation_prompts.json`), +not the overfit directory above — either preprocess into that path or pass +`--training.data.data_path` and `--callbacks.validation.dataset_file`. + +| | bf16 | +|---|---| +| Weights | 16.7 GB | +| Gradients | 16.7 GB | +| AdamW `exp_avg` + `exp_avg_sq` | 33.4 GB | +| **Optimizer + weights subtotal** | **66.8 GB** | +| Activations, gradient-checkpoint boundaries, VAE, allocator overhead | ~13 GB | +| **Total** | **~80 GB** | + +That subtotal is independent of sequence length, so shortening the clip +(`--training.data.num_latent_t`) barely helps — measured peak memory moved +by 0.07GB going from 21 latent frames down to 9. Options that do move the +needle: + +- **LoRA** (what the overfit config uses) — gradients and optimizer state + shrink to the adapter parameters, ~30GB total. +- **FSDP sharding across GPUs** — the 66.8GB subtotal divides by the number + of shards. +- **A memory-light optimizer** — SGD with momentum keeps a single state + tensor instead of two. `torch.optim.Adafactor` is smaller still, but is + not DTensor-safe: its in-place `.square_()` fails under FSDP2 with + `aten.pow_.Scalar: in-place operations that require placement changes + are not supported`. + +Set `training.dit_precision: bf16` for any single-device run. The default +is fp32, which doubles weights, gradients, and optimizer state. + +## Troubleshooting + +| Symptom | Cause and fix | +|---|---| +| `wandb.errors.UsageError: No API key configured` | Setting `tracker.project_name` enables W&B automatically. Use `export WANDB_MODE=offline` or `wandb login`. | +| `RuntimeError: Given normalized_shape=[1472] ... got [1, 0, 512]` during validation | Fixed upstream: the validation pipeline used to overwrite the loader-populated text-encoder config, so the zero-length ByT5 placeholder was sized from the static T5 default (512) instead of HY1.5's 1472. If you still see this, your branch predates that fix. | +| `OutOfMemoryError` at `optimizer.step()` | Optimizer state does not fit. Use the LoRA config, shard across GPUs, or switch optimizers — see the table above. | +| `No LoRA-compatible layers were found for the requested target modules` | HY1.5 names its attention projections `img_attn_qkv` / `txt_attn_qkv` / `img_attn_proj` / `txt_attn_proj`, which the default target list does not cover. The shipped config sets them explicitly. | +| `ValueError: No frames could be decoded from ...` | The clip must live under `data/hunyuan15_overfit/videos/`, not the manifest directory itself. | +| Training dies with SIGTERM and no traceback | The SSH session ended, or another user on a shared machine reclaimed memory. Use `tmux`, and coordinate before taking most of a shared box. | +| Hangs at "Downloading model snapshot from HF Hub" | Rate-limited Hub request. `export HF_HUB_OFFLINE=1` once the weights are cached, or set `HF_TOKEN`. | + +## Notes on other platforms + +The stack runs on aarch64 + Blackwell (DGX Spark, GB10, CUDA 13.0): +preprocessing and training both complete there. Unified memory makes +checkpoint loading much faster than on a discrete GPU — 17GB of weights +load in about 1.5s versus 30s over PCIe — while each training step runs +roughly 1.5x slower than an A100 80GB. `nvidia-smi` reports +`Memory-Usage: Not Supported` on that platform; use `free -m` instead, +since GPU allocations come out of the same pool as system memory. diff --git a/examples/train/configs/fine_tuning/hunyuan15/t2v.yaml b/examples/train/configs/fine_tuning/hunyuan15/t2v.yaml new file mode 100644 index 000000000..987cadb39 --- /dev/null +++ b/examples/train/configs/fine_tuning/hunyuan15/t2v.yaml @@ -0,0 +1,78 @@ +# HunyuanVideo 1.5 480p T2V finetune config. +# +# Data must be preprocessed with the HY1.5 VAE + Qwen2.5-VL / ByT5 +# text encoders into parquet format before training. + +models: + student: + _target_: fastvideo.train.models.hunyuan15.Hunyuan15Model + init_from: hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v + trainable: true + flow_shift: 5.0 + +method: + _target_: fastvideo.train.methods.fine_tuning.finetune.FineTuneMethod + +training: + distributed: + num_gpus: 8 + sp_size: 1 + tp_size: 1 + # Shard the model across all eight GPUs rather than replicating it: + # weights, gradients, and both AdamW states are 66.8GB in bf16, so a + # full replica per GPU would not fit an 80GB card. + hsdp_replicate_dim: 1 + hsdp_shard_dim: 8 + + data: + data_path: data/hunyuan15_preprocessed + dataloader_num_workers: 4 + train_batch_size: 1 + training_cfg_rate: 0.0 + seed: 1000 + # HY1.5 VAE: 4x temporal, 16x spatial compression. + # 81 frames -> 21 latent frames, 480x832 -> 30x52 + num_latent_t: 21 + num_height: 480 + num_width: 832 + num_frames: 81 + + optimizer: + learning_rate: 1.0e-5 + betas: [0.9, 0.999] + weight_decay: 0.01 + lr_scheduler: constant + lr_warmup_steps: 0 + + loop: + max_train_steps: 5000 + gradient_accumulation_steps: 1 + + checkpoint: + output_dir: outputs/hunyuan15_finetune + training_state_checkpointing_steps: 500 + checkpoints_total_limit: 3 + resume_from_checkpoint: latest + + tracker: + project_name: fastvideo_hunyuan15 + run_name: hunyuan15_finetune + + model: + precondition_outputs: true + enable_gradient_checkpointing_type: full + +callbacks: + grad_clip: + _target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback + max_grad_norm: 1.0 + validation: + _target_: fastvideo.train.callbacks.validation.ValidationCallback + pipeline_target: fastvideo.pipelines.basic.hunyuan15.hunyuan15_pipeline.HunyuanVideo15Pipeline + dataset_file: data/hunyuan15_preprocessed/validation_prompts.json + every_steps: 100 + sampling_steps: [50] + guidance_scale: 7.0 + +pipeline: + flow_shift: 5.0 diff --git a/examples/train/configs/overfit_hunyuan15_t2v.yaml b/examples/train/configs/overfit_hunyuan15_t2v.yaml new file mode 100644 index 000000000..ef12f38b3 --- /dev/null +++ b/examples/train/configs/overfit_hunyuan15_t2v.yaml @@ -0,0 +1,95 @@ +# HunyuanVideo 1.5 480p T2V overfitting test config. +# +# Overfits on a single short video (480x832, 81 frames) to verify the +# HunyuanVideo 1.5 training plugin works end-to-end. LoRA keeps the run +# within a single 80GB card (~30GB); see fine_tuning/hunyuan15/t2v.yaml +# for the multi-GPU full-parameter setup. +# +# Preprocess data first (the script expects the clip under +# data/hunyuan15_overfit/videos/ next to videos2caption.json): +# CUDA_VISIBLE_DEVICES=0 python fastvideo/pipelines/preprocess/preprocess_hunyuan15_overfit.py +# +# Run: +# bash examples/train/run.sh examples/train/configs/overfit_hunyuan15_t2v.yaml + +models: + student: + _target_: fastvideo.train.models.hunyuan15.Hunyuan15Model + init_from: hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v + trainable: true + enable_gradient_checkpointing_type: full + flow_shift: 5.0 + # Full-parameter 8B training needs ~80GB before activations (weights + + # grads + both AdamW states are 66.8GB on their own), which does not fit + # a single 80GB card; LoRA keeps this run at ~30GB. HY1.5 names its + # attention projections differently from Wan/Flux, so the default + # target_modules find no matching layers. + lora: + enable: true + rank: 64 + alpha: 64 + target_modules: [img_attn_qkv, txt_attn_qkv, img_attn_proj, txt_attn_proj] + +method: + _target_: fastvideo.train.methods.fine_tuning.finetune.FineTuneMethod + +training: + dit_precision: bf16 + distributed: + num_gpus: 1 + sp_size: 1 + tp_size: 1 + hsdp_replicate_dim: 1 + hsdp_shard_dim: 1 + + data: + data_path: data/hunyuan15_overfit_preprocessed + dataloader_num_workers: 0 + train_batch_size: 1 + training_cfg_rate: 0.0 + seed: 42 + # HY1.5 VAE: 4x temporal, 16x spatial compression. + # 81 frames -> 21 latent frames, 480x832 -> 30x52 + num_latent_t: 21 + num_height: 480 + num_width: 832 + num_frames: 81 + + optimizer: + learning_rate: 5.0e-5 + betas: [0.9, 0.999] + weight_decay: 0.0 + lr_scheduler: constant + lr_warmup_steps: 0 + + loop: + max_train_steps: 300 + gradient_accumulation_steps: 1 + + checkpoint: + output_dir: outputs/hunyuan15_overfit + training_state_checkpointing_steps: 50 + checkpoints_total_limit: 2 + + tracker: + project_name: fastvideo_hunyuan15 + run_name: hunyuan15_overfit + + model: + precondition_outputs: true + enable_gradient_checkpointing_type: full + +callbacks: + grad_clip: + _target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback + max_grad_norm: 1.0 + validation: + _target_: fastvideo.train.callbacks.validation.ValidationCallback + pipeline_target: fastvideo.pipelines.basic.hunyuan15.hunyuan15_pipeline.HunyuanVideo15Pipeline + dataset_file: data/hunyuan15_overfit_preprocessed/validation_prompts.json + every_steps: 150 + sampling_steps: [50] + guidance_scale: 7.0 + +pipeline: + flow_shift: 5.0 diff --git a/fastvideo/dataset/dataloader/schema.py b/fastvideo/dataset/dataloader/schema.py index 6fe807dbc..63e34833b 100644 --- a/fastvideo/dataset/dataloader/schema.py +++ b/fastvideo/dataset/dataloader/schema.py @@ -79,6 +79,17 @@ pyarrow_schema_t2v = pa.schema([ pa.field("fps", pa.float64()), ]) +# HunyuanVideo 1.5 conditions on two text streams (Qwen2.5-VL plus a ByT5 glyph +# stream), so its rows carry a second text-embedding triplet. The paired +# text_attention_mask_2 is synthesised by collate_rows_from_parquet_schema from +# the padding, exactly as text_attention_mask is for the primary stream. +pyarrow_schema_t2v_dual_text = pa.schema( + list(pyarrow_schema_t2v) + [ + pa.field("text_embedding_2_bytes", pa.binary()), + pa.field("text_embedding_2_shape", pa.list_(pa.int64())), + pa.field("text_embedding_2_dtype", pa.string()), + ]) + # One text-to-video-and-audio (T2VA) row owns synchronized video, audio, and text tensors so # collate_rows_from_parquet_schema cannot pair targets from different samples. # Each tensor uses the bytes/shape/dtype triplet that the collator discovers. diff --git a/fastvideo/dataset/utils.py b/fastvideo/dataset/utils.py index a20567aba..71d80462b 100644 --- a/fastvideo/dataset/utils.py +++ b/fastvideo/dataset/utils.py @@ -146,6 +146,15 @@ def collate_latents_embs_masks(batch_to_process, return all_latents, all_embs, all_masks, caption_text +# Text streams are padded to a fixed length and paired with the attention mask +# the trainer trims against. The second entry is HunyuanVideo 1.5's ByT5 glyph +# stream, which rides alongside the primary Qwen one. +_TEXT_STREAM_MASKS = { + "text_embedding": "text_attention_mask", + "text_embedding_2": "text_attention_mask_2", +} + + def collate_rows_from_parquet_schema(rows, parquet_schema, text_padding_length, @@ -191,19 +200,30 @@ def collate_rows_from_parquet_schema(rows, bytes_key = f"{tensor_name}_bytes" dtype_key = f"{tensor_name}_dtype" + # A secondary text stream that this parquet predates: leave the key out + # so the trainer's own zero-token fallback (and its warning) applies, + # rather than a placeholder of the wrong width. + if (tensor_name == "text_embedding_2" and not any(row.get(bytes_key) is not None for row in rows)): + continue + for row in rows: # Get tensor data from row using the existing helper function pattern if shape_key in row and bytes_key in row: shape = row[shape_key] bytes_data = row[bytes_key] - if len(bytes_data) == 0: - tensor = torch.zeros(0, dtype=torch.bfloat16) + if bytes_data is None or len(bytes_data) == 0: + # Keep the declared shape: a zero-token stream (e.g. a + # caption with no glyph text) still carries its width. + if shape: + tensor = torch.zeros(*shape, dtype=torch.bfloat16) + else: + tensor = torch.zeros(0, dtype=torch.bfloat16) else: # Deterministic per-sample CFG dropout # using sample index (resume-safe). drop = False - if (tensor_name == 'text_embedding' and cfg_rate > 0): + if (tensor_name in _TEXT_STREAM_MASKS and cfg_rate > 0): sample_idx = row.get("_sample_index") if sample_idx is not None: drop = (random.Random(seed ^ sample_idx).random() < cfg_rate) @@ -226,10 +246,18 @@ def collate_rows_from_parquet_schema(rows, tensor_list.append(torch.zeros(0, dtype=torch.bfloat16)) # Stack tensors with special handling for text embeddings - if tensor_name == 'text_embedding': + if tensor_name in _TEXT_STREAM_MASKS: # Handle text embeddings with padding padded_tensors = [] attention_masks = [] + # Empty entries have to stack against the real ones, so follow + # their dtype instead of the bfloat16 placeholder default. + empty_dtype = next((t.dtype for t in tensor_list if t.numel() > 0), torch.bfloat16) + # ...and their width: a row whose stream is missing entirely (a + # legacy shard mixed into the same batch) carries only a 1-D + # placeholder, and must not inject a 768-wide stub that cannot + # stack against the real rows. + empty_width = next((int(t.shape[1]) for t in tensor_list if t.dim() > 1), 768) for tensor in tensor_list: if tensor.numel() > 0: @@ -238,11 +266,12 @@ def collate_rows_from_parquet_schema(rows, attention_masks.append(mask) else: # Handle empty embeddings - assume default embedding dimension - padded_tensors.append(torch.zeros(text_padding_length, 768, dtype=torch.bfloat16)) + width = (int(tensor.shape[1]) if tensor.dim() > 1 else empty_width) + padded_tensors.append(torch.zeros(text_padding_length, width, dtype=empty_dtype)) attention_masks.append(torch.zeros(text_padding_length)) batch_data[tensor_name] = torch.stack(padded_tensors) - batch_data['text_attention_mask'] = torch.stack(attention_masks) + batch_data[_TEXT_STREAM_MASKS[tensor_name]] = torch.stack(attention_masks) else: # Stack all tensors to preserve batch consistency # Don't filter out None or empty tensors as this breaks batch sizing diff --git a/fastvideo/tests/dataset/test_hunyuan15_dual_text_schema.py b/fastvideo/tests/dataset/test_hunyuan15_dual_text_schema.py new file mode 100644 index 000000000..832fd7cad --- /dev/null +++ b/fastvideo/tests/dataset/test_hunyuan15_dual_text_schema.py @@ -0,0 +1,123 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The HY1.5 parquet schema must survive collation into a training batch. + +``Hunyuan15Model.prepare_batch`` reads ``text_embedding_2`` / +``text_attention_mask_2`` off ``raw_batch``, but ``collate_rows_from_parquet_schema`` +only emits the tensor fields its schema declares. A schema without the ByT5 +triplet therefore drops the glyph stream silently, so pin the contract against +a real parquet row instead of a hand-built dict. +""" + +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pyarrow as pa +import pyarrow.parquet as pq +import torch + +from fastvideo.dataset.dataloader.schema import ( + pyarrow_schema_t2v, + pyarrow_schema_t2v_dual_text, +) +from fastvideo.dataset.utils import collate_rows_from_parquet_schema + +_QWEN_DIM = 3584 +_BYT5_DIM = 1472 + + +def _write_row(path: Path, byt5_tokens: int, schema: pa.Schema) -> dict: + """Write one parquet row and read it back the way the dataset does.""" + qwen = np.zeros((4, _QWEN_DIM), dtype=np.float32) + latent = np.zeros((32, 2, 4, 4), dtype=np.float32) + record = { + "id": "clip", + "vae_latent_bytes": latent.tobytes(), + "vae_latent_shape": list(latent.shape), + "vae_latent_dtype": "float32", + "text_embedding_bytes": qwen.tobytes(), + "text_embedding_shape": list(qwen.shape), + "text_embedding_dtype": "float32", + "file_name": "clip.mp4", + "caption": "a caption", + "media_type": "video", + "width": 480, + "height": 832, + "num_frames": 81, + "duration_sec": 5.0, + "fps": 16.0, + } + if "text_embedding_2_bytes" in schema.names: + byt5 = np.zeros((byt5_tokens, _BYT5_DIM), dtype=np.float32) + record["text_embedding_2_bytes"] = byt5.tobytes() + record["text_embedding_2_shape"] = list(byt5.shape) + record["text_embedding_2_dtype"] = "float32" + + assert set(record) == set(schema.names) + pq.write_table(pa.table({k: [v] for k, v in record.items()}, schema=schema), path) + return pq.read_table(path).to_pylist()[0] + + +def test_collate_keeps_second_text_stream(tmp_path: Path) -> None: + row = _write_row(tmp_path / "data_00000.parquet", byt5_tokens=3, schema=pyarrow_schema_t2v_dual_text) + + batch = collate_rows_from_parquet_schema([row], pyarrow_schema_t2v_dual_text, text_padding_length=8) + + assert batch["text_embedding_2"].shape == (1, 8, _BYT5_DIM) + torch.testing.assert_close( + batch["text_attention_mask_2"], + torch.tensor([[1.0] * 3 + [0.0] * 5]), + ) + + +def test_collate_keeps_zero_token_stream_width(tmp_path: Path) -> None: + """A caption with no glyph text keeps its width, not a 768-wide stub.""" + row = _write_row(tmp_path / "data_00000.parquet", byt5_tokens=0, schema=pyarrow_schema_t2v_dual_text) + + batch = collate_rows_from_parquet_schema([row], pyarrow_schema_t2v_dual_text, text_padding_length=8) + + assert batch["text_embedding_2"].shape == (1, 8, _BYT5_DIM) + assert int(batch["text_attention_mask_2"].sum().item()) == 0 + + +def test_collate_omits_second_stream_for_legacy_parquet(tmp_path: Path) -> None: + """A t2v parquet predating the ByT5 columns must not fabricate a stream. + + Leaving the key out is what lets ``prepare_batch`` apply its own + zero-token fallback (and warn) instead of forwarding a mis-shaped tensor. + """ + row = _write_row(tmp_path / "data_00000.parquet", byt5_tokens=0, schema=pyarrow_schema_t2v) + + batch = collate_rows_from_parquet_schema([row], pyarrow_schema_t2v_dual_text, text_padding_length=8) + + assert "text_embedding_2" not in batch + assert "text_attention_mask_2" not in batch + assert batch["text_embedding"].shape == (1, 8, _QWEN_DIM) + + +def test_collate_mixed_legacy_and_dual_rows(tmp_path: Path) -> None: + """A legacy shard mixed into a dual-text batch must not break stacking. + + The all-legacy skip above cannot fire once one row carries the bytes; + the rows without them become zero-token streams of the batch's real + width (their masks say "no glyph text"), not a 768-wide stub that + cannot stack against the real rows. + """ + dual = _write_row(tmp_path / "data_00000.parquet", + byt5_tokens=3, + schema=pyarrow_schema_t2v_dual_text) + legacy = _write_row(tmp_path / "data_00001.parquet", + byt5_tokens=0, + schema=pyarrow_schema_t2v) + + batch = collate_rows_from_parquet_schema([dual, legacy], + pyarrow_schema_t2v_dual_text, + text_padding_length=8) + + assert batch["text_embedding_2"].shape == (2, 8, _BYT5_DIM) + torch.testing.assert_close( + batch["text_attention_mask_2"], + torch.tensor([[1.0] * 3 + [0.0] * 5, [0.0] * 8]), + ) + assert batch["text_embedding"].shape == (2, 8, _QWEN_DIM) diff --git a/fastvideo/tests/train/fixtures/hunyuan15_t2v_finetune_min.yaml b/fastvideo/tests/train/fixtures/hunyuan15_t2v_finetune_min.yaml new file mode 100644 index 000000000..ef2122b4d --- /dev/null +++ b/fastvideo/tests/train/fixtures/hunyuan15_t2v_finetune_min.yaml @@ -0,0 +1,50 @@ +# Minimum config to run a single training step of FineTuneMethod on +# Hunyuan15Model for the per-method smoke test. Uses the real +# HunyuanVideo-1.5 480p T2V checkpoint (~8B) with tiny synthetic +# latents. training_cfg_rate=0 keeps the CFG-dropout path out of this +# test (see the CFG semantics note in Hunyuan15Model), and +# precondition_outputs=true matches the flow-matching x0 objective. + +models: + student: + _target_: fastvideo.train.models.hunyuan15.Hunyuan15Model + init_from: hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v + trainable: true + flow_shift: 5.0 + +method: + _target_: fastvideo.train.methods.fine_tuning.finetune.FineTuneMethod + +training: + dit_precision: bf16 + + distributed: + num_gpus: 1 + sp_size: 1 + tp_size: 1 + + data: + seed: 42 + train_batch_size: 1 + training_cfg_rate: 0.0 + num_latent_t: 4 + num_height: 256 + num_width: 256 + num_frames: 13 + + optimizer: + learning_rate: 1.0e-6 + betas: [0.9, 0.999] + weight_decay: 0.01 + lr_scheduler: constant + lr_warmup_steps: 0 + + loop: + max_train_steps: 1 + gradient_accumulation_steps: 1 + + model: + precondition_outputs: true + +pipeline: + flow_shift: 5.0 diff --git a/fastvideo/tests/train/fixtures/hunyuan15_t2v_min.yaml b/fastvideo/tests/train/fixtures/hunyuan15_t2v_min.yaml new file mode 100644 index 000000000..3d5510cee --- /dev/null +++ b/fastvideo/tests/train/fixtures/hunyuan15_t2v_min.yaml @@ -0,0 +1,20 @@ +# Minimum config to instantiate Hunyuan15Model for the loading + forward +# smoke test. Loads the real HunyuanVideo-1.5 480p T2V checkpoint (~8B) +# via load_module_from_path. HY1.5 uses dual text encoders (Qwen2.5-VL +# 3584 + ByT5 1472) and a 32-channel VAE latent space. + +models: + student: + _target_: fastvideo.train.models.hunyuan15.Hunyuan15Model + init_from: hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v + trainable: false + flow_shift: 5.0 + +method: + _target_: fastvideo.train.methods.fine_tuning.finetune.FineTuneMethod + +training: + dit_precision: bf16 + +pipeline: + flow_shift: 5.0 diff --git a/fastvideo/tests/train/methods/grad_norm_regression.py b/fastvideo/tests/train/methods/grad_norm_regression.py index 8c4d1f771..e884bd422 100644 --- a/fastvideo/tests/train/methods/grad_norm_regression.py +++ b/fastvideo/tests/train/methods/grad_norm_regression.py @@ -84,7 +84,7 @@ def resolve_device_key(device_name: str | None = None) -> str | None: # Transformer block-list attribute names across model families. Wan / causal # Wan / Matrix-Game expose ``.blocks``; Cosmos (diffusers-derived) exposes # ``.transformer_blocks``. First non-empty match wins. -_BLOCK_LIST_ATTRS: tuple[str, ...] = ("blocks", "transformer_blocks") +_BLOCK_LIST_ATTRS: tuple[str, ...] = ("blocks", "transformer_blocks", "double_blocks") def resolve_blocks(transformer): diff --git a/fastvideo/tests/train/methods/test_hunyuan15_finetune.py b/fastvideo/tests/train/methods/test_hunyuan15_finetune.py new file mode 100644 index 000000000..199be8cd6 --- /dev/null +++ b/fastvideo/tests/train/methods/test_hunyuan15_finetune.py @@ -0,0 +1,145 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Per-method GPU smoke test: ``Hunyuan15Model`` + ``FineTuneMethod``. + +Mirrors ``test_cosmos_finetune.py`` for the HunyuanVideo 1.5 plugin. +The harness is intentionally identical so the tests are easy to compare; +the HY1.5-specific differences are in the synthetic ``raw_batch`` (dual +text embeddings -- Qwen 3584 plus ByT5 1472 -- and 32-channel latents) +and the fixture (``precondition_outputs=true``). + +The empty-ByT5 case is covered as a second parametrised run: captions +without glyph text carry a zero-token ByT5 stream, which must survive +``prepare_batch``'s trim and reach the transformer without producing +NaNs. +""" + +from __future__ import annotations + + +import fastvideo.envs as envs + +envs.setdefault_external("MASTER_ADDR", "localhost") +envs.setdefault_external("MASTER_PORT", "29523") + +from pathlib import Path + +import pytest +import torch + +from fastvideo.train.methods.fine_tuning.finetune import ( + FineTuneMethod, ) +from fastvideo.train.models.hunyuan15 import Hunyuan15Model +from fastvideo.train.utils.config import load_run_config + +from .grad_norm_regression import ( + check_grad_norm_regression, + resolve_blocks, +) + +_FIXTURE = str( + Path(__file__).resolve().parent.parent / "fixtures" + / "hunyuan15_t2v_finetune_min.yaml") + +_QWEN_DIM = 3584 +_BYT5_DIM = 1472 + + +def _build_synthetic_batch( + device: torch.device, + dtype: torch.dtype, + byt5_tokens: int, +) -> dict[str, torch.Tensor]: + """Tiny synthetic ``raw_batch`` matching ``Hunyuan15Model.prepare_batch``. + + HY1.5 stores dual text embeddings and 32-channel VAE latents. + ``byt5_tokens=0`` reproduces a caption with no glyph text. + """ + batch_size = 1 + return { + "text_embedding": + torch.randn(batch_size, 20, _QWEN_DIM, device=device, dtype=dtype), + "text_attention_mask": + torch.ones(batch_size, 20, device=device), + "text_embedding_2": + torch.randn(batch_size, + byt5_tokens, + _BYT5_DIM, + device=device, + dtype=dtype), + "text_attention_mask_2": + torch.ones(batch_size, byt5_tokens, device=device), + "vae_latent": + torch.randn(batch_size, 32, 4, 16, 16, device=device, dtype=dtype), + } + + +@pytest.mark.parametrize("byt5_tokens", [8, 0], ids=["with_byt5", "empty_byt5"]) +@pytest.mark.usefixtures("distributed_setup") +def test_hunyuan15_finetune_single_train_step( + monkeypatch: pytest.MonkeyPatch, + byt5_tokens: int, +) -> None: + if not torch.cuda.is_available(): + pytest.skip("requires CUDA") + + cfg = load_run_config(_FIXTURE) + + device = torch.device("cuda:0") + dtype = torch.bfloat16 + + # Feed a synthetic ``raw_batch`` straight into ``single_train_step``, + # so the parquet train dataloader built by ``init_preprocessors`` is + # never iterated. Stub it out so construction does not require a real + # ``training.data.data_path``. + monkeypatch.setattr( + "fastvideo.train.utils.dataloader." + "build_parquet_t2v_train_dataloader", + lambda *args, **kwargs: None, + ) + + model = Hunyuan15Model( + init_from=cfg.models["student"]["init_from"], + training_config=cfg.training, + trainable=True, + ) + model.transformer = model.transformer.to(device=device, dtype=dtype) + + method = FineTuneMethod( + cfg=cfg, + role_models={"student": model}, + ) + method.on_train_start() + + batch = _build_synthetic_batch(device, dtype, byt5_tokens) + loss_map, outputs, _metrics = method.single_train_step(batch, iteration=0) + + loss = loss_map["total_loss"] + assert torch.is_tensor(loss), "total_loss must be a torch.Tensor" + assert torch.isfinite(loss).item(), ( + f"total_loss is not finite: {loss.item()}") + + method.backward(loss_map, outputs, grad_accum_rounds=1) + + blocks = resolve_blocks(model.transformer) + assert blocks is not None and len(blocks) > 0, ( + "transformer is expected to expose a non-empty block list") + layer0 = blocks[0] + + trainable = [p for p in layer0.parameters() if p.requires_grad] + assert len(trainable) > 0, "layer 0 has no trainable parameters" + + for i, p in enumerate(trainable): + assert p.grad is not None, f"layer 0 param[{i}] has None grad" + assert torch.isfinite(p.grad).all().item(), ( + f"layer 0 param[{i}] grad contains NaN/Inf") + + any_nonzero = any( + p.grad.detach().float().norm().item() > 0.0 for p in trainable) + assert any_nonzero, ( + "all layer-0 grads are exactly zero; backward did not " + "reach the first transformer block") + + # Device-keyed grad-norm regression, same as the five sibling per-method + # tests. Skips cleanly on a device with no seeded reference, so it costs + # nothing until one exists. + check_grad_norm_regression("test_hunyuan15_finetune", model.transformer) diff --git a/fastvideo/tests/train/models/test_hunyuan15_smoke.py b/fastvideo/tests/train/models/test_hunyuan15_smoke.py new file mode 100644 index 000000000..69fdaad9d --- /dev/null +++ b/fastvideo/tests/train/models/test_hunyuan15_smoke.py @@ -0,0 +1,149 @@ +# SPDX-License-Identifier: Apache-2.0 +"""CPU smoke test for the HunyuanVideo 1.5 forward contract. + +Builds the kwargs dict that ``Hunyuan15Model._build_distill_input_kwargs`` +feeds to ``HunyuanVideo15Transformer3DModel.forward`` and checks it against +the transformer's real signature. No weights are loaded and no forward +pass runs, so this executes on CPU in well under a second -- it exists to +catch signature/shape drift between the training plugin and the inference +DiT without needing a GPU. + +Covered: + - every key we pass is a real forward parameter + - every required forward parameter is provided + - the kwargs that crash HY1.5 stay absent + - encoder_hidden_states is [qwen, byt5], including the zero-token case + - a text_dict without the ByT5 stream falls back to zero tokens + - clean_x / aug_t are rejected, since the forward takes neither + - encoder_hidden_states_image is a one-element all-zero list (T2V branch) +""" + +from __future__ import annotations + +import inspect +from types import SimpleNamespace + +import pytest +import torch + +from fastvideo.configs.models.dits.hunyuanvideo15 import ( + HunyuanVideo15ArchConfig, ) +from fastvideo.models.dits.hunyuanvideo15 import ( + HunyuanVideo15Transformer3DModel, ) +from fastvideo.train.models.hunyuan15 import Hunyuan15Model + +_QWEN_DIM = 3584 +_BYT5_DIM = 1472 +_IMAGE_TOKENS = 729 +_IMAGE_DIM = 1152 + +# Kwargs that break the HY1.5 forward: absent from its signature, or only +# valid when the arch config enables meanflow. +_FORBIDDEN = ("encoder_attention_mask", "return_dict", "timestep_r") + + +def _stub_model() -> Hunyuan15Model: + """Uninitialised instance plus the arch config the helper reads. + + No 8B checkpoint is loaded: ``_build_distill_input_kwargs`` only + needs the DiT arch config that constrains the tensors it builds. + """ + model = object.__new__(Hunyuan15Model) + model.training_config = SimpleNamespace( + pipeline_config=SimpleNamespace( + dit_config=SimpleNamespace(arch_config=HunyuanVideo15ArchConfig()))) + return model + + +def _build_kwargs(byt5_tokens: int = 12, batch_size: int = 1, with_byt5: bool = True) -> dict: + """Call the plugin helper on synthetic inputs.""" + model = _stub_model() + noise_input = torch.zeros(batch_size, 32, 4, 8, 8) + timestep = torch.full((batch_size, ), 500.0) + text_dict = { + "encoder_hidden_states": torch.zeros(batch_size, 20, _QWEN_DIM), + } + if with_byt5: + text_dict["encoder_hidden_states_2"] = torch.zeros(batch_size, byt5_tokens, _BYT5_DIM) + return model._build_distill_input_kwargs(noise_input, timestep, text_dict) + + +def test_kwargs_match_transformer_signature(): + kwargs = _build_kwargs() + params = inspect.signature(HunyuanVideo15Transformer3DModel.forward).parameters + + unknown = set(kwargs) - set(params) + assert not unknown, ("kwargs rejected by the HY1.5 forward signature: " + f"{sorted(unknown)}") + + required = { + name + for name, param in params.items() + if name != "self" and param.default is inspect.Parameter.empty and param.kind in ( + inspect.Parameter.POSITIONAL_OR_KEYWORD, + inspect.Parameter.KEYWORD_ONLY, + ) + } + missing = required - set(kwargs) + assert not missing, f"required forward params not provided: {sorted(missing)}" + + +@pytest.mark.parametrize("key", _FORBIDDEN) +def test_forbidden_kwarg_absent(key): + assert key not in _build_kwargs() + + +def test_dual_text_embeddings_are_a_two_element_list(): + encoder_hidden_states = _build_kwargs()["encoder_hidden_states"] + assert isinstance(encoder_hidden_states, list) + assert len(encoder_hidden_states) == 2 + qwen, byt5 = encoder_hidden_states + assert qwen.shape[-1] == _QWEN_DIM + assert byt5.shape[-1] == _BYT5_DIM + + +def test_zero_token_byt5_is_preserved(): + """Captions without glyph text carry a zero-length ByT5 stream.""" + byt5 = _build_kwargs(byt5_tokens=0)["encoder_hidden_states"][1] + assert byt5.shape[1] == 0, "zero tokens, not zero values" + assert byt5.shape[-1] == _BYT5_DIM + + +def test_missing_second_stream_falls_back_to_zero_tokens(): + """cfg_uncond.text="zero" hands over a dict with only the primary stream.""" + byt5 = _build_kwargs(with_byt5=False)["encoder_hidden_states"][1] + assert byt5.shape == (1, 0, _BYT5_DIM) + + +def test_clean_x_and_aug_t_are_rejected(): + """The HY1.5 forward takes neither, so they must not be dropped silently.""" + model = _stub_model() + text_dict = {"encoder_hidden_states": torch.zeros(1, 20, _QWEN_DIM)} + with pytest.raises(NotImplementedError): + model._build_distill_input_kwargs( + torch.zeros(1, 32, 4, 8, 8), + torch.zeros(1), + text_dict, + clean_x=torch.zeros(1, 32, 4, 8, 8), + ) + + +def test_t2v_image_placeholder_is_all_zero(): + kwargs = _build_kwargs(batch_size=2) + image = kwargs["encoder_hidden_states_image"] + assert isinstance(image, list) + assert len(image) == 1 + assert image[0].shape == (2, _IMAGE_TOKENS, _IMAGE_DIM) + assert torch.all(image[0] == 0), "all-zero content selects the T2V branch" + + +def test_hidden_states_are_packed_to_65_channels(): + """32 latent + 1 conditioning mask + 32 conditioning latent. + + predict_noise owns the (B,T,C,H,W) -> (B,C,T,H,W) permute; this + helper only packs the conditioning channels HY1.5's img_in expects. + """ + hidden_states = _build_kwargs()["hidden_states"] + assert hidden_states.shape == (1, 65, 4, 8, 8) + # T2V leaves both conditioning blocks zero. + assert torch.all(hidden_states[:, 32:] == 0) diff --git a/fastvideo/tests/train/models/test_load_hunyuan15.py b/fastvideo/tests/train/models/test_load_hunyuan15.py new file mode 100644 index 000000000..853a43f8f --- /dev/null +++ b/fastvideo/tests/train/models/test_load_hunyuan15.py @@ -0,0 +1,153 @@ +# SPDX-License-Identifier: Apache-2.0 +"""GPU loading + forward smoke test for ``Hunyuan15Model``. + +Loads the real HunyuanVideo-1.5 480p T2V checkpoint (~8B at bf16) via +``Hunyuan15Model.__init__`` and runs one transformer forward pass on +synthetic inputs. Catches loader or forward-signature regressions in +``fastvideo.train.models.hunyuan15.Hunyuan15Model`` and the underlying +``HunyuanVideo15Transformer3DModel``. + +HY1.5's forward differs from Wan's: ``encoder_hidden_states`` is a list +of two tensors (Qwen 3584 + ByT5 1472), ``encoder_hidden_states_image`` +is a one-element list whose all-zero content selects the T2V branch, and +there is no ``encoder_attention_mask`` / ``return_dict``. This mirrors +the kwargs in ``Hunyuan15Model._build_distill_input_kwargs``. +""" + +from __future__ import annotations + + +# Required by the ``distributed_setup`` fixture pulled from +# ``fastvideo/tests/conftest.py``. Set before any fastvideo import. +import fastvideo.envs as envs + +envs.setdefault_external("MASTER_ADDR", "localhost") +envs.setdefault_external("MASTER_PORT", "29522") + +from pathlib import Path + +import pytest +import torch + +from fastvideo.forward_context import set_forward_context +from fastvideo.train.models.hunyuan15 import Hunyuan15Model +from fastvideo.train.utils.config import load_run_config + +_FIXTURE = str( + Path(__file__).resolve().parent.parent / "fixtures" + / "hunyuan15_t2v_min.yaml") + +_QWEN_DIM = 3584 +_BYT5_DIM = 1472 +_IMAGE_TOKENS = 729 +_IMAGE_DIM = 1152 + + +@pytest.mark.usefixtures("distributed_setup") +def test_hunyuan15_model_loads_and_forwards(): + if not torch.cuda.is_available(): + pytest.skip("requires CUDA") + + cfg = load_run_config(_FIXTURE) + model = Hunyuan15Model( + init_from=cfg.models["student"]["init_from"], + training_config=cfg.training, + trainable=False, + ) + + transformer = model.transformer + assert isinstance(transformer, torch.nn.Module) + assert sum(p.numel() for p in transformer.parameters()) > 0 + + device = torch.device("cuda:0") + dtype = torch.bfloat16 + transformer = transformer.to(device=device, dtype=dtype).eval() + + # HY1.5 takes [B, C, T, H, W] with 32 latent channels. Small spatial + # + few frames so this fits next to the 8B model. + b, t, h, w = 1, 4, 16, 16 + latent = torch.randn(b, 32, t, h, w, device=device, dtype=dtype) + # 65 = 32 latent + 1 conditioning mask + 32 conditioning latent + # (both conditioning blocks stay zero for T2V). + cond_mask = torch.zeros(b, 1, t, h, w, device=device, dtype=dtype) + hidden_states = torch.cat([latent, cond_mask, torch.zeros_like(latent)], dim=1) + + qwen_embeds = torch.randn(b, 20, _QWEN_DIM, device=device, dtype=dtype) + byt5_embeds = torch.randn(b, 8, _BYT5_DIM, device=device, dtype=dtype) + timestep = torch.tensor([500], device=device, dtype=torch.long) + # All-zero image embeddings select the T2V branch. + zero_image_embeds = torch.zeros(b, + _IMAGE_TOKENS, + _IMAGE_DIM, + device=device, + dtype=dtype) + + with torch.no_grad(), set_forward_context( + current_timestep=0, + attn_metadata=None, + ): + out = transformer( + hidden_states=hidden_states, + encoder_hidden_states=[qwen_embeds, byt5_embeds], + timestep=timestep, + encoder_hidden_states_image=[zero_image_embeds], + ) + + if isinstance(out, tuple): + out = out[0] + # The transformer emits the 32 latent channels, not the 65 it takes in. + assert out.shape == latent.shape, ( + f"output shape {tuple(out.shape)} != latent shape " + f"{tuple(latent.shape)}") + assert torch.isfinite(out).all().item(), "output contains NaN/Inf" + + +@pytest.mark.usefixtures("distributed_setup") +def test_hunyuan15_forwards_with_empty_byt5(): + """Captions without glyph text carry a zero-length ByT5 stream.""" + if not torch.cuda.is_available(): + pytest.skip("requires CUDA") + + cfg = load_run_config(_FIXTURE) + model = Hunyuan15Model( + init_from=cfg.models["student"]["init_from"], + training_config=cfg.training, + trainable=False, + ) + + device = torch.device("cuda:0") + dtype = torch.bfloat16 + transformer = model.transformer.to(device=device, dtype=dtype).eval() + + b, t, h, w = 1, 4, 16, 16 + latent = torch.randn(b, 32, t, h, w, device=device, dtype=dtype) + # 65 = 32 latent + 1 conditioning mask + 32 conditioning latent + # (both conditioning blocks stay zero for T2V). + cond_mask = torch.zeros(b, 1, t, h, w, device=device, dtype=dtype) + hidden_states = torch.cat([latent, cond_mask, torch.zeros_like(latent)], dim=1) + qwen_embeds = torch.randn(b, 20, _QWEN_DIM, device=device, dtype=dtype) + # Zero tokens, not zero values. + byt5_embeds = torch.zeros(b, 0, _BYT5_DIM, device=device, dtype=dtype) + timestep = torch.tensor([500], device=device, dtype=torch.long) + zero_image_embeds = torch.zeros(b, + _IMAGE_TOKENS, + _IMAGE_DIM, + device=device, + dtype=dtype) + + with torch.no_grad(), set_forward_context( + current_timestep=0, + attn_metadata=None, + ): + out = transformer( + hidden_states=hidden_states, + encoder_hidden_states=[qwen_embeds, byt5_embeds], + timestep=timestep, + encoder_hidden_states_image=[zero_image_embeds], + ) + + if isinstance(out, tuple): + out = out[0] + assert out.shape == latent.shape + assert torch.isfinite(out).all().item(), ( + "empty ByT5 produced NaN/Inf in the output") diff --git a/fastvideo/tests/train/models/test_wan_override_contract.py b/fastvideo/tests/train/models/test_wan_override_contract.py index fc391c6a5..30dd7759a 100644 --- a/fastvideo/tests/train/models/test_wan_override_contract.py +++ b/fastvideo/tests/train/models/test_wan_override_contract.py @@ -6,11 +6,12 @@ import inspect import pytest from fastvideo.train.models.hunyuan.hunyuan import HunyuanModel +from fastvideo.train.models.hunyuan15.hunyuan15 import Hunyuan15Model from fastvideo.train.models.longcat.longcat import LongCatModel from fastvideo.train.models.matrixgame2.matrixgame2 import MatrixGame2Model -@pytest.mark.parametrize("model_cls", [HunyuanModel, LongCatModel, MatrixGame2Model]) +@pytest.mark.parametrize("model_cls", [HunyuanModel, Hunyuan15Model, LongCatModel, MatrixGame2Model]) def test_distill_kwargs_accept_wan_teacher_forcing_keywords(model_cls: type) -> None: parameters = inspect.signature(model_cls._build_distill_input_kwargs).parameters @@ -18,7 +19,7 @@ def test_distill_kwargs_accept_wan_teacher_forcing_keywords(model_cls: type) -> assert "aug_t" in parameters -@pytest.mark.parametrize("model_cls", [LongCatModel, MatrixGame2Model]) +@pytest.mark.parametrize("model_cls", [Hunyuan15Model, LongCatModel, MatrixGame2Model]) def test_predict_noise_override_preserves_wan_teacher_forcing_keywords(model_cls: type) -> None: parameters = inspect.signature(model_cls.predict_noise).parameters diff --git a/fastvideo/train/models/hunyuan15/__init__.py b/fastvideo/train/models/hunyuan15/__init__.py new file mode 100644 index 000000000..f73547853 --- /dev/null +++ b/fastvideo/train/models/hunyuan15/__init__.py @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: Apache-2.0 +"""HunyuanVideo 1.5 model plugin package.""" + +from fastvideo.train.models.hunyuan15.hunyuan15 import ( + Hunyuan15Model as Hunyuan15Model, ) \ No newline at end of file diff --git a/fastvideo/train/models/hunyuan15/hunyuan15.py b/fastvideo/train/models/hunyuan15/hunyuan15.py new file mode 100644 index 000000000..801d7a7bd --- /dev/null +++ b/fastvideo/train/models/hunyuan15/hunyuan15.py @@ -0,0 +1,581 @@ +# SPDX-License-Identifier: Apache-2.0 +"""HunyuanVideo 1.5 model plugin (per-role instance). + +Subclasses WanModel since HunyuanVideo 1.5 uses the same +FlowMatchEulerDiscreteScheduler and linear-interpolation noise +schedule. Differences: + - transformer class name: HunyuanVideo15Transformer3DModel + - normalize_dit_input("hunyuan15", ...): latents * scaling_factor + - dual text encoders: encoder_hidden_states is [qwen, byt5] + - forward kwargs: no encoder_attention_mask / return_dict; an + all-zero encoder_hidden_states_image selects the T2V branch + - hidden_states in (B, C, T, H, W) + - discrete timestep on the 0..1000 scale + - default flow_shift = 5.0 (480p T2V) +""" + +from __future__ import annotations + +from typing import Any, Literal, TYPE_CHECKING + +import os +import copy +import torch + +from fastvideo.train.models.wan.wan import WanModel +from fastvideo.forward_context import set_forward_context +from fastvideo.logger import init_logger +from fastvideo.pipelines import TrainingBatch +from fastvideo.training.training_utils import ( + normalize_dit_input, ) + +logger = init_logger(__name__) + +if TYPE_CHECKING: + from fastvideo.train.utils.lora import LoraConfig + from fastvideo.train.utils.training_config import ( + TrainingConfig, ) + + +class Hunyuan15Model(WanModel): + """HunyuanVideo 1.5 per-role model. + + Inherits most behaviour from WanModel (noise scheduler, + timestep sampling, attention metadata, backward). Overrides + only the pieces that differ for HunyuanVideo 1.5. + """ + + _transformer_cls_name: str = "HunyuanVideo15Transformer3DModel" + + def __init__( + self, + *, + init_from: str, + training_config: TrainingConfig, + trainable: bool = True, + disable_custom_init_weights: bool = False, + flow_shift: float = 5.0, + enable_gradient_checkpointing_type: str | None = None, + transformer_override_safetensor: str | None = None, + lora: LoraConfig | dict[str, Any] | None = None, + ) -> None: + super().__init__( + init_from=init_from, + training_config=training_config, + trainable=trainable, + disable_custom_init_weights=(disable_custom_init_weights), + flow_shift=flow_shift, + enable_gradient_checkpointing_type=(enable_gradient_checkpointing_type), + transformer_override_safetensor=(transformer_override_safetensor), + lora=lora, + ) + self.negative_prompt_embeds_2: torch.Tensor | None = None + # Warn once per model, not once per step: the condition is a + # property of the dataset, so repeating it every iteration would + # bury the log it is meant to surface. + self._warned_missing_byt5 = False + # No negative-prompt cache: FineTuneMethod hard-codes + # conditional=True, so encoding the negative prompt would load the + # ~16GB Qwen encoder on every rank for an embedding nothing reads. + # DMD2 re-enables this from its own __init__ when its cfg_uncond + # policy needs negative prompts. Mirrors ltx2.py, which opts out the + # same way rather than load Gemma. + self.set_requires_negative_conditioning(False) + + def init_preprocessors(self, training_config: TrainingConfig) -> None: + """Fix the dataloader text padding length before the base + class builds the dataloader. + + Qwen2_5_VLArchConfig leaves ``text_len`` at the base-config + default of 0, which would size the parquet dataloader's text + padding to zero tokens. The real padded length is the Qwen + tokenizer max length minus the cropped template tokens + (1108 - 108 = 1000), matching the preprocess output. + """ + pipeline_config = training_config.pipeline_config + assert pipeline_config is not None + text_len = int(pipeline_config.text_encoder_max_lengths[0] - pipeline_config.text_encoder_crop_start) + pipeline_config.text_encoder_configs[0].arch_config.text_len = (text_len) + super().init_preprocessors(training_config) + + def _t2v_parquet_schema(self) -> Any: + """HY1.5 rows carry a ByT5 triplet next to the Qwen one. + + The base schema declares neither ``text_embedding_2`` nor + ``text_attention_mask_2``, and the collator only emits the tensor + fields its schema names, so inheriting it drops the glyph stream + before ``prepare_batch`` ever sees it. + """ + from fastvideo.dataset.dataloader.schema import ( + pyarrow_schema_t2v_dual_text, ) + return pyarrow_schema_t2v_dual_text + + @torch.no_grad() + def decode_latents( + self, + latents_b_t_c_h_w: torch.Tensor, + ) -> torch.Tensor: + """Decode HY1.5 latents back to pixels. + + The Wan path denormalises with per-channel ``latents_mean`` / + ``latents_std``, which ``AutoencoderKLHunyuanVideo15`` does not + expose: it normalises by a single scaling factor instead, so undo + that rather than inheriting the Wan implementation. + """ + if self.vae is None: + raise RuntimeError("HunyuanVideo 1.5 VAE is not initialized") + latents = latents_b_t_c_h_w.permute(0, 2, 1, 3, 4).float() + if bool(getattr(self.vae, "handles_latent_denorm", False)): + denorm = latents + else: + denorm = latents / self.vae.scaling_factor + # The 480p config builds the VAE under fp16 (vae_precision="fp16"), + # so hand the decoder its own parameter dtype instead of fp32. + vae_dtype = next(self.vae.parameters()).dtype + media = self.vae.to(latents.device).decode(denorm.to(vae_dtype)) + return (media / 2 + 0.5).clamp(0, 1) + + def _arch_config(self) -> Any: + """The DiT arch config that constrains this model's tensors.""" + tc = self.training_config + assert tc is not None + return tc.pipeline_config.dit_config.arch_config # type: ignore[union-attr] + + def _byt5_embed_dim(self) -> int: + """Width of the ByT5 stream, from the config that actually constrains it. + + ``HunyuanVideo15ArchConfig.text_embed_2_dim`` is what builds the + ``nn.LayerNorm`` this tensor has to satisfy, so it is the authority. The + encoder-side ``T5ArchConfig`` default of 512 is unrelated to it. + """ + return int(self._arch_config().text_embed_2_dim) + + def prepare_batch( + self, + raw_batch: dict[str, Any], + *, + generator: torch.Generator, + latents_source: Literal["data", "zeros"] = "data", + ) -> TrainingBatch: + """Same flow as Wan, with three HY1.5 differences: dual text + embeddings (Qwen + ByT5), trim-to-longest padding removal (the + DiT has no text-mask input) and "hunyuan15" VAE scaling.""" + if self._requires_negative_conditioning: + self.ensure_negative_conditioning() + assert self.training_config is not None + tc = self.training_config + + dtype = self._get_training_dtype() + device = self.device + + training_batch = TrainingBatch() + encoder_hidden_states = raw_batch["text_embedding"] + encoder_attention_mask = raw_batch["text_attention_mask"] + infos = raw_batch.get("info_list") + batch_size = encoder_hidden_states.shape[0] + + # --- dual text embeddings ------------------------------------- + encoder_hidden_states_2 = raw_batch.get("text_embedding_2") + encoder_attention_mask_2 = raw_batch.get("text_attention_mask_2") + if (encoder_hidden_states_2 is None or encoder_attention_mask_2 is None): + # Legacy parquet without the ByT5 field: zero-token + # embedding, matching the inference empty-ByT5 convention. + # + # Say so. This is indistinguishable from healthy training -- finite + # grads, falling loss, full speed -- so without a line in the log a + # run conditioned on nothing looks exactly like a run that works. + # It also fires when the dataset's schema simply lacks the columns, + # because the collator derives its tensor fields from the schema. + if not self._warned_missing_byt5: + self._warned_missing_byt5 = True + logger.warning( + "No ByT5 embeddings in this batch (text_embedding_2=%s, " + "text_attention_mask_2=%s); training with glyph " + "conditioning DISABLED. Expected for data preprocessed " + "without the secondary text encoder; if you did " + "preprocess it, check that the dataset schema declares " + "the text_embedding_2 columns.", + "missing" if encoder_hidden_states_2 is None else "present", + "missing" if encoder_attention_mask_2 is None else "present", + ) + encoder_hidden_states_2 = torch.zeros( + batch_size, + 0, + self._byt5_embed_dim(), + dtype=encoder_hidden_states.dtype, + ) + encoder_attention_mask_2 = torch.zeros( + batch_size, + 0, + dtype=encoder_attention_mask.dtype, + ) + + # --- trim padding to the longest valid length in the batch ---- + # The HY1.5 DiT has no text-mask input; leftover padding would + # dilute the token refiner's mean-pool. With B=1 this equals the + # exact-length behaviour of the inference path. + prompt_lengths = encoder_attention_mask.sum(dim=1) + max_len = int(prompt_lengths.max().item()) + min_len = int(prompt_lengths.min().item()) + if min_len != max_len: + logger.warning( + "Batch mixes prompt lengths (%s..%s tokens). HY1.5's DiT takes " + "no text mask and mean-pools every token it is given, so the " + "shorter prompts keep padding and their conditioning depends " + "on the longest prompt in the batch. Use train_batch_size=1 " + "or bucket samples by prompt length.", + min_len, + max_len, + ) + encoder_hidden_states = encoder_hidden_states[:, :max_len] + encoder_attention_mask = encoder_attention_mask[:, :max_len] + max_len_2 = int(encoder_attention_mask_2.sum(dim=1).max().item()) + encoder_hidden_states_2 = encoder_hidden_states_2[:, :max_len_2] + encoder_attention_mask_2 = encoder_attention_mask_2[:, :max_len_2] + + # --- latents --------------------------------------------------- + if latents_source == "zeros": + vae_config = ( + tc.pipeline_config.vae_config # type: ignore[union-attr] + .arch_config) + num_channels = getattr(vae_config, "latent_channels", 32) + spatial_compression_ratio = (vae_config.spatial_compression_ratio) + latent_height = (tc.data.num_height // spatial_compression_ratio) + latent_width = (tc.data.num_width // spatial_compression_ratio) + latents = torch.zeros( + batch_size, + num_channels, + tc.data.num_latent_t, + latent_height, + latent_width, + device=device, + dtype=dtype, + ) + elif latents_source == "data": + if "vae_latent" not in raw_batch: + raise ValueError("vae_latent not found in batch " + "and latents_source='data'") + latents = raw_batch["vae_latent"] + latents = latents[:, :, :tc.data.num_latent_t] + latents = latents.to(device, dtype=dtype) + else: + raise ValueError(f"Unknown latents_source: " + f"{latents_source!r}") + + training_batch.latents = latents + training_batch.encoder_hidden_states = (encoder_hidden_states.to(device, dtype=dtype)) + training_batch.encoder_attention_mask = (encoder_attention_mask.to(device, dtype=dtype)) + training_batch.infos = infos + + # KEY DIFFERENCE: "hunyuan15" normalisation (latents * 1.03682; + # parquet stores raw latent_dist.mode() outputs). + training_batch.latents = normalize_dit_input( + "hunyuan15", + training_batch.latents, + self.vae, + ) + training_batch = self._prepare_dit_inputs(training_batch, generator) + + # The base class only knows the primary text stream; attach the + # ByT5 stream to the dicts consumed by _build_distill_input_kwargs. + assert training_batch.conditional_dict is not None + training_batch.conditional_dict["encoder_hidden_states_2"] = (encoder_hidden_states_2.to(device, dtype=dtype)) + if (training_batch.unconditional_dict is not None and self.negative_prompt_embeds_2 is not None): + neg_embeds_2 = self.negative_prompt_embeds_2 + if neg_embeds_2.shape[0] == 1 and batch_size > 1: + neg_embeds_2 = neg_embeds_2.expand(batch_size, *neg_embeds_2.shape[1:]).contiguous() + training_batch.unconditional_dict["encoder_hidden_states_2"] = neg_embeds_2 + + training_batch = self._build_attention_metadata(training_batch) + + # Shallow copy keeps the lru_cache'd LongTensor index fields shared + # with the original metadata; only the float ``VSA_sparsity`` differs + # between the two views. + training_batch.attn_metadata_vsa = copy.copy(training_batch.attn_metadata) + if training_batch.attn_metadata is not None: + training_batch.attn_metadata.VSA_sparsity = 0.0 # type: ignore[attr-defined] + + return training_batch + + def _build_distill_input_kwargs( + self, + noise_input: torch.Tensor, + timestep: torch.Tensor, + text_dict: dict[str, torch.Tensor] | None, + clean_x: torch.Tensor | None = None, + aug_t: torch.Tensor | None = None, + ) -> dict[str, Any]: + """Build transformer forward kwargs for HunyuanVideo 1.5. + + The HY1.5 transformer expects: + - hidden_states in (B, C, T, H, W) — predict_noise permutes + before calling this helper + - encoder_hidden_states as a list of two tensors + [qwen, byt5]; a zero-token ByT5 tensor (B, 0, 1472) is valid + - timestep on the discrete 0..1000 scale + - encoder_hidden_states_image as a one-element list holding an + all-zero tensor — the all-zero content selects the T2V + branch (is_t2v check); passing None crashes + - no encoder_attention_mask / return_dict / guidance kwargs; + timestep_r must stay unset while use_meanflow=False + + ``clean_x`` / ``aug_t`` are accepted for signature parity with + the base class but HunyuanVideo15Transformer3DModel.forward + takes neither, so supplying them is an error rather than a + silent drop. + """ + if clean_x is not None or aug_t is not None: + raise NotImplementedError("Hunyuan15Model does not support " + "clean-history teacher forcing") + if text_dict is None: + raise ValueError("text_dict cannot be None for " + "HunyuanVideo 1.5 forward pass") + + arch = self._arch_config() + batch_size = noise_input.shape[0] + # HY1.5's img_in takes 65 channels: the 32 latent channels plus a + # 1-channel conditioning mask and a 32-channel conditioning latent. + # T2V leaves both conditioning blocks zero, mirroring + # Hy15ImageEncodingStage + DenoisingStage on the inference path. + cond_mask = torch.zeros( + batch_size, + 1, + *noise_input.shape[2:], + device=noise_input.device, + dtype=noise_input.dtype, + ) + cond_latent = torch.zeros_like(noise_input) + hidden_states = torch.cat([noise_input, cond_mask, cond_latent], dim=1) + assert hidden_states.shape[1] == int(arch.in_channels), ( + f"packed {hidden_states.shape[1]} conditioning channels but " + f"img_in expects {int(arch.in_channels)}") + + zero_image_embeds = torch.zeros( + batch_size, + 729, + int(arch.image_embed_dim), + device=noise_input.device, + dtype=noise_input.dtype, + ) + # cfg_uncond.text="zero" synthesises a fresh dict carrying only the + # primary stream; keep the inference empty-ByT5 convention. + encoder_hidden_states_2 = text_dict.get("encoder_hidden_states_2") + if encoder_hidden_states_2 is None: + encoder_hidden_states_2 = torch.zeros( + batch_size, + 0, + self._byt5_embed_dim(), + device=noise_input.device, + dtype=noise_input.dtype, + ) + return { + "hidden_states": hidden_states, + "encoder_hidden_states": [ + text_dict["encoder_hidden_states"], + encoder_hidden_states_2, + ], + "timestep": timestep, + "encoder_hidden_states_image": [zero_image_embeds], + } + + def _forward( + self, + noisy_latents: torch.Tensor, + timestep: torch.Tensor, + batch: TrainingBatch, + *, + conditional: bool, + cfg_uncond: dict[str, Any] | None, + attn_kind: Literal["dense", "vsa"], + r_timestep: torch.Tensor | None = None, + clean_x: torch.Tensor | None = None, + aug_t: torch.Tensor | None = None, + ) -> torch.Tensor: + device_type = self.device.type + dtype = self._get_training_dtype() + if conditional: + text_dict = batch.conditional_dict + if text_dict is None: + raise RuntimeError("Missing conditional_dict in " + "TrainingBatch") + else: + text_dict = self._get_uncond_text_dict(batch, cfg_uncond=cfg_uncond) + + if attn_kind == "dense": + attn_metadata = batch.attn_metadata + elif attn_kind == "vsa": + attn_metadata = batch.attn_metadata_vsa + else: + raise ValueError(f"Unknown attn_kind: {attn_kind!r}") + + # The method layer hands noisy_latents in (B, T, C, H, W) (Wan + # canonical); HY1.5's transformer expects (B, C, T, H, W). + noisy_latents = noisy_latents.permute(0, 2, 1, 3, 4) + if noisy_latents.is_floating_point(): + noisy_latents = noisy_latents.to(dtype=dtype) + + with torch.autocast(device_type, dtype=dtype), set_forward_context( + current_timestep=batch.timesteps, + attn_metadata=attn_metadata, + ): + input_kwargs = self._build_distill_input_kwargs( + noisy_latents, + timestep, + text_dict, + clean_x=clean_x, + aug_t=aug_t, + ) + if r_timestep is not None: + # HY1.5 spells the reference timestep ``timestep_r``. + input_kwargs["timestep_r"] = r_timestep + transformer = self._get_transformer(timestep) + model_output = transformer(**input_kwargs) + + # (B, C, T, H, W) → (B, T, C, H, W) back to the method layer. + return model_output.permute(0, 2, 1, 3, 4) + + def predict_noise( + self, + noisy_latents: torch.Tensor, + timestep: torch.Tensor, + batch: TrainingBatch, + *, + conditional: bool, + cfg_uncond: dict[str, Any] | None = None, + attn_kind: Literal["dense", "vsa"] = "dense", + clean_x: torch.Tensor | None = None, + aug_t: torch.Tensor | None = None, + ) -> torch.Tensor: + return self._forward( + noisy_latents, + timestep, + batch, + conditional=conditional, + cfg_uncond=cfg_uncond, + attn_kind=attn_kind, + clean_x=clean_x, + aug_t=aug_t, + ) + + def predict_velocity_with_r( + self, + noisy_latents: torch.Tensor, + timestep: torch.Tensor, + r_timestep: torch.Tensor, + batch: TrainingBatch, + *, + conditional: bool, + cfg_uncond: dict[str, Any] | None = None, + attn_kind: Literal["dense", "vsa"] = "dense", + ) -> torch.Tensor: + """AnyFlow forward: average velocity from ``t`` back to ``r``. + + Same plumbing as :meth:`predict_noise`, but HY1.5's dual-timestep + branch only exists when the arch config sets ``use_meanflow``: + ``HunyuanVideo15TimeEmbedding`` builds ``timestep_embedder_r`` + conditionally, so a reference timestep would otherwise be dropped. + """ + if not bool(getattr(self._arch_config(), "use_meanflow", False)): + raise ValueError( + "Hunyuan15Model.predict_velocity_with_r needs " + "dit_config.arch_config.use_meanflow=True: " + "HunyuanVideo15TimeEmbedding only builds " + "timestep_embedder_r when use_meanflow is set.") + return self._forward( + noisy_latents, + timestep, + batch, + conditional=conditional, + cfg_uncond=cfg_uncond, + attn_kind=attn_kind, + r_timestep=r_timestep, + ) + + def ensure_negative_conditioning(self) -> None: + """Encode the HY1.5 negative prompt with the Qwen encoder. + + The shared ``encode_negative_prompt`` helper does not fit + HY1.5: its Qwen preprocess returns a chat message list (which + needs ``apply_chat_template``), its postprocess takes the + attention mask as a second argument and returns a tuple, and + it reads ``hidden_states[-3]`` so the encoder must run with + ``output_hidden_states=True``. Mirror the preprocess script's + ``encode_qwen_caption`` instead, so training and the parquet + data are produced by the same code path. + + Every rank encodes independently to avoid NCCL deadlocks when + only a subset of ranks would otherwise participate. + """ + if self.negative_prompt_embeds is not None: # type: ignore[has-type] + return + assert self.training_config is not None + tc = self.training_config + pipeline_config = tc.pipeline_config + assert pipeline_config is not None + device = self.device + dtype = self._get_training_dtype() + + from transformers import AutoTokenizer, Qwen2_5_VLTextModel + + from fastvideo.configs.pipelines.hunyuan15 import ( + qwen_postprocess_text, + qwen_preprocess_text, + ) + from fastvideo.utils import maybe_download_model + + model_path = maybe_download_model(tc.model_path) + max_length = int(pipeline_config.text_encoder_max_lengths[0]) + + tokenizer = AutoTokenizer.from_pretrained(os.path.join(model_path, "tokenizer")) + encoder = Qwen2_5_VLTextModel.from_pretrained( + os.path.join(model_path, "text_encoder"), + torch_dtype=dtype, + ).to(device).eval() + + # HY1.5 encodes an empty negative prompt as "." rather than the + # empty string (treat_empty_as_dot on the inference path). + messages = qwen_preprocess_text(".") + + with torch.no_grad(): + tokens = tokenizer.apply_chat_template( + messages, + add_generation_prompt=True, + tokenize=True, + return_dict=True, + return_tensors="pt", + max_length=max_length, + truncation=True, + ) + input_ids = tokens["input_ids"].to(device) + attention_mask = tokens.get("attention_mask") + if attention_mask is None: + attention_mask = torch.ones_like(input_ids) + attention_mask = attention_mask.to(device) + + outputs = encoder( + input_ids=input_ids, + attention_mask=attention_mask, + output_hidden_states=True, + return_dict=True, + ) + neg_embeds, neg_mask = qwen_postprocess_text(outputs, attention_mask) + + # Trim to the valid length: the DiT has no text-mask input, + # matching the trim done in prepare_batch. + valid_len = int(neg_mask.sum().item()) + neg_embeds = neg_embeds[:, :valid_len] + neg_mask = neg_mask[:, :valid_len] + + del encoder, tokenizer + + self.negative_prompt_embeds = neg_embeds.to(device=device, dtype=dtype) + self.negative_prompt_attention_mask = (neg_mask.to(device=device, dtype=dtype)) + # The negative prompt carries no glyph text: inference uses a + # zero-length ByT5 tensor, not an encoded empty string. + self.negative_prompt_embeds_2 = torch.zeros( + 1, + 0, + self._byt5_embed_dim(), + device=device, + dtype=dtype, + ) diff --git a/fastvideo/train/models/wan/wan.py b/fastvideo/train/models/wan/wan.py index aaf74e98b..40074b115 100644 --- a/fastvideo/train/models/wan/wan.py +++ b/fastvideo/train/models/wan/wan.py @@ -154,6 +154,16 @@ class WanModel(ModelBase): # Lifecycle # ------------------------------------------------------------------ + def _t2v_parquet_schema(self) -> Any: + """Parquet schema for the t2v rows this model trains on. + + Models whose rows carry extra tensor fields (e.g. HunyuanVideo + 1.5's second text stream) override this. + """ + from fastvideo.dataset.dataloader.schema import ( + pyarrow_schema_t2v, ) + return pyarrow_schema_t2v + def init_preprocessors(self, training_config: TrainingConfig) -> None: self.vae = load_module_from_path( model_path=str(training_config.model_path), @@ -167,9 +177,7 @@ class WanModel(ModelBase): self._init_timestep_mechanics() from fastvideo.dataset.dataloader.schema import ( - pyarrow_schema_t2v, - pyarrow_schema_text_only, - ) + pyarrow_schema_text_only, ) from fastvideo.train.utils.dataloader import ( build_parquet_t2v_train_dataloader, ) @@ -178,7 +186,7 @@ class WanModel(ModelBase): "preprocessed_data_type", "t2v", )).strip().lower() - parquet_schema = pyarrow_schema_t2v + parquet_schema = self._t2v_parquet_schema() if preprocessed_data_type == "text_only": parquet_schema = pyarrow_schema_text_only elif preprocessed_data_type != "t2v": diff --git a/fastvideo/training/training_utils.py b/fastvideo/training/training_utils.py index a026435de..9d8e22bc9 100644 --- a/fastvideo/training/training_utils.py +++ b/fastvideo/training/training_utils.py @@ -797,6 +797,10 @@ def load_distillation_checkpoint( def normalize_dit_input(model_type, latents, vae) -> torch.Tensor: if model_type == "hunyuan_hf" or model_type == "hunyuan": return latents * vae.config.scaling_factor + elif model_type == "hunyuan15": + # Parquet latents are raw latent_dist.mode() outputs; inference + # divides by scaling_factor before decode, so training multiplies. + return latents * vae.scaling_factor elif model_type == "wan": latents_mean = torch.tensor(vae.latents_mean) latents_std = 1.0 / torch.tensor(vae.latents_std) diff --git a/mkdocs.yml b/mkdocs.yml index 53ed86a51..2a17393ce 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -205,6 +205,7 @@ nav: - Data Preprocessing: training/data_preprocess.md - Fine-tuning: training/finetune.md - Attn-QAT Training: training/attn_qat.md + - HunyuanVideo 1.5 T2V: training/hunyuan15_overfit.md - Examples: - Training Index: training/examples/examples_training_index.md - Wan T2V 1.3B: training/examples/wan_t2v_1.3B.md