tests

update
fix rebase
2025-09-10 01:57:42 +00:00 · 2025-09-10 01:52:21 +00:00 · 2025-09-09 23:30:19 +00:00 · 2025-09-09 23:29:37 +00:00 · 2025-09-09 23:29:37 +00:00 · 2025-09-09 23:29:37 +00:00
136 changed files with 395 additions and 4144 deletions
@@ -18,8 +18,7 @@ steps:
            - path: 
                - "fastvideo/models/encoders/**"
                - "fastvideo/models/loader/**"
-                - "tests/encoders/**"
-                - "tests/utils.py"
+                - "fastvideo/tests/encoders/**"
                - "pyproject.toml"
                - "docker/Dockerfile.python3.12"
              config:
@@ -32,8 +31,7 @@ steps:
            - path:
                - "fastvideo/models/vaes/**"
                - "fastvideo/models/loader/**"
-                - "tests/vaes/**"
-                - "tests/utils.py"
+                - "fastvideo/tests/vaes/**"
                - "pyproject.toml"
                - "docker/Dockerfile.python3.12"
              config:
@@ -46,8 +44,7 @@ steps:
            - path:
                - "fastvideo/models/dits/**"
                - "fastvideo/models/loader/**"
-                - "tests/transformers/**"
-                - "tests/utils.py"
+                - "fastvideo/tests/transformers/**"
                - "fastvideo/layers/**"
                - "fastvideo/attention/**"
                - "pyproject.toml"
@@ -61,8 +58,6 @@ steps:
                  queue: "default"
            - path: 
                - "fastvideo/**/*.py"
-                - "tests/ssim/**"
-                - "tests/utils.py"
                - "pyproject.toml"
                - "docker/Dockerfile.python3.12"
              config:
@@ -73,10 +68,9 @@ steps:
                agents:
                  queue: "default"
            - path:
-                - "tests/inference/lora/**"
+                - "fastvideo/tests/lora/**"
                - "fastvideo/models/loader/**"
-                - "tests/transformers/**"
-                - "tests/utils.py"
+                - "fastvideo/tests/transformers/**"
                - "fastvideo/pipelines/**"
                - "fastvideo/layers/lora/**"
                - "pyproject.toml"
@@ -90,7 +84,6 @@ steps:
                  queue: "default"
            - path:
                - "fastvideo/**"
-                - "tests/training/Vanilla/**"
                - "pyproject.toml"
                - "docker/Dockerfile.python3.12"
              config:
@@ -102,7 +95,6 @@ steps:
                  queue: "default"
            - path:
                - "fastvideo/training/*distillation_pipeline.py"
-                - "tests/training/distill/**"
                - "pyproject.toml"
                - "docker/Dockerfile.python3.12"
              config:
@@ -114,7 +106,6 @@ steps:
                  queue: "default"
            - path:
                - "fastvideo/**"
-                - "tests/training/lora/**"
                - "pyproject.toml"
                - "docker/Dockerfile.python3.12"
              config:
@@ -126,7 +117,6 @@ steps:
                  queue: "default"
            - path:
                - "fastvideo/**"
-                - "tests/training/VSA/**"
                - "csrc/attn/video_sparse_attn/**"
                - "csrc/attn/video_sparse_attn/tk/**"
                - "csrc/attn/video_sparse_attn/setup.py"
@@ -143,7 +133,6 @@ steps:
                  queue: "default"
            - path:
                - "fastvideo/**"
-                - "tests/inference/STA/**"
                - "csrc/attn/sliding_tile_attn/**"
                - "csrc/attn/sliding_tile_attn/setup.py"
                - "csrc/attn/sliding_tile_attn/config_sta.py"
@@ -201,7 +190,6 @@ steps:
            - path:
                - "csrc/attn/vmoba_attn/vmoba/**"
                - "fastvideo/attention/backends/vmoba.py"
-                - "tests/inference/vmoba/**"
                - "pyproject.toml"
                - "docker/Dockerfile.python3.12"
              config:
@@ -210,17 +198,4 @@ steps:
                env:  
                  - TEST_TYPE=inference_vmoba
                agents:
-                  queue: "default"
-            - path:
-                - "fastvideo/**"
-                - "tests/dataset/**"
-                - "tests/workflow/**"
-                - "pyproject.toml"
-                - "docker/Dockerfile.python3.12"
-              config:
-                command: "timeout 15m .buildkite/scripts/pr_test.sh"
-                label: "Unit Tests"
-                env:
-                  - TEST_TYPE=unit_test
-                agents:
-                  queue: "default"
+                  queue: "default"
@@ -50,7 +50,7 @@ else
    exit 1
 fi

-MODAL_TEST_FILE="tests/modal/pr_test.py"
+MODAL_TEST_FILE="fastvideo/tests/modal/pr_test.py"

 if [ -z "${TEST_TYPE:-}" ]; then
    log "Error: TEST_TYPE environment variable is not set"
@@ -118,10 +118,6 @@ case "$TEST_TYPE" in
        log "Running V-MoBA precision tests..."
        MODAL_COMMAND="$MODAL_ENV python3 -m modal run $MODAL_TEST_FILE::run_precision_tests_vmoba"
        ;;
-    "unit_test")
-        log "Running unit tests..."
-        MODAL_COMMAND="$MODAL_ENV python3 -m modal run $MODAL_TEST_FILE::run_unit_test"
-        ;;
    *)
        log "Error: Unknown test type: $TEST_TYPE"
        exit 1
@@ -62,8 +62,8 @@ on:
        required: false
        default: false
        type: boolean
-      run_unit_test:
-        description: "Run unit-test"
+      run_nightly_test:
+        description: "Run nightly-test"
        required: false
        default: false
        type: boolean
@@ -93,7 +93,6 @@ jobs:
      inference-test-STA: ${{ steps.filter.outputs.inference-test-STA }}
      precision-test-STA: ${{ steps.filter.outputs.precision-test-STA }}
      precision-test-VSA: ${{ steps.filter.outputs.precision-test-VSA }}
-      unit-test: ${{ steps.filter.outputs.unit-test }}
    steps:
      - uses: actions/checkout@v4
      - uses: dorny/paths-filter@v3
@@ -103,8 +102,6 @@ jobs:
            # Define reusable path patterns
            common-paths: &common-paths
              - 'pyproject.toml'
-              - 'docker/Dockerfile.python3.10'
-              - 'docker/Dockerfile.python3.11'
              - 'docker/Dockerfile.python3.12'
            sta-kernel-paths: &sta-kernel-paths
              - 'csrc/attn/sliding_tile_attn/**'
@@ -127,35 +124,29 @@ jobs:
            encoder-test:
              - 'fastvideo/models/encoders/**'
              - 'fastvideo/models/loader/**'
-              - 'tests/encoders/**'
-              - 'tests/utils.py'
+              - 'fastvideo/tests/encoders/**'
              - *common-paths
            vae-test:
              - 'fastvideo/models/vaes/**'
              - 'fastvideo/models/loader/**'
-              - 'tests/vaes/**'
-              - 'tests/utils.py'
+              - 'fastvideo/tests/vaes/**'
              - *common-paths
            transformer-test:
              - 'fastvideo/models/dits/**'
              - 'fastvideo/models/loader/**'
-              - 'tests/transformers/**'
-              - 'tests/utils.py'
+              - 'fastvideo/tests/transformers/**'
              - 'fastvideo/layers/**'
              - 'fastvideo/attention/**'
              - *common-paths
            training-test:
              - 'fastvideo/**'
-              - 'tests/training/Vanilla/**'
              - *common-paths
            training-test-VSA:
              - 'fastvideo/**'
-              - 'tests/training/VSA/**'
              - *common-paths
              - *vsa-kernel-paths
            inference-test-STA:
              - 'fastvideo/**'
-              - 'tests/inference/STA/**'
              - *common-paths
              - *sta-kernel-paths
            precision-test-STA:
@@ -164,11 +155,6 @@ jobs:
            precision-test-VSA:
              - *common-paths
              - *vsa-kernel-paths
-            unit-test:
-              - 'fastvideo/**'
-              - 'tests/dataset/**'
-              - 'tests/workflow/**'
-              - *common-paths

  encoder-test:
    needs: change-filter
@@ -182,7 +168,7 @@ jobs:
      gpu_count: 1
      volume_size: 100
      image: "ghcr.io/${{ github.repository }}/fastvideo-dev:py3.12-latest"
-      test_command: "uv pip install -e .[test] && pytest ./tests/encoders -s"
+      test_command: "uv pip install -e .[test] && pytest ./fastvideo/tests/encoders -s"
      timeout_minutes: 30
    secrets:
      RUNPOD_API_KEY: ${{ secrets.RUNPOD_API_KEY }}
@@ -200,7 +186,7 @@ jobs:
      gpu_count: 1
      volume_size: 100
      image: "ghcr.io/${{ github.repository }}/fastvideo-dev:py3.12-latest"
-      test_command: "uv pip install -e .[test] && pytest ./tests/vaes -s"
+      test_command: "uv pip install -e .[test] && pytest ./fastvideo/tests/vaes -s"
      timeout_minutes: 30
    secrets:
      RUNPOD_API_KEY: ${{ secrets.RUNPOD_API_KEY }}
@@ -218,7 +204,7 @@ jobs:
      gpu_count: 1
      volume_size: 100
      image: "ghcr.io/${{ github.repository }}/fastvideo-dev:py3.12-latest"
-      test_command: "uv pip install -e .[test] && pytest ./tests/transformers -s"
+      test_command: "uv pip install -e .[test] && pytest ./fastvideo/tests/transformers -s"
      timeout_minutes: 30
    secrets:
      RUNPOD_API_KEY: ${{ secrets.RUNPOD_API_KEY }}
@@ -244,7 +230,7 @@ jobs:
      volume_size: 200
      disk_size: 200
      image: "ghcr.io/${{ github.repository }}/fastvideo-dev:${{ matrix.python-version.tag }}"
-      test_command: "uv pip install -e .[test] && pytest ./tests/ssim -vs"
+      test_command: "uv pip install -e .[test] && pytest ./fastvideo/tests/ssim -vs"
      timeout_minutes: 60
    secrets:
      RUNPOD_API_KEY: ${{ secrets.RUNPOD_API_KEY }}
@@ -263,7 +249,7 @@ jobs:
      volume_size: 100
      disk_size: 100
      image: "ghcr.io/${{ github.repository }}/fastvideo-dev:py3.12-latest"
-      test_command: "wandb login $WANDB_API_KEY && uv pip install -e .[test] && pytest ./tests/training/Vanilla -srP"
+      test_command: "wandb login $WANDB_API_KEY && uv pip install -e .[test] && pytest ./fastvideo/tests/training/Vanilla -srP"
      timeout_minutes: 30
    secrets:
      RUNPOD_API_KEY: ${{ secrets.RUNPOD_API_KEY }}
@@ -283,7 +269,7 @@ jobs:
      volume_size: 100
      disk_size: 100
      image: "ghcr.io/${{ github.repository }}/fastvideo-dev:py3.12-latest"
-      test_command: "wandb login $WANDB_API_KEY && uv pip install -e .[test] && pytest ./tests/training/VSA -srP"
+      test_command: "wandb login $WANDB_API_KEY && uv pip install -e .[test] && pytest ./fastvideo/tests/training/VSA -srP"
      timeout_minutes: 30
    secrets:
      RUNPOD_API_KEY: ${{ secrets.RUNPOD_API_KEY }}
@@ -303,7 +289,7 @@ jobs:
      volume_size: 100
      disk_size: 100
      image: "ghcr.io/${{ github.repository }}/fastvideo-dev:py3.12-latest"
-      test_command: "uv pip install -e .[test] && pytest ./tests/inference/STA -srP"
+      test_command: "uv pip install -e .[test] && pytest ./fastvideo/tests/inference/STA -srP"
      timeout_minutes: 30
    secrets:
      RUNPOD_API_KEY: ${{ secrets.RUNPOD_API_KEY }}
@@ -347,42 +333,23 @@ jobs:
      RUNPOD_API_KEY: ${{ secrets.RUNPOD_API_KEY }}
      RUNPOD_PRIVATE_KEY: ${{ secrets.RUNPOD_PRIVATE_KEY }}

-  unit-test:
-    needs: change-filter
+  nightly-test:
    if: >-
-      (github.event_name != 'workflow_dispatch' && needs.change-filter.outputs.unit-test == 'true') || 
-      (github.event_name == 'workflow_dispatch' && github.event.inputs.run_unit_test == 'true')
+      (github.event_name == 'workflow_dispatch' && github.event.inputs.run_nightly_test == 'true')
    uses: ./.github/workflows/runpod-test.yml
    with:
-      job_id: "unit-test"
-      gpu_type: "NVIDIA L40S"
-      gpu_count: 1
+      job_id: "nightly-test"
+      gpu_type: "NVIDIA A40"
+      gpu_count: 4
      volume_size: 100
      disk_size: 100
      image: "ghcr.io/${{ github.repository }}/fastvideo-dev:py3.12-latest"
-      test_command: "uv pip install -e .[test] && pytest ./fastvideo/dataset/ -vs && pytest ./fastvideo/workflow/ -vs"
+      test_command: "wandb login $WANDB_API_KEY && uv pip install -e .[test] && pytest ./fastvideo/tests/nightly/test_e2e_overfit_single_sample.py -vs"
      timeout_minutes: 30
    secrets:
      RUNPOD_API_KEY: ${{ secrets.RUNPOD_API_KEY }}
      RUNPOD_PRIVATE_KEY: ${{ secrets.RUNPOD_PRIVATE_KEY }}
-
-  # nightly-test:
-  #   if: >-
-  #     (github.event_name == 'workflow_dispatch' && github.event.inputs.run_nightly_test == 'true')
-  #   uses: ./.github/workflows/runpod-test.yml
-  #   with:
-  #     job_id: "nightly-test"
-  #     gpu_type: "NVIDIA A40"
-  #     gpu_count: 4
-  #     volume_size: 100
-  #     disk_size: 100
-  #     image: "ghcr.io/${{ github.repository }}/fastvideo-dev:py3.12-latest"
-#     test_command: "wandb login $WANDB_API_KEY && uv pip install -e .[test] && pytest ./tests/nightly/test_e2e_overfit_single_sample.py -vs"
-  #     timeout_minutes: 30
-  #   secrets:
-  #     RUNPOD_API_KEY: ${{ secrets.RUNPOD_API_KEY }}
-  #     RUNPOD_PRIVATE_KEY: ${{ secrets.RUNPOD_PRIVATE_KEY }}
-  #     WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}
+      WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}

  runpod-cleanup:
    # Add other jobs to this list as you create them
@@ -64,6 +64,3 @@ docs/source/distillation/examples/
 !docs/source/_static/images/**/*.png
 !comfyui/assets/**/*.png
 !comfyui/assets/**/*.gif
-
-dmd_t2v_output/
-preprocess_output_text/
@@ -12,6 +12,9 @@ exclude: |
    scripts/.*|
    fastvideo/data_preprocess/.*|
    fastvideo/dataset/.*|
+    fastvideo/distill/.*|
+    fastvideo/distill\.py|
+    fastvideo/distill_adv\.py|
    fastvideo/models/.*|
    fastvideo/sample/.*|
    fastvideo/train\.py|
@@ -41,10 +44,10 @@ repos:
  - id: codespell
    additional_dependencies: ['tomli']
    args: ['--toml', 'pyproject.toml']
-# - repo: https://github.com/PyCQA/isort
-#   rev: 6.0.1
-#   hooks:
-#   - id: isort
+- repo: https://github.com/PyCQA/isort
+  rev: 6.0.1
+  hooks:
+  - id: isort
 - repo: https://github.com/jackdewinter/pymarkdown
  rev: v0.9.30
  hooks:
@@ -7,7 +7,7 @@
 FastVideo features an end-to-end unified pipeline for accelerating diffusion models, starting from data preprocessing to model training, finetuning, distillation, and inference. FastVideo is designed to be modular and extensible, allowing users to easily add new optimizations and techniques. Whether it is training-free optimizations or post-training optimizations, FastVideo has you covered.

 <p align="center">
-    | 🕹️ <a href="https://fastwan.fastvideo.org/"<b>Online Demo</b></a> | <a href="https://hao-ai-lab.github.io/FastVideo"><b>Documentation</b></a> | <a href="https://hao-ai-lab.github.io/FastVideo/inference/inference_quick_start.html"><b> Quick Start</b></a> | 🤗 <a href="https://huggingface.co/collections/FastVideo/fastwan-6886a305d9799c8cd1496408"  target="_blank"><b>FastWan</b></a>  | 🟣💬 <a href="https://join.slack.com/t/fastvideo/shared_invite/zt-3csdw1isz-Euq8_Q8~baewG8hxjXs2gQ" target="_blank"> <b>Slack</b> </a> |  🟣💬 <a href="https://ibb.co/q46BbX6" target="_blank"> <b> WeChat </b> </a> |
+    | 🕹️ <a href="https://fastwan.fastvideo.org/"<b>Online Demo</b></a> | <a href="https://hao-ai-lab.github.io/FastVideo"><b>Documentation</b></a> | <a href="https://hao-ai-lab.github.io/FastVideo/inference/inference_quick_start.html"><b> Quick Start</b></a> | 🤗 <a href="https://huggingface.co/collections/FastVideo/fastwan-6886a305d9799c8cd1496408"  target="_blank"><b>FastWan</b></a>  | 🟣💬 <a href="https://join.slack.com/t/fastvideo/shared_invite/zt-3csdw1isz-Euq8_Q8~baewG8hxjXs2gQ" target="_blank"> <b>Slack</b> </a> |  🟣💬 <a href="https://ibb.co/S7HLCSTh" target="_blank"> <b> WeChat </b> </a> |
 </p>

 <div align="center">
@@ -155,8 +155,8 @@ If you find FastVideo useful, please considering citing our work:
 }

@article{zhang2025vsa,
-  title={Vsa: Faster video diffusion with trainable sparse attention},
-  author={Zhang, Peiyuan and Chen, Yongqi and Huang, Haofeng and Lin, Will and Liu, Zhengzhong and Stoica, Ion and Xing, Eric and Zhang, Hao},
+  title={VSA: Faster Video Diffusion with Trainable Sparse Attention},
+  author={Zhang, Peiyuan and Huang, Haofeng and Chen, Yongqi and Lin, Will and Liu, Zhengzhong and Stoica, Ion and Xing, Eric and Zhang, Hao},
  journal={arXiv preprint arXiv:2505.13389},
  year={2025}
 }
@@ -20,7 +20,5 @@ setup(
        "License :: OSI Approved :: Apache Software License",
    ],
    python_requires='>=3.12',
-    install_requires=[
-        "flash-attn >= 2.7.1",
-    ]
+    install_requires=[]
 )
@@ -6,16 +6,8 @@ import time
 import os
 import torch
 from typing import Tuple
-try:
-    from flash_attn import flash_attn_varlen_func  # Use the new flash attention function
-    from flash_attn.flash_attn_interface import _flash_attn_varlen_forward, _flash_attn_varlen_backward
-except ImportError:
-    def _unsupported(*args, **kwargs):
-        raise ImportError("flash-attn is not installed. Please install it, e.g., `pip install flash-attn`.")
-    _flash_attn_varlen_forward = _unsupported
-    _flash_attn_varlen_backward = _unsupported
-    flash_attn_varlen_func = _unsupported
-    
+from flash_attn import flash_attn_varlen_func  # Use the new flash attention function
+from flash_attn.flash_attn_interface import _flash_attn_varlen_forward, _flash_attn_varlen_backward
 from functools import lru_cache
 from einops import rearrange

@@ -1,40 +1,32 @@
 (inference-optimizations)=
-
 # Optimizations

 This page describes the various options for speeding up generation times in FastVideo.

 ## Table of Contents
-
 - Optimized Attention Backends
-
  - [Flash Attention](#optimizations-flash)
  - [Sliding Tile Attention](#optimizations-sta)
  - [Sage Attention](#optimizations-sage)
-  - [Sage Attention 3](#optimizations-sage3)

 - Caching Techniques
  - [TeaCache](#optimizations-teacache)

 (optimizations-backends)=
-
 ## Attention Backends

 ### Available Backends
-
 - Torch SDPA: `FASTVIDEO_ATTENTION_BACKEND=TORCH_SDPA`
 - Flash Attention 2 and 3: `FASTVIDEO_ATTENTION_BACKEND=FLASH_ATTN`
 - Sliding Tile Attention: `FASTVIDEO_ATTENTION_BACKEND=SLIDING_TILE_ATTN`
 - Video Sparse Attention: `FASTVIDEO_ATTENTION_BACKEND=VIDEO_SPARSE_ATTN`
 - Sage Attention: `FASTVIDEO_ATTENTION_BACKEND=SAGE_ATTN`
- Sage Attention 3: `FASTVIDEO_ATTENTION_BACKEND=SAGE_ATTN_THREE`

 ### Configuring Backends

 There are two ways to configure the attention backend in FastVideo.

 #### 1. In Python
-
 In python, set the `FASTVIDEO_ATTENTION_BACKEND` environment variable before instantiating `VideoGenerator` like this:

 ```python
@@ -42,7 +34,6 @@ os.environ["FASTVIDEO_ATTENTION_BACKEND"] = "SLIDING_TILE_ATTN"
 ```

 #### 2. In CLI
-
 You can also set the environment variable on the command line:

 ```bash
@@ -50,7 +41,6 @@ FASTVIDEO_ATTENTION_BACKEND=SAGE_ATTN python example.py
 ```

 (optimizations-flash)=
-
 ### Flash Attention

 **`FLASH_ATTN`**
@@ -67,7 +57,7 @@ And if using a Hopper+ GPU (ie H100), installing [Flash Attention 3](https://git
 git clone https://github.com/Dao-AILab/flash-attention.git && cd flash-attention

 cd hopper
-pip install ninja
+pip install ninja 
 python setup.py install
 ```

@@ -76,9 +66,7 @@ FastVideo will automatically detect and use `FA3` if it is installed when using
 :::

 (optimizations-sta)=
-
 ### Sliding Tile Attention
-
 **`SLIDING_TILE_ATTN`**

 ```bash
@@ -88,9 +76,7 @@ pip install st_attn==0.0.4
 Please see [this page](#sta-installation) for more installation instructions.

 (optimizations-vsa)=
-
 ### Video Sparse Attention
-
 **`VIDEO_SPARSE_ATTN`**

 ```bash
@@ -101,45 +87,19 @@ python setup_vsa.py install
 Please see [this page](#vsa-installation) for more installation instructions.

 (optimizations-sage)=
-
 ### Sage Attention
-
 **`SAGE_ATTN`**

 To use [SageAttention](https://github.com/thu-ml/SageAttention) 2.1.1, please compile from source:

 ```bash
 git clone https://github.com/thu-ml/SageAttention.git
-cd sageattention
+cd sageattention 
 python setup.py install  # or pip install -e .
 ```

-(optimizations-sage3)=
-
-### Sage Attention 3
-
-**`SAGE_ATTN_THREE`**
-
-[SageAttention 3](https://huggingface.co/jt-zhang/SageAttention3) is an advanced attention mechanism that leverages FP4 quantization and Blackwell GPU Tensor Cores for significant performance improvements.
-
-#### Hardware Requirements
-
- RTX5090
-
-#### Installation
-
-Note that Sage Attention 3 requires `python>=3.13`, `torch>=2.8.0`, `CUDA >=12.8`. If you are using `uv` and using `torch==2.8.0` make sure that `sentencepiece==0.2.1` in the pyproject.toml file.
-
-To use Sage Attention 3 in FastVideo, first get access to the SageAttention3 code, then move `sageattn/` and `setup.py` to the directory `fastvideo/attention/backends`, then install from using:
-
-```bash
-python setup.py install
-```
-
 (optimizations-teacache)=
-
 ## Teacache
-
 TeaCache is an optimization technique supported in FastVideo that can significantly speed up video generation by skipping redundant calculations across diffusion steps. This guide explains how to enable and configure TeaCache for optimal performance in FastVideo.

 ### What is TeaCache?
@@ -1,9 +0,0 @@
-# VidProm Dataset
-
-From [Self-Forcing](https://github.com/gdhe17/Self-Forcing) repository.
-
-## Download the dataset
-
-```bash
-./download_dataset.sh
-```
@@ -1,3 +0,0 @@
-#! /bin/bash
-
-huggingface-cli download gdhe17/Self-Forcing vidprom_filtered_extended.txt --local-dir prompts
@@ -1,151 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=t2v
-#SBATCH --partition=main
-#SBATCH --nodes=1
-#SBATCH --ntasks=1
-#SBATCH --ntasks-per-node=1
-#SBATCH --gres=gpu:1
-#SBATCH --cpus-per-task=128
-#SBATCH --mem=1440G
-#SBATCH --output=dmd_t2v_output/t2v_%j.out
-#SBATCH --error=dmd_t2v_output/t2v_%j.err
-#SBATCH --exclusive
-
-# Basic Info
-export NCCL_P2P_DISABLE=1
-export TORCH_NCCL_ENABLE_MONITORING=0
-# different cache dir for different processes
-export TRITON_CACHE_DIR=/tmp/triton_cache_${SLURM_PROCID}
-export MASTER_PORT=29503
-export TOKENIZERS_PARALLELISM=false
-export WANDB_API_KEY=your_wandb_api_key
-export WANDB_BASE_URL="https://api.wandb.ai"
-export WANDB_MODE=online
-export FASTVIDEO_ATTENTION_BACKEND=FLASH_ATTN
-
-# Configs
-NUM_GPUS=1
-
-# Model paths for Self-Forcing DMD distillation:
-GENERATOR_MODEL_PATH="wlsaidhi/SFWan2.1-T2V-1.3B-Diffusers"
-REAL_SCORE_MODEL_PATH="Wan-AI/Wan2.1-T2V-14B-Diffusers"  # Teacher model
-FAKE_SCORE_MODEL_PATH="Wan-AI/Wan2.1-T2V-1.3B-Diffusers"  # Critic model
-
-DATA_DIR=your_data_dir
-VALIDATION_DATASET_FILE=your_validation_data_dir
-# export CUDA_VISIBLE_DEVICES=4,5
-# IP=[MASTER NODE IP]
-
-# Training arguments
-training_args=(
-  --tracker_project_name SFwan_t2v_distill_self_forcing_dmd  # Updated for self-forcing DMD
-  --output_dir your_output_dir
-  --max_train_steps 4000
-  --train_batch_size 1
-  --train_sp_batch_size 1
-  --gradient_accumulation_steps 1
-  --num_latent_t 21
-  --num_height 480
-  --num_width 832
-  --num_frames 81  # Must be divisible by num_frame_per_block (81 % 3 = 0 ✓)
-  --enable_gradient_checkpointing_type "full"
-  --log_visualization
-  --simulate_generator_forward
-  --num_frame_per_block 3  # Frame generation block size for self-forcing
-  --enable_gradient_masking
-  --gradient_mask_last_n_frames 21
-)
-
-# Parallel arguments
-parallel_args=(
-  --num_gpus $NUM_GPUS # 64
-  --sp_size 1
-  --tp_size 1
-  --hsdp_replicate_dim 1 # 64
-  --hsdp_shard_dim $NUM_GPUS
-)
-
-# Model arguments
-model_args=(
-  --model_path $GENERATOR_MODEL_PATH  # TODO: check if you can remove this in this script
-  --pretrained_model_name_or_path $GENERATOR_MODEL_PATH
-  --generator_model_path $GENERATOR_MODEL_PATH
-  --real_score_model_path $REAL_SCORE_MODEL_PATH
-  --fake_score_model_path $FAKE_SCORE_MODEL_PATH
-)
-
-# Dataset arguments
-dataset_args=(
-  --data_path "$DATA_DIR"
-  --dataloader_num_workers 4
-)
-
-# Validation arguments
-validation_args=(
-  --log_validation
-  --validation_dataset_file "$VALIDATION_DATASET_FILE"
-  --validation_steps 50
-  --validation_sampling_steps "4"
-  --validation_guidance_scale "6.0" # not used for dmd inference
-)
-
-# Optimizer arguments
-optimizer_args=(
-  --learning_rate 1e-5
-  --mixed_precision "bf16"
-  --training_state_checkpointing_steps 500
-  --weight_only_checkpointing_steps 500
-  --weight_decay 0.01
-  --betas '0.0,0.999'
-  --max_grad_norm 1.0
-)
-
-# Miscellaneous arguments
-miscellaneous_args=(
-  --inference_mode False
-  --checkpoints_total_limit 3
-  --training_cfg_rate 0.0
-  --dit_precision "fp32"
-  --flow_shift 5
-  --seed 1000
-  --use_ema True
-  --ema_decay 0.99
-  --ema_start_step 100
-  --init_weights_from_safetensors your_ode_init_weights_path
-)
-
-# Self-forcing DMD arguments
-dmd_args=(
-  --dmd_denoising_steps '1000,750,500,250'
-  --min_timestep_ratio 0.02
-  --max_timestep_ratio 0.98
-  --dfake_gen_update_ratio 5
-  --real_score_guidance_scale 3.0
-  --fake_score_learning_rate 8e-6
-  --fake_score_betas '0.0,0.999'
-  --warp_denoising_step
-)
-
-# Self-forcing specific arguments
-self_forcing_args=(
-  --independent_first_frame False  # Whether to treat first frame independently
-  --same_step_across_blocks True  # Whether to use same denoising step across all blocks
-  --last_step_only False  # Whether to only use the last denoising step
-  --context_noise 0  # Amount of noise to add during context caching (0 = no noise)
-  --validate_cache_structure False  # Set to True for debugging KV cache issues
-)
-
-torchrun \
--nnodes 1 \
--master_port $MASTER_PORT \
--nproc_per_node $NUM_GPUS \
-    fastvideo/training/wan_self_forcing_distillation_pipeline.py \
-    "${parallel_args[@]}" \
-    "${model_args[@]}" \
-    "${dataset_args[@]}" \
-    "${training_args[@]}" \
-    "${optimizer_args[@]}" \
-    "${validation_args[@]}" \
-    "${miscellaneous_args[@]}" \
-    "${dmd_args[@]}" \
-    "${self_forcing_args[@]}"
@@ -1,3 +0,0 @@
-#!/bin/bash
-
-python scripts/huggingface/download_hf.py --repo_id "wlsaidhi/crush-smol-merged" --local_dir "data/crush-smol" --repo_type "dataset"
@@ -1,24 +0,0 @@
-#!/bin/bash
-
-GPU_NUM=1 # 2,4,8
-MODEL_PATH="Wan-AI/Wan2.1-T2V-1.3B-Diffusers"
-MODEL_TYPE="wan"
-DATA_MERGE_PATH="data/crush-smol/merge.txt"
-OUTPUT_DIR="data/crush-smol_processed_t2v/"
-
-torchrun --nproc_per_node=$GPU_NUM \
-    fastvideo/pipelines/preprocess/v1_preprocess.py \
-    --model_path $MODEL_PATH \
-    --data_merge_path $DATA_MERGE_PATH \
-    --preprocess_video_batch_size 8 \
-    --seed 42 \
-    --max_height 480 \
-    --max_width 832 \
-    --num_frames 81 \
-    --dataloader_num_workers 0 \
-    --output_dir=$OUTPUT_DIR \
-    --train_fps 16 \
-    --samples_per_file 8 \
-    --flush_frequency 8 \
-    --video_length_tolerance_range 5 \
-    --preprocess_task "t2v" 
@@ -41,14 +41,13 @@ NUM_GPUS=8
 MODEL_PATH="Wan-AI/Wan2.1-T2V-1.3B-Diffusers"
 DATA_DIR=your_data_dir
 VALIDATION_DATASET_FILE=your_validation_dataset_file
-OUTPUT_DIR="checkpoints/wan_t2v_finetune"
 # export CUDA_VISIBLE_DEVICES=4,5
 # IP=[MASTER NODE IP]

 # Training arguments
 training_args=(
  --tracker_project_name wan_t2v_distill_dmd_VSA
-  --output_dir $OUTPUT_DIR
+  --output_dir"checkpoints/wan_t2v_finetune"
  --max_train_steps 4000
  --train_batch_size 1
  --train_sp_batch_size 1
@@ -92,7 +91,7 @@ validation_args=(

 # Optimizer arguments
 optimizer_args=(
-  --learning_rate 2e-6
+  --learning_rate 1e-5
  --mixed_precision "bf16"
  --training_state_checkpointing_steps 500
  --weight_only_checkpointing_steps 500
@@ -135,4 +134,4 @@ srun torchrun \
    "${optimizer_args[@]}" \
    "${validation_args[@]}" \
    "${miscellaneous_args[@]}" \
-    "${dmd_args[@]}"
+    "${dmd_args[@]}"
@@ -41,14 +41,13 @@ NUM_GPUS=8
 MODEL_PATH="Wan-AI/Wan2.1-T2V-14B-Diffusers"
 DATA_DIR=your_data_dir
 VALIDATION_DATASET_FILE=your_validation_dataset_file
-OUTPUT_DIR="checkpoints/wan_t2v_finetune"
 # export CUDA_VISIBLE_DEVICES=4,5
 # IP=[MASTER NODE IP]

 # Training arguments
 training_args=(
  --tracker_project_name wan_t2v_distill_dmd_VSA
-  --output_dir "$OUTPUT_DIR"
+  --output_dir "checkpoints/wan_t2v_finetune"
  --max_train_steps 4000
  --train_batch_size 1
  --train_sp_batch_size 1
@@ -92,7 +91,7 @@ validation_args=(

 # Optimizer arguments
 optimizer_args=(
-  --learning_rate 2e-6
+  --learning_rate 1e-5
  --mixed_precision "bf16"
  --training_state_checkpointing_steps 500
  --weight_only_checkpointing_steps 500
@@ -135,4 +134,4 @@ srun torchrun \
    "${optimizer_args[@]}" \
    "${validation_args[@]}" \
    "${miscellaneous_args[@]}" \
-    "${dmd_args[@]}"
+    "${dmd_args[@]}"
@@ -41,14 +41,13 @@ NUM_GPUS=8
 MODEL_PATH="Wan-AI/Wan2.1-T2V-1.3B-Diffusers"
 DATA_DIR=your_data_dir
 VALIDATION_DATASET_FILE=your_validation_dataset_file
-OUTPUT_DIR="checkpoints/wan_t2v_finetune"
 # export CUDA_VISIBLE_DEVICES=4,5
 # IP=[MASTER NODE IP]

 # Training arguments
 training_args=(
  --tracker_project_name wan_t2v_distill_dmd
-  --output_dir "$OUTPUT_DIR"
+  --output_dir "checkpoints/wan_t2v_finetune"
  --max_train_steps 4000
  --train_batch_size 1
  --train_sp_batch_size 1
@@ -92,7 +91,7 @@ validation_args=(

 # Optimizer arguments
 optimizer_args=(
-  --learning_rate 2e-6
+  --learning_rate 1e-5
  --mixed_precision "bf16"
  --training_state_checkpointing_steps 500
  --weight_only_checkpointing_steps 500
@@ -134,4 +133,4 @@ srun torchrun \
    "${optimizer_args[@]}" \
    "${validation_args[@]}" \
    "${miscellaneous_args[@]}" \
-    "${dmd_args[@]}"
+    "${dmd_args[@]}"
@@ -1,3 +1,3 @@
 #!/bin/bash

-python scripts/huggingface/download_hf.py --repo_id "FastVideo/Wan-Syn_77x448x832_600k" --local_dir "FastVideo/Wan-Syn_77x448x832_600k" --repo_type "dataset"
+python scripts/huggingface/download_hf.py --repo_id "FastVideo/Wan-Syn_77x448x832_600k" --local_dir "FastVideo/Wan-Syn_77x448x832_600k" --repo_type "dataset"
@@ -42,14 +42,13 @@ NUM_GPUS=8
 MODEL_PATH="Wan-AI/Wan2.2-TI2V-5B-Diffusers"
 DATA_DIR=your_data_dir
 VALIDATION_DIR=your_validation_path  #(example:validation_64.json)
-OUTPUT_DIR="checkpoints/wan_t2v_finetune"
 # export CUDA_VISIBLE_DEVICES=4,5
 # IP=[MASTER NODE IP]

 # Training arguments
 training_args=(
  --tracker_project_name Wan_distillation
-  --output_dir "$OUTPUT_DIR"
+  --output_dir "your_output_dir"
  --max_train_steps 4000
  --train_batch_size 1
  --train_sp_batch_size 1
@@ -93,11 +92,11 @@ validation_args=(

 # Optimizer arguments
 optimizer_args=(
-  --learning_rate 4e-6
+  --learning_rate 2e-5
  --lr_scheduler "cosine_with_min_lr"
  --min_lr_ratio 0.5
  --lr_warmup_steps 100
-  --fake_score_learning_rate 2e-6
+  --fake_score_learning_rate 1e-5
  --fake_score_lr_scheduler "cosine_with_min_lr"
  --mixed_precision "bf16"
  --training_state_checkpointing_steps 500
@@ -142,4 +141,4 @@ srun torchrun \
    "${optimizer_args[@]}" \
    "${validation_args[@]}" \
    "${miscellaneous_args[@]}" \
-    "${dmd_args[@]}"
+    "${dmd_args[@]}"
@@ -92,11 +92,11 @@ validation_args=(

 # Optimizer arguments
 optimizer_args=(
-  --learning_rate 4e-6
+  --learning_rate 2e-5
  --lr_scheduler "cosine_with_min_lr"
  --min_lr_ratio 0.5
  --lr_warmup_steps 100
-  --fake_score_learning_rate 2e-6
+  --fake_score_learning_rate 1e-5
  --fake_score_lr_scheduler "cosine_with_min_lr"
  --mixed_precision "bf16"
  --training_state_checkpointing_steps 500
@@ -142,4 +142,4 @@ srun torchrun \
    "${optimizer_args[@]}" \
    "${validation_args[@]}" \
    "${miscellaneous_args[@]}" \
-    "${dmd_args[@]}"
+    "${dmd_args[@]}"
@@ -16,25 +16,24 @@ NUM_GPUS=1
 MODEL_PATH="Wan-AI/Wan2.2-TI2V-5B-Diffusers"
 DATA_DIR="data/crush-smol_processed_ti2v/combined_parquet_dataset/"
 VALIDATION_DATASET_FILE="examples/distill/Wan2.2-TI2V-5B-Diffusers/crush_smol/validation.json"
-OUTPUT_DIR="checkpoints/wan_t2v_finetune"
 # export CUDA_VISIBLE_DEVICES=4,5
 # IP=[MASTER NODE IP]

 # Training arguments
 training_args=(
  --tracker_project_name wan_t2v_distill_dmd_VSA
-  --output_dir "$OUTPUT_DIR"
-  --max_train_steps 4000
-  --train_batch_size 1
+  --output_dir="checkpoints/wan_t2v_finetune"
+  --max_train_steps=4000
+  --train_batch_size=1
  --train_sp_batch_size 1
-  --gradient_accumulation_steps 1
+  --gradient_accumulation_steps=1
  --num_latent_t 31
  --num_height 704
  --num_width 1280
  --num_frames 121
  --enable_gradient_checkpointing_type "full"
-  --training_state_checkpointing_steps 500
-  --weight_only_checkpointing_steps 500
+  --training_state_checkpointing_steps=500
+  --weight_only_checkpointing_steps=500
 )

 # Parallel arguments
@@ -69,8 +68,8 @@ validation_args=(

 # Optimizer arguments
 optimizer_args=(
-  --learning_rate 2e-6
-  --mixed_precision "bf16"
+  --learning_rate=1e-5
+  --mixed_precision="bf16"
  --weight_decay 0.01
  --max_grad_norm 1.0
 )
@@ -108,4 +107,4 @@ torchrun \
    "${optimizer_args[@]}" \
    "${validation_args[@]}" \
    "${miscellaneous_args[@]}" \
-    "${dmd_args[@]}"
+    "${dmd_args[@]}"
@@ -109,4 +109,4 @@ torchrun \
    "${optimizer_args[@]}" \
    "${validation_args[@]}" \
    "${miscellaneous_args[@]}" \
-    "${dmd_args[@]}"
+    "${dmd_args[@]}"
@@ -1,3 +1,3 @@
 #!/bin/bash

-python scripts/huggingface/download_hf.py --repo_id "wlsaidhi/crush-smol-merged" --local_dir "data/crush-smol" --repo_type "dataset"
+python scripts/huggingface/download_hf.py --repo_id "wlsaidhi/crush-smol-merged" --local_dir "data/crush-smol" --repo_type "dataset"
@@ -21,4 +21,4 @@ torchrun --nproc_per_node=$GPU_NUM \
    --samples_per_file 8 \
    --flush_frequency 8 \
    --video_length_tolerance_range 5 \
-    --preprocess_task "t2v" 
+    --preprocess_task "t2v" 
@@ -1,135 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=2e6B8_16kFV_ode_vidprom
-#SBATCH --partition=main
-#SBATCH --nodes=1
-#SBATCH --ntasks=1
-#SBATCH --ntasks-per-node=1
-#SBATCH --gres=gpu:8
-#SBATCH --cpus-per-task=128
-#SBATCH --mem=1440G
-#SBATCH --output=ode_vidprom16k/ode_vidprom8b16k_2e-6.out
-#SBATCH --error=ode_vidprom16k/ode_vidprom8b16k_2e-6.err
-#SBATCH --exclusive
-set -e -x
-
-# Environment Setup
-source ~/conda/miniconda/bin/activate
-conda activate your-conda-env
-
-export WANDB_MODE="online"
-export NCCL_P2P_DISABLE=1
-export TORCH_NCCL_ENABLE_MONITORING=0
-# different cache dir for different processes
-export TRITON_CACHE_DIR=/tmp/triton_cache_${SLURM_PROCID}
-export MASTER_PORT=29500
-export NODE_RANK=$SLURM_PROCID
-nodes=( $(scontrol show hostnames $SLURM_JOB_NODELIST) )
-export MASTER_ADDR=${nodes[0]}
-export CUDA_VISIBLE_DEVICES=$SLURM_LOCALID
-export TOKENIZERS_PARALLELISM=false
-export WANDB_BASE_URL="https://api.wandb.ai"
-export WANDB_API_KEY=your-wandb-api-key
-# export FASTVIDEO_ATTENTION_BACKEND=TORCH_SDPA
-
-echo "MASTER_ADDR: $MASTER_ADDR"
-echo "NODE_RANK: $NODE_RANK"
-
-
-MODEL_PATH="wlsaidhi/SFWan2.1-T2V-1.3B-Diffusers"
-DATA_DIR="your-data-dir"
-VALIDATION_DATASET_FILE="examples/training/consistency_finetune/causal_ode_init/validation.json"
-OUTPUT_DIR="your-output-dir"
-INIT_WEIGHTS_FROM_SAFETENSORS="your-init-weights-from-safetensors" # bidirectional weights from Wan2.1-T2V-1.3B-Diffusers
-NUM_GPUS=8
-# export CUDA_VISIBLE_DEVICES=4,5
-# IP=[MASTER NODE IP]
-
-# Training arguments
-training_args=(
-  --tracker_project_name "wan_ode_init"
-  --output_dir $OUTPUT_DIR
-  --override_transformer_cls_name "CausalWanTransformer3DModel"
-  --wandb_run_name "vidprom_8b16k_ode_init_2e-6"
-  # --resume_from_checkpoint "ode_init_diffusers/"
-  --warp_denoising_step
-  --log_visualization
-  --max_train_steps 6001
-  --train_batch_size 1
-  --train_sp_batch_size 1
-  --gradient_accumulation_steps 1
-  --num_latent_t 21
-  --num_height 480
-  --num_width 832
-  --num_frames 81
-  --dmd_denoising_steps "1000,750,500,250"
-  --enable_gradient_checkpointing_type "full"
-)
-
-# Parallel arguments
-parallel_args=(
-  --num_gpus $NUM_GPUS
-  --sp_size 1
-  --tp_size 1
-  --hsdp_replicate_dim $NUM_GPUS
-  --hsdp_shard_dim 1
-)
-
-# Model arguments
-model_args=(
-  --model_path $MODEL_PATH
-  --pretrained_model_name_or_path $MODEL_PATH
-)
-
-# Dataset arguments
-dataset_args=(
-  --data_path "$DATA_DIR" 
-  --dataloader_num_workers 1
-)
-
-# Validation arguments
-validation_args=(
-  --log_validation
-  --validation_dataset_file "$VALIDATION_DATASET_FILE"
-  --validation_steps 50
-  --validation_sampling_steps "50"
-  --validation_guidance_scale "6.0"
-  --init_weights_from_safetensors $INIT_WEIGHTS_FROM_SAFETENSORS
-)
-
-# Optimizer arguments
-optimizer_args=(
-  --learning_rate 2e-6
-  --mixed_precision "bf16"
-  --checkpointing_steps 500
-  --weight_decay 1e-4
-  --max_grad_norm 1.0
-)
-
-# Miscellaneous arguments
-miscellaneous_args=(
-  --inference_mode False
-  --checkpoints_total_limit 3
-  --training_cfg_rate 0.1
-  --multi_phased_distill_schedule "4000-1"
-  --not_apply_cfg_solver
-  --dit_precision "fp32"
-  --num_euler_timesteps 50
-  --ema_start_step 0
-  # --enable_gradient_checkpointing_type "full"
-)
-
-# If you do not have 32 GPUs and to fit in memory, you can: 1. increase sp_size. 2. reduce num_latent_t
-srun torchrun \
-  --nnodes $SLURM_JOB_NUM_NODES \
-  --nproc_per_node $NUM_GPUS \
-  --node_rank $SLURM_PROCID \
-  --rdzv_backend=c10d \
-  --rdzv_endpoint="$MASTER_ADDR:$MASTER_PORT" \
-    fastvideo/training/ode_causal_pipeline.py \
-    "${parallel_args[@]}" \
-    "${model_args[@]}" \
-    "${dataset_args[@]}" \
-    "${training_args[@]}" \
-    "${optimizer_args[@]}" \
-    "${validation_args[@]}" \
-    "${miscellaneous_args[@]}"
@@ -15,7 +15,6 @@ torchrun --nproc_per_node=$GPU_NUM \
    --max_height 480 \
    --max_width 832 \
    --num_frames 81 \
-    --flow_shift 5.0 \
    --dataloader_num_workers 0 \
    --output_dir=$OUTPUT_DIR \
    --train_fps 16 \
@@ -1,5 +1,14 @@
 {
  "data": [
+    {
+      "caption": "A large metal cylinder is seen pressing down on a pile of Oreo cookies, flattening them as if they were under a hydraulic press.",
+      "image_path": null,
+      "video_path": null,
+      "num_inference_steps": 40,
+      "height": 480,
+      "width": 832,
+      "num_frames": 77
+    },
    {
      "caption": "A stylish woman walks down a Tokyo street filled with warm glowing neon and animated city signage. She wears a black leather jacket, a long red dress, and black boots, and carries a black purse. She wears sunglasses and red lipstick. She walks confidently and casually. The street is damp and reflective, creating a mirror effect of the colorful lights. Many pedestrians walk about.",
      "image_path": null,
@@ -19,52 +28,7 @@
      "num_frames": 77
    },
    {
-      "caption": "Elon Musk, dressed in a sleek white spacesuit with a reflective visor, walks confidently across the lunar surface. His posture is upright, and he moves steadily with purpose. The moon's rocky terrain and scattered boulders surround him, casting shadows under the dim sunlight. The background shows vast stretches of the moon's barren landscape with craters and dust clouds kicked up by his boots. The scene captures a wide shot, emphasizing the vastness and desolation of the lunar environment. ",
-      "image_path": null,
-      "video_path": null,
-      "num_inference_steps": 40,
-      "height": 480,
-      "width": 832,
-      "num_frames": 77
-    },
-    {
-      "caption": "In a dynamic action-packed sequence set in the Marvel multiverse, Spider-Man and Venom engage in an intense battle. Spider-Man, in his classic red and blue suit, swings and dodges venomous attacks from the black symbiote-covered Venom. Both characters display a range of acrobatic moves and powerful strikes. The environment is a chaotic urban landscape with crumbling buildings and neon lights, reflecting the multiversal theme. The camera captures the epic fight from various angles, including wide shots to show the scale of destruction and close-ups to highlight their fierce expressions and physical combat. ",
-      "image_path": null,
-      "video_path": null,
-      "num_inference_steps": 40,
-      "height": 480,
-      "width": 832,
-      "num_frames": 77
-    },
-    {
-      "caption": "A warm, family-oriented scene depicting a father getting ready to leave the house to buy milk. The father, a middle-aged man with a kind face and a casual outfit, picks up a jacket from the coat rack. His posture is upright as he bends down slightly to put on his shoes. In the background, there are glimpses of a cozy living room with a family photograph on the wall. The camera focuses closely on the father, capturing his gentle smile and reassuring nod towards the camera before he opens the front door and steps outside. Static medium close-up shot. ",
-      "image_path": null,
-      "video_path": null,
-      "num_inference_steps": 40,
-      "height": 480,
-      "width": 832,
-      "num_frames": 77
-    },
-    {
-      "caption": "Close-up shot of a man with a prosthetic hand that functions as a rocket launcher. He looks at his new hand with a mix of amazement and concern, his facial expression showing a blend of curiosity and apprehension. The prosthetic hand is sleek and metallic, with intricate details that resemble a high-tech weapon. The background is a dimly lit laboratory with various scientific equipment and monitors displaying data. The man stands in a relaxed posture, his other hand resting on his hip, as he inspects his new limb. The scene is rendered in a realistic sci-fi style, emphasizing the futuristic technology and the man's emotional response to his new appendage. ",
-      "image_path": null,
-      "video_path": null,
-      "num_inference_steps": 40,
-      "height": 480,
-      "width": 832,
-      "num_frames": 77
-    },
-    {
-      "caption": "Realistic CCTV footage style, Kim Taehyung from the band BTS is involved in a drug deal, caught on camera. Kim Taehyung appears nervous and cautious, wearing casual clothing typical of a public space. He exchanges items discreetly with another person, who is partially obscured. Both individuals maintain a watchful demeanor, occasionally glancing around to ensure no one is watching them. The lighting is dim, with flickering fluorescent lights casting shadows on their faces. The background shows a typical urban setting with blurred figures moving in the distance. Static camera angle, medium close-up shot focusing on the interaction between Taehyung and the other individual. ",
-      "image_path": null,
-      "video_path": null,
-      "num_inference_steps": 40,
-      "height": 480,
-      "width": 832,
-      "num_frames": 77
-    },
-    {
-      "caption": "Photorealistic studio setup with professional lighting, showcasing detailed cubic dissections of experimental plastic and felt-like materials on a pristine white background. Each cube reveals intricate layers and textures of the materials, emphasizing their unique properties. The scene has a shallow depth of field initially, then slowly pulls out to reveal the full arrangement of cubes, maintaining a wide depth of field throughout the transition. ",
+      "caption": "A watermelon wearing a helmet is crushed by a hydraulic press, causing it to flatten and burst open.",
      "image_path": null,
      "video_path": null,
      "num_inference_steps": 40,
@@ -1,133 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=t2v
-#SBATCH --partition=main
-#SBATCH --nodes=8
-#SBATCH --ntasks=8
-#SBATCH --ntasks-per-node=1
-#SBATCH --gres=gpu:8
-#SBATCH --cpus-per-task=128
-#SBATCH --mem=1440G
-#SBATCH --output=VSA_t2v_output/t2v_%j.out
-#SBATCH --error=VSA_t2v_output/t2v_%j.err
-#SBATCH --exclusive
-set -e -x
-
-# Environment Setup
-source ~/conda/miniconda/bin/activate
-conda activate your_env
-
-# Basic Info
-export WANDB_MODE="online"
-export NCCL_P2P_DISABLE=1
-export TORCH_NCCL_ENABLE_MONITORING=0
-# different cache dir for different processes
-export TRITON_CACHE_DIR=/tmp/triton_cache_${SLURM_PROCID}
-export MASTER_PORT=29500
-export NODE_RANK=$SLURM_PROCID
-nodes=( $(scontrol show hostnames $SLURM_JOB_NODELIST) )
-export MASTER_ADDR=${nodes[0]}
-export CUDA_VISIBLE_DEVICES=$SLURM_LOCALID
-export TOKENIZERS_PARALLELISM=false
-export WANDB_BASE_URL="https://api.wandb.ai"
-export WANDB_MODE=online
-export FASTVIDEO_ATTENTION_BACKEND=VIDEO_SPARSE_ATTN
-# export FASTVIDEO_ATTENTION_BACKEND=TORCH_SDPA
-
-echo "MASTER_ADDR: $MASTER_ADDR"
-echo "NODE_RANK: $NODE_RANK"
-
-# Configs
-NUM_GPUS=8
-MODEL_PATH="Wan-AI/Wan2.1-T2V-1.3B-Diffusers"
-DATA_DIR=your_data_dir
-VALIDATION_DATASET_FILE=your_validation_dataset_file
-# export CUDA_VISIBLE_DEVICES=4,5
-# IP=[MASTER NODE IP]
-
-# Training arguments
-training_args=(
-  --tracker_project_name wan_t2v_VSA
-  --output_dir "checkpoints/wan_t2v_finetune_VSA"
-  --max_train_steps 4000
-  --train_batch_size 1
-  --train_sp_batch_size 1
-  --gradient_accumulation_steps 1
-  --num_latent_t 21
-  --num_height 480
-  --num_width 832
-  --num_frames 81
-  # --enable_gradient_checkpointing_type "full" # if OOM enable this
-)
-
-# Parallel arguments
-parallel_args=(
-  --num_gpus 64
-  --sp_size 1
-  --tp_size 1
-  --hsdp_replicate_dim 64
-  --hsdp_shard_dim 1
-)
-
-# Model arguments
-model_args=(
-  --model_path $MODEL_PATH
-  --pretrained_model_name_or_path $MODEL_PATH
-)
-
-# Dataset arguments
-dataset_args=(
-  --data_path "$DATA_DIR"
-  --dataloader_num_workers 4
-)
-
-# Validation arguments
-validation_args=(
-  --log_validation
-  --validation_dataset_file $VALIDATION_DATASET_FILE
-  --validation_steps 200
-  --validation_sampling_steps "50"
-  --validation_guidance_scale "5.0"
-)
-
-# Optimizer arguments
-optimizer_args=(
-  --learning_rate 1e-5
-  --mixed_precision "bf16"
-  --checkpointing_steps 1000
-  --weight_decay 0.01
-  --max_grad_norm 1.0
-)
-
-# Miscellaneous arguments
-miscellaneous_args=(
-  --inference_mode False
-  --checkpoints_total_limit 3
-  --training_cfg_rate 0.1
-  --dit_precision "fp32"
-  --ema_start_step 0
-  --flow_shift 1
-  --seed 1000
-)
-
-# VSA arguments
-vsa_args=(
-  --VSA_decay_rate 0.03 \
-  --VSA_decay_interval_steps 50 \
-  --VSA_sparsity 0.9 \
-)
-
-srun torchrun \
--nnodes $SLURM_JOB_NUM_NODES \
--nproc_per_node $NUM_GPUS \
--node_rank $SLURM_PROCID \
--rdzv_backend=c10d \
--rdzv_endpoint="$MASTER_ADDR:$MASTER_PORT" \
-    fastvideo/training/wan_training_pipeline.py \
-    "${parallel_args[@]}" \
-    "${model_args[@]}" \
-    "${dataset_args[@]}" \
-    "${training_args[@]}" \
-    "${optimizer_args[@]}" \
-    "${validation_args[@]}" \
-    "${miscellaneous_args[@]}" \
-    "${vsa_args[@]}"  
@@ -1,72 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-
-import torch
-from fastvideo.attention.backends.sageattn.api import sageattn_blackwell
-
-from fastvideo.attention.backends.abstract import (AttentionBackend,
-                                                   AttentionImpl,
-                                                   AttentionMetadata,
-                                                   AttentionMetadataBuilder)
-from fastvideo.logger import init_logger
-
-logger = init_logger(__name__)
-
-
-class SageAttention3Backend(AttentionBackend):
-
-    accept_output_buffer: bool = True
-
-    @staticmethod
-    def get_supported_head_sizes() -> list[int]:
-        return [64, 128, 256]
-
-    @staticmethod
-    def get_name() -> str:
-        return "SAGE_ATTN_THREE"
-
-    @staticmethod
-    def get_impl_cls() -> type["SageAttention3Impl"]:
-        return SageAttention3Impl
-
-    @staticmethod
-    def get_metadata_cls() -> type["AttentionMetadata"]:
-        raise NotImplementedError
-
-    @staticmethod
-    def get_builder_cls() -> type["AttentionMetadataBuilder"]:
-        raise NotImplementedError
-
-    # @staticmethod
-    # def get_metadata_cls() -> Type["AttentionMetadata"]:
-    #     return FlashAttentionMetadata
-
-
-class SageAttention3Impl(AttentionImpl):
-
-    def __init__(
-        self,
-        num_heads: int,
-        head_size: int,
-        causal: bool,
-        softmax_scale: float,
-        num_kv_heads: int | None = None,
-        prefix: str = "",
-        **extra_impl_args,
-    ) -> None:
-        self.causal = causal
-        self.softmax_scale = softmax_scale
-        self.dropout = extra_impl_args.get("dropout_p", 0.0)
-
-    def forward(
-        self,
-        query: torch.Tensor,
-        key: torch.Tensor,
-        value: torch.Tensor,
-        attn_metadata: AttentionMetadata,
-    ) -> torch.Tensor:
-        query = query.transpose(1, 2)
-        key = key.transpose(1, 2)
-        value = value.transpose(1, 2)
-        output = sageattn_blackwell(query, key, value, is_causal=self.causal)
-        output = output.transpose(1, 2)
-        return output
@@ -5,6 +5,7 @@ from dataclasses import dataclass

 import torch
 from einops import rearrange
+from flash_attn.bert_padding import pad_input

 from csrc.attn.vmoba_attn.vmoba import (moba_attn_varlen, process_moba_input,
                                        process_moba_output)
@@ -133,8 +134,6 @@ class VMOBAAttentionImpl(AttentionImpl):
                 **extra_impl_args) -> None:
        self.prefix = prefix
        self.layer_idx = self._get_layer_idx(prefix)
-        from flash_attn.bert_padding import pad_input
-        self.pad_input = pad_input

    def _get_layer_idx(self, prefix: str) -> int | None:
        match = re.search(r"blocks\.(\d+)", prefix)
@@ -170,6 +169,7 @@ class VMOBAAttentionImpl(AttentionImpl):
            moba_chunk_size = attn_metadata.st_chunk_size
            moba_topk = attn_metadata.st_topk

+        # torch.distributed.breakpoint()
        query, chunk_size = process_moba_input(query,
                                               attn_metadata.patch_resolution,
                                               moba_chunk_size)
@@ -205,8 +205,8 @@ class VMOBAAttentionImpl(AttentionImpl):
            simsum_threshold=attn_metadata.moba_threshold,
            threshold_type=attn_metadata.moba_threshold_type,
        )
-        hidden_states = self.pad_input(hidden_states, indices_q, batch_size,
-                                       sequence_length)
+        hidden_states = pad_input(hidden_states, indices_q, batch_size,
+                                  sequence_length)
        hidden_states = process_moba_output(hidden_states,
                                            attn_metadata.patch_resolution,
                                            moba_chunk_size)
@@ -15,16 +15,18 @@ class DiTArchConfig(ArchConfig):
    reverse_param_names_mapping: dict = field(default_factory=dict)
    lora_param_names_mapping: dict = field(default_factory=dict)
    _supported_attention_backends: tuple[AttentionBackendEnum, ...] = (
-        AttentionBackendEnum.SLIDING_TILE_ATTN, AttentionBackendEnum.SAGE_ATTN,
-        AttentionBackendEnum.FLASH_ATTN, AttentionBackendEnum.TORCH_SDPA,
-        AttentionBackendEnum.VIDEO_SPARSE_ATTN, AttentionBackendEnum.VMOBA_ATTN,
-        AttentionBackendEnum.SAGE_ATTN_THREE)
+        AttentionBackendEnum.SLIDING_TILE_ATTN,
+        AttentionBackendEnum.SAGE_ATTN,
+        AttentionBackendEnum.FLASH_ATTN,
+        AttentionBackendEnum.TORCH_SDPA,
+        AttentionBackendEnum.VIDEO_SPARSE_ATTN,
+        AttentionBackendEnum.VMOBA_ATTN,
+    )

    hidden_size: int = 0
    num_attention_heads: int = 0
    num_channels_latents: int = 0
    exclude_lora_layers: list[str] = field(default_factory=list)
-    boundary_ratio: float | None = None

    def __post_init__(self) -> None:
        if not self._compile_conditions:
@@ -92,9 +92,6 @@ class WanVideoArchConfig(DiTArchConfig):
    pos_embed_seq_len: int | None = None
    exclude_lora_layers: list[str] = field(default_factory=lambda: ["embedder"])

-    # Wan MoE
-    boundary_ratio: float | None = None
-
    # Causal Wan
    local_attn_size: int = -1  # Window size for temporal local attention (-1 indicates global attention)
    sink_size: int = 0  # Size of the attention sink, we keep the first `sink_size` frames unchanged when rolling the KV cache
@@ -87,7 +87,6 @@ class PipelineConfig:

    # Wan2.2 TI2V parameters
    ti2v_task: bool = False
-    boundary_ratio: float | None = None

    # Compilation
    # enable_torch_compile: bool = False
@@ -82,7 +82,7 @@ class WanI2V480PConfig(WanT2V480PConfig):
        default_factory=CLIPVisionConfig)
    image_encoder_precision: str = "fp32"

-    def __post_init__(self) -> None:
+    def __post_init__(self):
        self.vae_config.load_encoder = True
        self.vae_config.load_decoder = True

@@ -108,17 +108,19 @@ class FastWan2_1_T2V_480P_Config(WanT2V480PConfig):
    dmd_denoising_steps: list[int] | None = field(
        default_factory=lambda: [1000, 757, 522])

+    def __post_init__(self) -> None:
+        self.vae_config.load_encoder = True
+        self.vae_config.load_decoder = True
+

@dataclass
 class Wan2_2_TI2V_5B_Config(WanT2V480PConfig):
    flow_shift: float | None = 5.0
    ti2v_task: bool = True
-    expand_timesteps: bool = True

    def __post_init__(self) -> None:
        self.vae_config.load_encoder = True
        self.vae_config.load_decoder = True
-        self.dit_config.expand_timesteps = self.expand_timesteps


@dataclass
@@ -130,21 +132,12 @@ class FastWan2_2_TI2V_5B_Config(Wan2_2_TI2V_5B_Config):

@dataclass
 class Wan2_2_T2V_A14B_Config(WanT2V480PConfig):
-    flow_shift: float | None = 12.0
-    boundary_ratio: float | None = 0.875
-
-    def __post_init__(self) -> None:
-        self.dit_config.boundary_ratio = self.boundary_ratio
+    pass


@dataclass
-class Wan2_2_I2V_A14B_Config(WanI2V480PConfig):
-    flow_shift: float | None = 5.0
-    boundary_ratio: float | None = 0.900
-
-    def __post_init__(self) -> None:
-        super().__post_init__()
-        self.dit_config.boundary_ratio = self.boundary_ratio
+class Wan2_2_I2V_A14B_Config(WanT2V480PConfig):
+    pass


 # =============================================
@@ -40,7 +40,6 @@ class SamplingParam:
    num_inference_steps: int = 50
    guidance_scale: float = 1.0
    guidance_rescale: float = 0.0
-    boundary_ratio: float | None = None

    # TeaCache parameters
    enable_teacache: bool = False
@@ -170,12 +169,6 @@ class SamplingParam:
            default=SamplingParam.guidance_rescale,
            help="Guidance rescale factor",
        )
-        parser.add_argument(
-            "--boundary-ratio",
-            type=float,
-            default=SamplingParam.boundary_ratio,
-            help="Boundary timestep ratio",
-        )
        parser.add_argument(
            "--save-video",
            action="store_true",
@@ -144,22 +144,18 @@ class Wan2_2_TI2V_5B_SamplingParam(Wan2_2_Base_SamplingParam):

@dataclass
 class Wan2_2_T2V_A14B_SamplingParam(Wan2_2_Base_SamplingParam):
-    guidance_scale: float = 4.0  # high_noise
-    guidance_scale_2: float = 3.0  # low_noise
+    guidance_scale: float = 4.0
+    guidance_scale_2: float = 3.0
    num_inference_steps: int = 40
    fps: int = 16
-    # NOTE(will): default boundary timestep is tracked by PipelineConfig, but
-    # can be overridden during sampling


@dataclass
 class Wan2_2_I2V_A14B_SamplingParam(Wan2_2_Base_SamplingParam):
-    guidance_scale: float = 3.5  # high_noise
-    guidance_scale_2: float = 3.5  # low_noise
+    guidance_scale: float = 3.5
+    guidance_scale_2: float = 3.5
    num_inference_steps: int = 40
    fps: int = 16
-    # NOTE(will): default boundary timestep is tracked by PipelineConfig, but
-    # can be overridden during sampling


 # =============================================
@@ -102,17 +102,3 @@ pyarrow_schema_ode_trajectory_text_only = pa.schema([
    pa.field("caption", pa.string()),
    pa.field("media_type", pa.string()),  # Always 'text' for text-only
 ])
-
-
-pyarrow_schema_text_only = pa.schema([
-    pa.field("id", pa.string()),
-    # --- Text encoder output tensor ---
-    # Tensors are stored as raw bytes with shape and dtype info for loading
-    pa.field("text_embedding_bytes", pa.binary()),
-    # e.g., [SeqLen, Dim]
-    pa.field("text_embedding_shape", pa.list_(pa.int64())),
-    # e.g., 'bfloat16' or 'float32'
-    pa.field("text_embedding_dtype", pa.string()),
-    # --- Metadata ---
-    pa.field("caption", pa.string()),
-])
@@ -175,7 +175,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
    # - "SLIDING_TILE_ATTN" : use Sliding Tile Attention
    # - "VIDEO_SPARSE_ATTN": use Video Sparse Attention
    # - "SAGE_ATTN": use Sage Attention
-    # - "SAGE_ATTN_THREE": use Sage Attention 3
    "FASTVIDEO_ATTENTION_BACKEND":
    lambda: os.getenv("FASTVIDEO_ATTENTION_BACKEND", None),

@@ -158,7 +158,6 @@ class FastVideoArgs:
        "transformer": True,
        "vae": True,
    })
-    override_transformer_cls_name: str | None = None

    # # DMD parameters
    # dmd_denoising_steps: List[int] | None = field(default=None)
@@ -397,12 +396,6 @@ class FastVideoArgs:
            default=FastVideoArgs.enable_stage_verification,
            help="Enable input/output verification for pipeline stages",
        )
-        parser.add_argument(
-            "--override-transformer-cls-name",
-            type=str,
-            default=FastVideoArgs.override_transformer_cls_name,
-            help="Override transformer cls name",
-        )
        # Add pipeline configuration arguments
        PipelineConfig.add_cli_args(parser)

@@ -612,11 +605,6 @@ class TrainingArgs(FastVideoArgs):
    pretrained_model_name_or_path: str = ""
    dit_model_name_or_path: str = ""

-    # DMD model paths - separate paths for each network
-    generator_model_path: str = ""  # path for generator (student) model
-    real_score_model_path: str = ""  # path for real score (teacher) model
-    fake_score_model_path: str = ""  # path for fake score (critic) model
-
    # diffusion setting
    ema_decay: float = 0.0
    ema_start_step: int = 0
@@ -639,7 +627,6 @@ class TrainingArgs(FastVideoArgs):
    checkpoints_total_limit: int = 0
    checkpointing_steps: int = 0
    resume_from_checkpoint: str = ""  # specify the checkpoint folder to resume from
-    init_weights_from_safetensors: str = ""  # path to safetensors file for initial weight loading

    # optimizer & scheduler
    num_train_epochs: int = 0
@@ -671,7 +658,6 @@ class TrainingArgs(FastVideoArgs):
    linear_quadratic_threshold: float = 0.0
    linear_range: float = 0.0
    weight_decay: float = 0.0
-    betas: str = "0.9,0.999"  # betas for optimizer, format: "beta1,beta2"
    use_ema: bool = False
    multi_phased_distill_schedule: str = ""
    pred_decay_weight: float = 0.0
@@ -692,29 +678,16 @@ class TrainingArgs(FastVideoArgs):

    # distillation args
    generator_update_interval: int = 5
-    dfake_gen_update_ratio: int = 5  # self-forcing: how often to train generator vs critic
    min_timestep_ratio: float = 0.2
    max_timestep_ratio: float = 0.98
    real_score_guidance_scale: float = 3.5
    fake_score_learning_rate: float = 0.0  # separate learning rate for fake_score_transformer, if 0.0, use learning_rate
    fake_score_lr_scheduler: str = "constant"  # separate lr scheduler for fake_score_transformer, if not set, use lr_scheduler
-    fake_score_betas: str = "0.9,0.999"  # betas for fake score optimizer, format: "beta1,beta2"
    training_state_checkpointing_steps: int = 0  # for resuming training
    weight_only_checkpointing_steps: int = 0  # for inference
    log_visualization: bool = False
    # simulate generator forward to match inference
    simulate_generator_forward: bool = False
-    warp_denoising_step: bool = False
-
-    # Self-forcing specific arguments
-    num_frame_per_block: int = 3
-    independent_first_frame: bool = False
-    enable_gradient_masking: bool = True
-    gradient_mask_last_n_frames: int = 21
-    validate_cache_structure: bool = False  # Debug flag for cache validation
-    same_step_across_blocks: bool = False  # Use same exit timestep for all blocks
-    last_step_only: bool = False  # Only use the last timestep for training
-    context_noise: int = 0  # Context noise level for cache updates

    @classmethod
    def from_cli_args(cls, args: argparse.Namespace) -> "TrainingArgs":
@@ -816,20 +789,6 @@ class TrainingArgs(FastVideoArgs):
                            type=str,
                            help="Directory to cache models")

-        # DMD model paths - separate paths for each network
-        parser.add_argument(
-            "--generator-model-path",
-            type=str,
-            help="Path to generator (student) model for DMD distillation")
-        parser.add_argument(
-            "--real-score-model-path",
-            type=str,
-            help="Path to real score (teacher) model for DMD distillation")
-        parser.add_argument(
-            "--fake-score-model-path",
-            type=str,
-            help="Path to fake score (critic) model for DMD distillation")
-
        # Diffusion settings
        parser.add_argument("--ema-decay",
                            type=float,
@@ -900,10 +859,6 @@ class TrainingArgs(FastVideoArgs):
        parser.add_argument("--resume-from-checkpoint",
                            type=str,
                            help="Path to checkpoint to resume from")
-        parser.add_argument(
-            "--init-weights-from-safetensors",
-            type=str,
-            help="Path to safetensors file for initial weight loading")
        parser.add_argument("--logging-dir",
                            type=str,
                            help="Directory for logging")
@@ -1008,10 +963,6 @@ class TrainingArgs(FastVideoArgs):
                            help="Linear quadratic threshold")
        parser.add_argument("--linear-range", type=float, help="Linear range")
        parser.add_argument("--weight-decay", type=float, help="Weight decay")
-        parser.add_argument("--betas",
-                            type=str,
-                            default=TrainingArgs.betas,
-                            help="Betas for optimizer (format: 'beta1,beta2')")
        parser.add_argument("--use-ema",
                            action=StoreBoolean,
                            help="Whether to use EMA")
@@ -1062,13 +1013,6 @@ class TrainingArgs(FastVideoArgs):
                            type=int,
                            default=TrainingArgs.generator_update_interval,
                            help="Ratio of student updates to critic updates.")
-        parser.add_argument(
-            "--dfake-gen-update-ratio",
-            type=int,
-            default=TrainingArgs.dfake_gen_update_ratio,
-            help=
-            "Self-forcing: How often to train generator vs critic (train generator every N steps)."
-        )
        parser.add_argument("--min-timestep-ratio",
                            type=float,
                            default=TrainingArgs.min_timestep_ratio,
@@ -1085,11 +1029,6 @@ class TrainingArgs(FastVideoArgs):
                            type=float,
                            default=TrainingArgs.fake_score_learning_rate,
                            help="Learning rate for fake score transformer")
-        parser.add_argument(
-            "--fake-score-betas",
-            type=str,
-            default=TrainingArgs.fake_score_betas,
-            help="Betas for fake score optimizer (format: 'beta1,beta2')")
        parser.add_argument(
            "--fake-score-lr-scheduler",
            type=str,
@@ -1102,48 +1041,6 @@ class TrainingArgs(FastVideoArgs):
            "--simulate-generator-forward",
            action=StoreBoolean,
            help="Whether to simulate generator forward to match inference")
-        parser.add_argument(
-            "--warp-denoising-step",
-            action=StoreBoolean,
-            help=
-            "Whether to warp denoising step according to the scheduler time shift"
-        )
-
-        # Self-forcing specific arguments
-        parser.add_argument(
-            "--num-frame-per-block",
-            type=int,
-            default=TrainingArgs.num_frame_per_block,
-            help="Number of frames per block for causal generation")
-        parser.add_argument(
-            "--independent-first-frame",
-            action=StoreBoolean,
-            help="Whether the first frame is independent in causal generation")
-        parser.add_argument(
-            "--enable-gradient-masking",
-            action=StoreBoolean,
-            help="Whether to enable frame-level gradient masking")
-        parser.add_argument(
-            "--gradient-mask-last-n-frames",
-            type=int,
-            default=TrainingArgs.gradient_mask_last_n_frames,
-            help="Number of last frames to enable gradients for")
-        parser.add_argument(
-            "--validate-cache-structure",
-            action=StoreBoolean,
-            help="Whether to validate KV cache structure (debug flag)")
-        parser.add_argument(
-            "--same-step-across-blocks",
-            action=StoreBoolean,
-            help="Whether to use the same exit timestep for all blocks")
-        parser.add_argument(
-            "--last-step-only",
-            action=StoreBoolean,
-            help="Whether to only use the last timestep for training")
-        parser.add_argument("--context-noise",
-                            type=int,
-                            default=TrainingArgs.context_noise,
-                            help="Context noise level for cache updates")

        return parser

@@ -77,11 +77,9 @@ class BaseLayerWithLoRA(nn.Module):
            lora_A = self.lora_A.to_local()

        if not self.merged and not self.disable_lora:
-            lora_A_sliced = self.slice_lora_a_weights(
-                lora_A.to(x, non_blocking=True))
-            lora_B_sliced = self.slice_lora_b_weights(
-                lora_B.to(x, non_blocking=True))
-            delta = x @ lora_A_sliced.T @ lora_B_sliced.T
+            delta = x @ (
+                self.slice_lora_b_weights(lora_B.to(x, non_blocking=True))
+                @ self.slice_lora_a_weights(lora_A.to(x, non_blocking=True)))
            if self.lora_alpha != self.lora_rank:
                delta = delta * (
                    self.lora_alpha / self.lora_rank  # type: ignore
@@ -147,9 +147,6 @@ class CausalWanSelfAttention(nn.Module):
                # Assign new keys/values directly up to current_end
                local_end_index = kv_cache["local_end_index"].item() + current_end - kv_cache["global_end_index"].item()
                local_start_index = local_end_index - num_new_tokens
-                kv_cache["k"] = kv_cache["k"].detach()
-                kv_cache["v"] = kv_cache["v"].detach()
-                # logger.info("kv_cache['k'] is in comp graph: %s", kv_cache["k"].requires_grad or kv_cache["k"].grad_fn is not None)
                kv_cache["k"][:, local_start_index:local_end_index] = roped_key
                kv_cache["v"][:, local_start_index:local_end_index] = v
            x = self.attn(
@@ -378,8 +375,7 @@ class CausalWanTransformer3DModel(BaseDiT):

        # Causal-specific
        self.block_mask = None
-        self.num_frame_per_block = config.arch_config.num_frames_per_block
-        assert self.num_frame_per_block <= 3
+        self.num_frame_per_block = 1
        self.independent_first_frame = False

        self.__post_init__()
@@ -416,11 +416,6 @@ class TransformerLoader(ComponentLoader):
                "Model config does not contain a _class_name attribute. "
                "Only diffusers format is supported.")

-        logger.info("transformer cls_name: %s", cls_name)
-        if fastvideo_args.override_transformer_cls_name is not None:
-            cls_name = fastvideo_args.override_transformer_cls_name
-            logger.info("Overriding transformer cls_name to %s", cls_name)
-
        fastvideo_args.model_paths["transformer"] = model_path

        # Config from Diffusers supersedes fastvideo's model config
@@ -435,24 +430,8 @@ class TransformerLoader(ComponentLoader):
        if not safetensors_list:
            raise ValueError(f"No safetensors files found in {model_path}")

-        # Check if we should use custom initialization weights
-        custom_weights_path = getattr(fastvideo_args, 'init_weights_from_safetensors', None)
-        use_custom_weights = (custom_weights_path and os.path.exists(custom_weights_path) and 
-                            fastvideo_args.training_mode and 
-                            not hasattr(fastvideo_args, '_loading_teacher_critic_model'))
-
-        if use_custom_weights:
-            logger.info("Using custom initialization weights from: %s", custom_weights_path)
-            assert custom_weights_path is not None, "Custom initialization weights must be provided"
-            if os.path.isdir(custom_weights_path):
-                safetensors_list = glob.glob(
-                    os.path.join(str(custom_weights_path), "*.safetensors"))
-            else:
-                assert custom_weights_path.endswith(".safetensors"), "Custom initialization weights must be a safetensors file"
-                safetensors_list = [custom_weights_path]
-
-        logger.info("Loading model from %s safetensors files: %s",
-                    len(safetensors_list), safetensors_list)
+        logger.info("Loading model from %s safetensors files in %s",
+                    len(safetensors_list), model_path)

        default_dtype = PRECISION_TO_TYPE[
            fastvideo_args.pipeline_config.dit_precision]
@@ -475,7 +454,6 @@ class TransformerLoader(ComponentLoader):
            pin_cpu_memory=fastvideo_args.pin_cpu_memory,
            fsdp_inference=fastvideo_args.use_fsdp_inference,
            # TODO(will): make these configurable
-            default_dtype=default_dtype,
            param_dtype=torch.bfloat16,
            reduce_dtype=torch.float32,
            output_dtype=None,
@@ -485,8 +463,9 @@ class TransformerLoader(ComponentLoader):
        total_params = sum(p.numel() for p in model.parameters())
        logger.info("Loaded model with %.2fB parameters", total_params / 1e9)

-        assert next(model.parameters()).dtype == default_dtype, "Model dtype does not match default dtype"
-
+        dtypes = set(param.dtype for param in model.parameters())
+        if len(dtypes) > 1:
+            model = model.to(default_dtype)
        model = model.eval()
        return model

@@ -62,7 +62,6 @@ def maybe_load_fsdp_model(
    device: torch.device,
    hsdp_replicate_dim: int,
    hsdp_shard_dim: int,
-    default_dtype: torch.dtype,
    param_dtype: torch.dtype,
    reduce_dtype: torch.dtype,
    cpu_offload: bool = False,
@@ -88,8 +87,7 @@ def maybe_load_fsdp_model(
        mp_policy=mp_policy,
    )

-    logger.info("Loading model with default_dtype: %s", default_dtype)
-    with set_default_dtype(default_dtype), torch.device("meta"):
+    with set_default_dtype(param_dtype), torch.device("meta"):
        model = model_cls(**init_params)

    # Check if we should use FSDP
@@ -127,7 +125,7 @@ def maybe_load_fsdp_model(
        model,
        weight_iterator,
        device,
-        default_dtype,
+        param_dtype,
        strict=True,
        cpu_offload=cpu_offload,
        param_names_mapping=param_names_mapping_fn,
@@ -635,31 +635,8 @@ class FlowMatchEulerDiscreteScheduler(SchedulerMixin, ConfigMixin,
        noise: torch.Tensor,
        timestep: torch.IntTensor,
    ) -> torch.Tensor:
-
-        """
-        Args:
-            clean_latent: the clean latent with shape [B, C, H, W],
-                where B is batch_size or batch_size * num_frames
-            noise: the noise with shape [B, C, H, W]
-            timestep: the timestep with shape [1] or [bs * num_frames] or [bs, num_frames]
-
-        Returns:
-            the corrupted latent with shape [B, C, H, W]
-        """
-        # If timestep is [bs, num_frames]
-        if timestep.ndim == 2:
-            timestep = timestep.flatten(0, 1)
-            assert timestep.numel() == clean_latent.shape[0]
-        elif timestep.ndim == 1:
-            # If timestep is [1]
-            if timestep.shape[0] == 1:
-                timestep = timestep.expand(clean_latent.shape[0])
-            else:
-                assert timestep.numel() == clean_latent.shape[0]
-        else:
-            raise ValueError(f"[add_noise] Invalid timestep shape: {timestep.shape}")
-        # timestep shape should be [B]
        self.sigmas = self.sigmas.to(noise.device)
+        timestep = timestep.expand(clean_latent.shape[0])
        self.timesteps = self.timesteps.to(noise.device)
        timestep_id = torch.argmin(
            (self.timesteps.unsqueeze(0) - timestep.unsqueeze(1)).abs(), dim=1)
@@ -22,10 +22,8 @@ class SelfForcingFlowMatchSchedulerOutput(BaseOutput):
    prev_sample: torch.FloatTensor

 class SelfForcingFlowMatchScheduler(BaseScheduler, ConfigMixin, SchedulerMixin):
-    
-    config_name = "scheduler_config.json"
+
    order = 1
-    @register_to_config
    def __init__(self, num_inference_steps=100, num_train_timesteps=1000, shift=3.0, sigma_max=1.0, sigma_min=0.003 / 1.002, inverse_timesteps=False, extra_one_step=False, reverse_sigmas=False, training=False):
        self.num_train_timesteps = num_train_timesteps
        self.shift = shift
@@ -64,15 +62,8 @@ class SelfForcingFlowMatchScheduler(BaseScheduler, ConfigMixin, SchedulerMixin):
    def step(self, model_output: torch.FloatTensor, timestep: torch.FloatTensor, sample: torch.FloatTensor, to_final=False, return_dict=False, **kwargs):
        if timestep.ndim == 2:
            timestep = timestep.flatten(0, 1)
-        elif timestep.ndim == 0:
-            # handles the case where timestep is a scalar, this occurs when we
-            # use this scheduler for ODE trajectory
-            timestep = timestep.unsqueeze(0)
-
        self.sigmas = self.sigmas.to(model_output.device)
        self.timesteps = self.timesteps.to(model_output.device)
-        timestep = timestep.to(model_output.device)
-
        timestep_id = torch.argmin(
            (self.timesteps.unsqueeze(0) - timestep.unsqueeze(1)).abs(), dim=1)
        sigma = self.sigmas[timestep_id].reshape(-1, 1, 1, 1)
@@ -171,10 +171,10 @@ def pred_noise_to_pred_video(pred_noise: torch.Tensor,
    # timestep shape should be [B]
    dtype = pred_noise.dtype
    device = pred_noise.device
-    pred_noise = pred_noise.double().to(device)
-    noise_input_latent = noise_input_latent.double().to(device)
-    sigmas = scheduler.sigmas.double().to(device)
-    timesteps = scheduler.timesteps.double().to(device)
+    pred_noise = pred_noise.float().to(device)
+    noise_input_latent = noise_input_latent.float().to(device)
+    sigmas = scheduler.sigmas.float().to(device)
+    timesteps = scheduler.timesteps.float().to(device)
    timestep_id = torch.argmin(
        (timesteps.unsqueeze(0) - timestep.unsqueeze(1)).abs(), dim=1)
    sigma_t = sigmas[timestep_id].reshape(-1, 1, 1, 1)
@@ -7,6 +7,8 @@ This module wires the causal DMD denoising stage into the modular pipeline.

 from fastvideo.fastvideo_args import FastVideoArgs
 from fastvideo.logger import init_logger
+from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import (
+    FlowMatchEulerDiscreteScheduler)
 from fastvideo.pipelines import ComposedPipelineBase, LoRAPipeline

 # isort: off
@@ -26,6 +28,10 @@ class WanCausalDMDPipeline(LoRAPipeline, ComposedPipelineBase):
        "text_encoder", "tokenizer", "vae", "transformer", "scheduler"
    ]

+    def initialize_pipeline(self, fastvideo_args: FastVideoArgs):
+        self.modules["scheduler"] = FlowMatchEulerDiscreteScheduler(
+            shift=fastvideo_args.pipeline_config.flow_shift)
+
    def create_pipeline_stages(self, fastvideo_args: FastVideoArgs) -> None:
        """Set up pipeline stages with proper dependency injection."""

@@ -237,19 +237,20 @@ class ComposedPipelineBase(ABC):
        # remove keys that are not pipeline modules
        model_index.pop("_class_name")
        model_index.pop("_diffusers_version")
+        # @TODO(Wei): Temporary hack
        if "boundary_ratio" in model_index and model_index[
                "boundary_ratio"] is not None:
            logger.info(
                "MoE pipeline detected. Adding transformer_2 to self.required_config_modules..."
            )
            self.required_config_modules.append("transformer_2")
-            logger.info("MoE pipeline detected. Setting boundary ratio to %s",
-                        model_index["boundary_ratio"])
-            fastvideo_args.pipeline_config.dit_config.boundary_ratio = model_index[
-                "boundary_ratio"]
+            if fastvideo_args.boundary_ratio is None:
+                logger.info(
+                    "MoE pipeline detected. Setting boundary ratio to %s",
+                    model_index["boundary_ratio"])
+                fastvideo_args.boundary_ratio = model_index["boundary_ratio"]

        model_index.pop("boundary_ratio", None)
-        # used by Wan2.2 ti2v
        model_index.pop("expand_timesteps", None)

        # some sanity checks
@@ -282,8 +283,8 @@ class ComposedPipelineBase(ABC):
                          architecture) in model_index.items():
            if transformers_or_diffusers is None:
                logger.warning(
-                    "Module %s in model_index.json has null value, removing from required_config_modules",
-                    module_name)
+                    "Module in model_index.json has null value, removing from required_config_modules"
+                )
                if module_name in self.required_config_modules:
                    self.required_config_modules.remove(module_name)
                continue
@@ -129,7 +129,6 @@ class ForwardBatch:
    timesteps: torch.Tensor | None = None
    timestep: torch.Tensor | float | int | None = None
    step_index: int | None = None
-    boundary_ratio: float | None = None

    # Scheduler parameters
    num_inference_steps: int = 50
@@ -246,7 +245,6 @@ class TrainingBatch:
    fake_score_loss: float = 0.0

    dmd_latent_vis_dict: dict[str, Any] = field(default_factory=dict)
-    latent_vis_dict: dict[str, Any] = field(default_factory=dict)
    fake_score_latent_vis_dict: dict[str, Any] = field(default_factory=dict)


@@ -10,8 +10,6 @@ from torch.utils.data import DataLoader
 from tqdm import tqdm

 from fastvideo.dataset import getdataset
-from fastvideo.dataset.dataloader.parquet_io import (ParquetDatasetWriter,
-                                                     records_to_table)
 from fastvideo.dataset.preprocessing_datasets import PreprocessBatch
 from fastvideo.distributed import get_local_torch_device
 from fastvideo.fastvideo_args import FastVideoArgs
@@ -19,6 +17,8 @@ from fastvideo.logger import init_logger
 from fastvideo.pipelines.composed_pipeline_base import ComposedPipelineBase
 from fastvideo.pipelines.pipeline_batch_info import ForwardBatch
 from fastvideo.pipelines.stages import TextEncodingStage
+from fastvideo.workflow.preprocess.parquet_io import (ParquetDatasetWriter,
+                                                      records_to_table)

 logger = init_logger(__name__)

@@ -423,3 +423,7 @@ class BasePreprocessPipeline(ComposedPipelineBase):
                written = self.dataset_writer.flush()
                logger.info("Flushed %s samples to parquet", written)
                num_processed_samples = 0
+
+    def _final_flush_if_any(self):
+        if hasattr(self, 'dataset_writer'):
+            self.dataset_writer.flush()
@@ -12,6 +12,7 @@ import os
 from collections.abc import Iterator
 from typing import Any

+import numpy as np
 import pyarrow as pa
 import torch
 from torch.utils.data import DataLoader
@@ -20,10 +21,6 @@ from tqdm import tqdm

 from fastvideo.configs.sample import SamplingParam
 from fastvideo.dataset import gettextdataset
-from fastvideo.dataset.dataloader.parquet_io import (ParquetDatasetWriter,
-                                                     records_to_table)
-from fastvideo.dataset.dataloader.record_schema import (
-    ode_text_only_record_creator)
 from fastvideo.dataset.dataloader.schema import (
    pyarrow_schema_ode_trajectory_text_only)
 from fastvideo.fastvideo_args import FastVideoArgs
@@ -39,6 +36,8 @@ from fastvideo.pipelines.stages import (DecodingStage, DenoisingStage,
                                        TextEncodingStage,
                                        TimestepPreparationStage)
 from fastvideo.utils import save_decoded_latents_as_video, shallow_asdict
+from fastvideo.workflow.preprocess.parquet_io import (ParquetDatasetWriter,
+                                                      records_to_table)

 logger = init_logger(__name__)

@@ -60,13 +59,26 @@ class PreprocessPipeline_ODE_Trajectory(BasePreprocessPipeline):

    def create_pipeline_stages(self, fastvideo_args: FastVideoArgs):
        """Set up pipeline stages with proper dependency injection."""
+        logger.info('WTF flow_shift: %s',
+                    fastvideo_args.pipeline_config.flow_shift)
        assert fastvideo_args.pipeline_config.flow_shift == 5
+        # self.modules["scheduler"] = FlowMatchEulerDiscreteScheduler(
+        # shift=fastvideo_args.pipeline_config.flow_shift)
        self.modules["scheduler"] = SelfForcingFlowMatchScheduler(
            shift=fastvideo_args.pipeline_config.flow_shift,
            sigma_min=0.0,
            extra_one_step=True)
        self.modules["scheduler"].set_timesteps(num_inference_steps=48,
                                                denoising_strength=1.0)
+        # logger.info('WTF scheduler timesteps: %s',
+        #             self.modules["scheduler"].timesteps)
+        # scheduler = FlowMatchScheduler(
+        #     shift=8.0, sigma_min=0.0, extra_one_step=True)
+        # device = get_local_torch_device()
+        # # scheduler.num_train_timesteps = 100
+        # scheduler.set_timesteps(num_inference_steps=50, denoising_strength=1.0)
+        # scheduler.sigmas = scheduler.sigmas.to(device)
+        # self.modules["scheduler"] = scheduler

        self.add_stage(stage_name="input_validation_stage",
                       stage=InputValidationStage())
@@ -174,8 +186,6 @@ class PreprocessPipeline_ODE_Trajectory(BasePreprocessPipeline):
                    ]
                    batch.num_inference_steps = 48
                    batch.return_trajectory_latents = True
-                    # Enabling this will save the decoded trajectory videos.
-                    # Used for debugging.
                    batch.return_trajectory_decoded = False
                    batch.height = args.max_height
                    batch.width = args.max_width
@@ -215,7 +225,7 @@ class PreprocessPipeline_ODE_Trajectory(BasePreprocessPipeline):
                                args.train_fps)

                # Prepare batch data for Parquet dataset
-                batch_data: list[dict[str, Any]] = []
+                batch_data = []

                # Add progress bar for saving outputs
                save_pbar = tqdm(enumerate(valid_data["path"]),
@@ -244,16 +254,14 @@ class PreprocessPipeline_ODE_Trajectory(BasePreprocessPipeline):
                                else:
                                    sample_extra_features[key] = value[idx]

-                    # Create record for Parquet dataset (text-only ODE schema)
-                    record: dict[str, Any] = ode_text_only_record_creator(
+                    # Create record for Parquet dataset (without VAE latents for text-only)
+                    record = self.create_text_only_record(
+                        args,
                        video_name=video_name,
                        text_embedding=text_embedding,
-                        caption=valid_data["text"][idx],
-                        trajectory_latents=sample_extra_features[
-                            "trajectory_latents"],
-                        trajectory_timesteps=sample_extra_features[
-                            "trajectory_timesteps"],
-                    )
+                        valid_data=valid_data,
+                        idx=idx,
+                        extra_features=sample_extra_features)
                    batch_data.append(record)

                if batch_data:
@@ -281,10 +289,78 @@ class PreprocessPipeline_ODE_Trajectory(BasePreprocessPipeline):

        # Final flush for any remaining samples
        if hasattr(self, 'dataset_writer'):
-            written = self.dataset_writer.flush(write_remainder=True)
+            written = self.dataset_writer.flush()
            if written:
                logger.info("Final flush wrote %s samples", written)

+    def create_text_only_record(
+            self,
+            args,
+            video_name: str,
+            text_embedding: np.ndarray,
+            valid_data: dict[str, Any],
+            idx: int,
+            extra_features: dict[str, Any] | None = None) -> dict[str, Any]:
+        """Create a record for text-only preprocessing using text-only schema."""
+
+        # Create base record using only fields from text-only schema
+        record = {
+            "id": f"text_{video_name}_{idx}",
+            "text_embedding_bytes": text_embedding.tobytes(),
+            "text_embedding_shape": list(text_embedding.shape),
+            "text_embedding_dtype": str(text_embedding.dtype),
+            "file_name": video_name,
+            "caption": valid_data["text"][idx],
+            "media_type": "text",
+        }
+
+        assert extra_features is not None, "extra_features is required"
+        assert "trajectory_latents" in extra_features, "trajectory_latents is required"
+        assert "trajectory_timesteps" in extra_features, "trajectory_timesteps is required"
+
+        # Add trajectory data if available
+        if extra_features and "trajectory_latents" in extra_features:
+            trajectory_latents = extra_features[
+                "trajectory_latents"][idx] if isinstance(
+                    extra_features["trajectory_latents"],
+                    list) else extra_features["trajectory_latents"]
+            record.update({
+                "trajectory_latents_bytes":
+                trajectory_latents.tobytes(),
+                "trajectory_latents_shape":
+                list(trajectory_latents.shape),
+                "trajectory_latents_dtype":
+                str(trajectory_latents.dtype),
+            })
+        else:
+            record.update({
+                "trajectory_latents_bytes": b"",
+                "trajectory_latents_shape": [],
+                "trajectory_latents_dtype": "",
+            })
+
+        if extra_features and "trajectory_timesteps" in extra_features:
+            trajectory_timesteps = extra_features[
+                "trajectory_timesteps"][idx] if isinstance(
+                    extra_features["trajectory_timesteps"],
+                    list) else extra_features["trajectory_timesteps"]
+            record.update({
+                "trajectory_timesteps_bytes":
+                trajectory_timesteps.tobytes(),
+                "trajectory_timesteps_shape":
+                list(trajectory_timesteps.shape),
+                "trajectory_timesteps_dtype":
+                str(trajectory_timesteps.dtype),
+            })
+        else:
+            record.update({
+                "trajectory_timesteps_bytes": b"",
+                "trajectory_timesteps_shape": [],
+                "trajectory_timesteps_dtype": "",
+            })
+
+        return record
+
    def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs, args):
        if not self.post_init_called:
            self.post_init()
@@ -1,184 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-"""
-Text-only Data Preprocessing pipeline implementation.
-
-This module contains an implementation of the Text-only Data Preprocessing pipeline
-using the modular pipeline architecture, based on the ODE Trajectory preprocessing.
-"""
-
-import os
-from collections.abc import Iterator
-from typing import Any
-
-import torch
-from torch.utils.data import DataLoader
-from torchdata.stateful_dataloader import StatefulDataLoader
-from tqdm import tqdm
-
-from fastvideo.dataset import gettextdataset
-from fastvideo.dataset.dataloader.parquet_io import (ParquetDatasetWriter,
-                                                     records_to_table)
-from fastvideo.dataset.dataloader.record_schema import text_only_record_creator
-from fastvideo.dataset.dataloader.schema import pyarrow_schema_text_only
-from fastvideo.fastvideo_args import FastVideoArgs
-from fastvideo.logger import init_logger
-from fastvideo.pipelines.pipeline_batch_info import ForwardBatch
-from fastvideo.pipelines.preprocess.preprocess_pipeline_base import (
-    BasePreprocessPipeline)
-from fastvideo.pipelines.stages import TextEncodingStage
-
-logger = init_logger(__name__)
-
-
-class PreprocessPipeline_Text(BasePreprocessPipeline):
-    """Text-only preprocessing pipeline implementation."""
-
-    _required_config_modules = ["text_encoder", "tokenizer"]
-    preprocess_dataloader: StatefulDataLoader
-    preprocess_loader_iter: Iterator[dict[str, Any]]
-    pbar: Any
-    num_processed_samples: int = 0
-
-    def get_pyarrow_schema(self):
-        """Return the PyArrow schema for text-only pipeline."""
-        return pyarrow_schema_text_only
-
-    def create_pipeline_stages(self, fastvideo_args: FastVideoArgs):
-        """Set up pipeline stages with proper dependency injection."""
-        self.add_stage(stage_name="prompt_encoding_stage",
-                       stage=TextEncodingStage(
-                           text_encoders=[self.get_module("text_encoder")],
-                           tokenizers=[self.get_module("tokenizer")],
-                       ))
-
-    def preprocess_text_only(self, fastvideo_args: FastVideoArgs, args):
-        """Preprocess text-only data."""
-
-        for batch_idx, data in enumerate(self.pbar):
-            if data is None:
-                continue
-
-            with torch.inference_mode():
-                # For text-only processing, we only need text data
-                # Filter out samples without text
-                valid_indices = []
-                for i, text in enumerate(data["text"]):
-                    if text and text.strip():  # Check if text is not empty
-                        valid_indices.append(i)
-                self.num_processed_samples += len(valid_indices)
-
-                if not valid_indices:
-                    continue
-
-                # Create new batch with only valid samples (text-only)
-                valid_data = {
-                    "text": [data["text"][i] for i in valid_indices],
-                    "path": [data["path"][i] for i in valid_indices],
-                }
-
-                batch_captions = valid_data["text"]
-                # Encode text using the standalone TextEncodingStage API
-                prompt_embeds_list, prompt_masks_list = self.prompt_encoding_stage.encode_text(
-                    batch_captions,
-                    fastvideo_args,
-                    encoder_index=[0],
-                    return_attention_mask=True,
-                )
-                prompt_embeds = prompt_embeds_list[0]
-                prompt_attention_masks = prompt_masks_list[0]
-                assert prompt_embeds.shape[0] == prompt_attention_masks.shape[0]
-
-                logger.info("===== prompt_embeds: %s", prompt_embeds.shape)
-                logger.info("===== prompt_attention_masks: %s",
-                            prompt_attention_masks.shape)
-
-                # Prepare batch data for Parquet dataset
-                batch_data = []
-
-                # Add progress bar for saving outputs
-                save_pbar = tqdm(enumerate(valid_data["path"]),
-                                 desc="Saving outputs",
-                                 unit="item",
-                                 leave=False)
-
-                for idx, text_path in save_pbar:
-                    text_name = os.path.basename(text_path).split(".")[0]
-
-                    # Convert tensors to numpy arrays
-                    text_embedding = prompt_embeds[idx].cpu().numpy()
-
-                    # Create record for Parquet dataset (text-only schema)
-                    record = text_only_record_creator(
-                        text_name=text_name,
-                        text_embedding=text_embedding,
-                        caption=valid_data["text"][idx],
-                    )
-                    batch_data.append(record)
-
-                if batch_data:
-                    write_pbar = tqdm(total=1,
-                                      desc="Writing to Parquet dataset",
-                                      unit="batch")
-                    table = records_to_table(batch_data,
-                                             pyarrow_schema_text_only)
-                    write_pbar.update(1)
-                    write_pbar.close()
-
-                    if not hasattr(self, 'dataset_writer'):
-                        self.dataset_writer = ParquetDatasetWriter(
-                            out_dir=self.combined_parquet_dir,
-                            samples_per_file=args.samples_per_file,
-                        )
-                    self.dataset_writer.append_table(table)
-
-                    logger.info("Collected batch with %s samples", len(table))
-
-                if self.num_processed_samples >= args.flush_frequency:
-                    written = self.dataset_writer.flush()
-                    logger.info("Flushed %s samples to parquet", written)
-                    self.num_processed_samples = 0
-
-        # Final flush for any remaining samples
-        if hasattr(self, 'dataset_writer'):
-            written = self.dataset_writer.flush(write_remainder=True)
-            if written:
-                logger.info("Final flush wrote %s samples", written)
-
-    # Text-only record creation moved to fastvideo.dataset.dataloader.record_schema
-
-    def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs, args):
-        if not self.post_init_called:
-            self.post_init()
-
-        self.local_rank = int(os.getenv("RANK", 0))
-        os.makedirs(args.output_dir, exist_ok=True)
-        # Create directory for combined data
-        self.combined_parquet_dir = os.path.join(args.output_dir,
-                                                 "combined_parquet_dataset")
-        os.makedirs(self.combined_parquet_dir, exist_ok=True)
-
-        # Loading text dataset
-        train_dataset = gettextdataset(args)
-
-        self.preprocess_dataloader = DataLoader(
-            train_dataset,
-            batch_size=args.preprocess_video_batch_size,
-            num_workers=args.dataloader_num_workers,
-        )
-
-        self.preprocess_loader_iter = iter(self.preprocess_dataloader)
-
-        self.num_processed_samples = 0
-        # Add progress bar for text preprocessing
-        self.pbar = tqdm(self.preprocess_loader_iter,
-                         desc="Processing text",
-                         unit="batch",
-                         disable=self.local_rank != 0)
-
-        # Initialize class variables for data sharing
-        self.text_data: dict[str, Any] = {}  # Store text metadata and paths
-
-        self.preprocess_text_only(fastvideo_args, args)
-
-
-EntryClass = PreprocessPipeline_Text
@@ -1,6 +1,5 @@
 import argparse
 import os
-from typing import Any

 from fastvideo import PipelineConfig
 from fastvideo.configs.models.vaes import WanVAEConfig
@@ -14,8 +13,6 @@ from fastvideo.pipelines.preprocess.preprocess_pipeline_ode_trajectory import (
    PreprocessPipeline_ODE_Trajectory)
 from fastvideo.pipelines.preprocess.preprocess_pipeline_t2v import (
    PreprocessPipeline_T2V)
-from fastvideo.pipelines.preprocess.preprocess_pipeline_text import (
-    PreprocessPipeline_Text)
 from fastvideo.utils import maybe_download_model

 logger = init_logger(__name__)
@@ -26,22 +23,13 @@ def main(args) -> None:
    maybe_init_distributed_environment_and_model_parallel(1, 1)
    num_gpus = int(os.environ["WORLD_SIZE"])
    assert num_gpus == 1, "Only support 1 GPU"
-
    pipeline_config = PipelineConfig.from_pretrained(args.model_path)
-
-    kwargs: dict[str, Any] = {}
-    if args.preprocess_task == "text_only":
-        kwargs = {
-            "text_encoder_cpu_offload": False,
-        }
-    else:
-        # Full config for video/image processing
-        kwargs = {
-            "vae_precision": "fp32",
-            "vae_config": WanVAEConfig(load_encoder=True, load_decoder=True),
-        }
+    kwargs = {
+        "vae_precision": "fp32",
+        "vae_config": WanVAEConfig(load_encoder=True, load_decoder=True),
+        "flow_shift": 5,
+    }
    pipeline_config.update_config_from_dict(kwargs)
-
    fastvideo_args = FastVideoArgs(
        model_path=args.model_path,
        num_gpus=get_world_size(),
@@ -54,19 +42,13 @@ def main(args) -> None:
        PreprocessPipeline = PreprocessPipeline_T2V
    elif args.preprocess_task == "i2v":
        PreprocessPipeline = PreprocessPipeline_I2V
-    elif args.preprocess_task == "text_only":
-        PreprocessPipeline = PreprocessPipeline_Text
    elif args.preprocess_task == "ode_trajectory":
-        assert args.flow_shift is not None, "flow_shift is required for ode_trajectory"
-        fastvideo_args.pipeline_config.flow_shift = args.flow_shift
        PreprocessPipeline = PreprocessPipeline_ODE_Trajectory
    else:
-        raise ValueError(f"Invalid preprocess task: {args.preprocess_task}. "
-                         f"Valid options: t2v, i2v, ode_trajectory, text_only")
+        raise ValueError(f"Invalid preprocess task: {args.preprocess_task}")

    logger.info("Preprocess task: %s using %s", args.preprocess_task,
                PreprocessPipeline.__name__)
-
    pipeline = PreprocessPipeline(args.model_path, fastvideo_args)
    pipeline.forward(batch=None, fastvideo_args=fastvideo_args, args=args)

@@ -105,12 +87,7 @@ if __name__ == "__main__":
    parser.add_argument("--video_length_tolerance_range", type=int, default=2.0)
    parser.add_argument("--group_frame", action="store_true")  # TODO
    parser.add_argument("--group_resolution", action="store_true")  # TODO
-    parser.add_argument("--flow_shift", type=float, default=None)
-    parser.add_argument("--preprocess_task",
-                        type=str,
-                        default="t2v",
-                        choices=["t2v", "i2v", "text_only", "ode_trajectory"],
-                        help="Type of preprocessing task to run")
+    parser.add_argument("--preprocess_task", type=str, default="t2v")
    parser.add_argument("--train_fps", type=int, default=30)
    parser.add_argument("--use_image_num", type=int, default=0)
    parser.add_argument("--text_max_length", type=int, default=256)
@@ -85,9 +85,8 @@ class DenoisingStage(PipelineStage):
                AttentionBackendEnum.SLIDING_TILE_ATTN,
                AttentionBackendEnum.VIDEO_SPARSE_ATTN,
                AttentionBackendEnum.VMOBA_ATTN,
-                AttentionBackendEnum.FLASH_ATTN,
-                AttentionBackendEnum.TORCH_SDPA,
-                AttentionBackendEnum.SAGE_ATTN_THREE)  # hack
+                AttentionBackendEnum.FLASH_ATTN, AttentionBackendEnum.TORCH_SDPA
+            )  # hack
        )

    def forward(
@@ -205,14 +204,8 @@ class DenoisingStage(PipelineStage):
                neg_prompt_embeds[0]).any(), "neg_prompt_embeds contains nan"

        # (Wan2.2) Calculate timestep to switch from high noise expert to low noise expert
-        boundary_ratio = fastvideo_args.pipeline_config.dit_config.boundary_ratio
-        if batch.boundary_ratio is not None:
-            logger.info("Overriding boundary ratio from %s to %s",
-                        boundary_ratio, batch.boundary_ratio)
-            boundary_ratio = batch.boundary_ratio
-
-        if boundary_ratio is not None:
-            boundary_timestep = boundary_ratio * self.scheduler.num_train_timesteps
+        if fastvideo_args.boundary_ratio is not None:
+            boundary_timestep = fastvideo_args.boundary_ratio * self.scheduler.num_train_timesteps
        else:
            boundary_timestep = None
        latent_model_input = latents.to(target_dtype)
@@ -115,7 +115,6 @@ class CudaPlatformBase(Platform):

        logger.info("Trying FASTVIDEO_ATTENTION_BACKEND=%s",
                    envs.FASTVIDEO_ATTENTION_BACKEND)
-        logger.info("Selected backend: %s", selected_backend)
        if selected_backend == AttentionBackendEnum.SLIDING_TILE_ATTN:
            try:
                from st_attn import sliding_tile_attention  # noqa: F401
@@ -145,20 +144,6 @@ class CudaPlatformBase(Platform):
                logger.info(
                    "Sage Attention backend is not installed. Fall back to Flash Attention."
                )
-        elif selected_backend == AttentionBackendEnum.SAGE_ATTN_THREE:
-            try:
-                from fastvideo.attention.backends.sageattn.api import sageattn_blackwell  # noqa: F401
-
-                from fastvideo.attention.backends.sage_attn3 import (  # noqa: F401
-                    SageAttention3Backend)
-                logger.info("Using Sage Attention 3 backend.")
-
-                return "fastvideo.attention.backends.sage_attn3.SageAttention3Backend"
-            except ImportError as e:
-                logger.info(e)
-                logger.info(
-                    "Sage Attention 3 backend is not installed. Fall back to Flash Attention."
-                )
        elif selected_backend == AttentionBackendEnum.VIDEO_SPARSE_ATTN:
            try:
                from vsa import block_sparse_attn  # noqa: F401
@@ -18,7 +18,6 @@ class AttentionBackendEnum(enum.Enum):
    SLIDING_TILE_ATTN = enum.auto()
    TORCH_SDPA = enum.auto()
    SAGE_ATTN = enum.auto()
-    SAGE_ATTN_THREE = enum.auto()
    VIDEO_SPARSE_ATTN = enum.auto()
    VMOBA_ATTN = enum.auto()
    NO_ATTENTION = enum.auto()
@@ -6,7 +6,7 @@ import pytest

 from fastvideo import VideoGenerator
 from fastvideo.logger import init_logger
-from tests.utils import compute_video_ssim_torchvision, write_ssim_results
+from fastvideo.tests.utils import compute_video_ssim_torchvision, write_ssim_results
 from diffusers import DiffusionPipeline
 from fastvideo.fastvideo_args import FastVideoArgs
 from fastvideo.pipelines import build_pipeline
@@ -192,3 +192,4 @@ def test_lora_inference_similarity(ATTENTION_BACKEND, model_id):



+
@@ -64,35 +64,35 @@ def run_test(pytest_command: str):

@app.function(gpu="L40S:1", image=image, timeout=900)
 def run_encoder_tests():
-    run_test("pytest ./tests/encoders -vs")
+    run_test("pytest ./fastvideo/tests/encoders -vs")

@app.function(gpu="L40S:1", image=image, timeout=900)
 def run_vae_tests():
-    run_test("pytest ./tests/vaes -vs")
+    run_test("pytest ./fastvideo/tests/vaes -vs")

@app.function(gpu="L40S:1", image=image, timeout=900)
 def run_transformer_tests():
-    run_test("pytest ./tests/transformers -vs")
+    run_test("pytest ./fastvideo/tests/transformers -vs")

@app.function(gpu="L40S:2", image=image, timeout=2700)
 def run_ssim_tests():
-    run_test("pytest ./tests/ssim -vs")
+    run_test("pytest ./fastvideo/tests/ssim -vs")

@app.function(gpu="L40S:4", image=image, timeout=900, secrets=[modal.Secret.from_dict({"WANDB_API_KEY": os.environ.get("WANDB_API_KEY", "")})])
 def run_training_tests():
-    run_test("wandb login $WANDB_API_KEY && pytest ./tests/training/Vanilla -srP")
+    run_test("wandb login $WANDB_API_KEY && pytest ./fastvideo/tests/training/Vanilla -srP")

@app.function(gpu="L40S:2", image=image, timeout=900, secrets=[modal.Secret.from_dict({"WANDB_API_KEY": os.environ.get("WANDB_API_KEY", "")})])
 def run_training_lora_tests():
-    run_test("wandb login $WANDB_API_KEY && pytest ./tests/training/lora/test_lora_training.py -srP")
+    run_test("wandb login $WANDB_API_KEY && pytest ./fastvideo/tests/training/lora/test_lora_training.py -srP")

@app.function(gpu="H100:2", image=image, timeout=900, secrets=[modal.Secret.from_dict({"WANDB_API_KEY": os.environ.get("WANDB_API_KEY", "")})])
 def run_training_tests_VSA():
-    run_test("wandb login $WANDB_API_KEY && pytest ./tests/training/VSA -srP")
+    run_test("wandb login $WANDB_API_KEY && pytest ./fastvideo/tests/training/VSA -srP")

@app.function(gpu="H100:2", image=image, timeout=900)
 def run_inference_tests_STA():
-    run_test("pytest ./tests/inference/STA -srP")
+    run_test("pytest ./fastvideo/tests/inference/STA -srP")

@app.function(gpu="H100:1", image=image, timeout=900)
 def run_precision_tests_STA():
@@ -108,16 +108,12 @@ def run_precision_tests_vmoba():

@app.function(gpu="L40S:1", image=image, timeout=900)
 def run_inference_tests_vmoba():
-    run_test('python tests/inference/vmoba/test_vmoba_inference.py')
+    run_test('python fastvideo/tests/inference/vmoba/test_vmoba_inference.py')

@app.function(gpu="L40S:1", image=image, timeout=3600)
 def run_inference_lora_tests():
-    run_test("pytest ./tests/inference/lora/test_lora_inference_similarity.py -vs")
+    run_test("pytest ./fastvideo/tests/inference/lora/test_lora_inference_similarity.py -vs")

@app.function(gpu="L40S:2", image=image, timeout=900)
 def run_distill_dmd_tests():
-    run_test("pytest ./tests/training/distill/test_distill_dmd.py -vs")
-
-@app.function(gpu="L40S:1", image=image, timeout=900)
-def run_unit_test():
-    run_test("pytest ./tests/dataset/ ./tests/workflow/ -vs")
+    run_test("pytest ./fastvideo/tests/training/distill/test_distill_dmd.py -vs")
@@ -4,7 +4,7 @@ from huggingface_hub import snapshot_download
 import shutil
 import subprocess
 import sys
-from tests.ssim.test_inference_similarity import compute_video_ssim_torchvision
+from fastvideo.tests.ssim.test_inference_similarity import compute_video_ssim_torchvision

 # Import the training pipeline
 sys.path.append(str(Path(__file__).parent.parent.parent.parent.parent))
@@ -169,4 +169,4 @@ def test_e2e_overfit_single_sample():


 if __name__ == "__main__":
-    test_e2e_overfit_single_sample()
+    test_e2e_overfit_single_sample()
@@ -4,7 +4,7 @@ from huggingface_hub import snapshot_download
 import shutil
 import subprocess
 import sys
-from tests.ssim.test_inference_similarity import compute_video_ssim_torchvision
+from fastvideo.tests.ssim.test_inference_similarity import compute_video_ssim_torchvision

 # Import the training pipeline
 sys.path.append(str(Path(__file__).parent.parent.parent.parent.parent))
@@ -62,11 +62,6 @@ def download_data():


 def run_preprocessing():
-    # remove the local_preprocessed_data_dir if it exists
-    if LOCAL_PREPROCESSED_DATA_DIR.exists():
-        print(f"Removing local_preprocessed_data_dir: {LOCAL_PREPROCESSED_DATA_DIR}")
-        shutil.rmtree(LOCAL_PREPROCESSED_DATA_DIR)
-
    # Run torchrun command
    cmd = [
        "torchrun",
@@ -174,4 +169,4 @@ def test_e2e_overfit_single_sample():


 if __name__ == "__main__":
-    test_e2e_overfit_single_sample()
+    test_e2e_overfit_single_sample()
@@ -10,10 +10,10 @@ torchrun --standalone --nnodes=1 --nproc_per_node=$num_gpus \
 num_gpus=2
 torchrun --standalone --nnodes=1 --nproc_per_node=$num_gpus \
    --master_port 29503  \
-    tests/test_hunyuanvideo_load.py --sequence_model_parallel_size $num_gpus 
+    fastvideo/tests/test_hunyuanvideo_load.py --sequence_model_parallel_size $num_gpus 

-torchrun --nnodes=1 --nproc_per_node=1 --master_port 29503  tests/test_llama_encoder.py
+torchrun --nnodes=1 --nproc_per_node=1 --master_port 29503  fastvideo/tests/test_llama_encoder.py


 export FASTVIDEO_ATTENTION_BACKEND=FLASH_ATTN
-torchrun --nnodes=1 --nproc_per_node=1 --master_port 29503  tests/test_clip_encoder.py
+torchrun --nnodes=1 --nproc_per_node=1 --master_port 29503  fastvideo/tests/test_clip_encoder.py
@@ -2,7 +2,7 @@ The reference videos in the `*_reference_videos` directory are used as part of a

 `A40_reference_videos` are generated on A40s and so on.

-run `bash update_reference_videos.sh` from inside the `tests/ssim/` directory after running `test_inference_similarity.py` to update reference videos. Note: make sure to update the path to the corresponding device.
+run `bash update_reference_videos.sh` from inside the `fastvideo/tests/ssim/` directory after running `test_inference_similarity.py` to update reference videos. Note: make sure to update the path to the corresponding device.

 reference videos were generated on commit `4aeabbc629e0edf91477e80e795e7bb1823c71cb`
 causal videos were generated on commit b318063c0a4618f1d5d99ea82ca67a06aad0d19d
@@ -7,7 +7,7 @@ import pytest

 from fastvideo import VideoGenerator
 from fastvideo.logger import init_logger
-from tests.utils import compute_video_ssim_torchvision, write_ssim_results
+from fastvideo.tests.utils import compute_video_ssim_torchvision, write_ssim_results
 from fastvideo.worker.multiproc_executor import MultiprocExecutor

 logger = init_logger(__name__)
@@ -7,7 +7,7 @@ import pytest

 from fastvideo import VideoGenerator
 from fastvideo.logger import init_logger
-from tests.utils import compute_video_ssim_torchvision, write_ssim_results
+from fastvideo.tests.utils import compute_video_ssim_torchvision, write_ssim_results
 from fastvideo.worker.multiproc_executor import MultiprocExecutor

 logger = init_logger(__name__)
@@ -0,0 +1 @@
+{"step_time":0.6983645600266755,"_wandb":{"runtime":107},"grad_norm":0.50390625,"avg_step_time":1.002151239803061,"_step":5,"validation_videos_50_steps":{"captions":false,"_type":"videos","count":8,"videos":[{"size":159131,"path":"media/videos/validation_videos_50_steps_0_dc447599dbe48350e9c9.mp4","_type":"video-file","sha256":"dc447599dbe48350e9c920f4971e1786bde580dd20d52b4aa147ae8d3dc564d6"},{"_type":"video-file","sha256":"4e283876ddfbf5a2cb6f8aca07a39f832b5f806fbefde8854bc73ed904ff20ee","size":160315,"path":"media/videos/validation_videos_50_steps_0_4e283876ddfbf5a2cb6f.mp4"},{"size":135225,"path":"media/videos/validation_videos_50_steps_0_78185c41e1935306e93c.mp4","_type":"video-file","sha256":"78185c41e1935306e93c2d416ee40b31d038abffb029cb5bfb11c2a634eb2fcf"},{"_type":"video-file","sha256":"27e9819d002d3f63c8918bbdc5bf2857b5effe0caf5d5b8374b9c590fc6432eb","size":197873,"path":"media/videos/validation_videos_50_steps_0_27e9819d002d3f63c891.mp4"},{"_type":"video-file","sha256":"46fe548e86144ca60a9396fcd15e8788d3a05c066c6819ccdd6a9041feaaec8f","size":170601,"path":"media/videos/validation_videos_50_steps_0_46fe548e86144ca60a93.mp4"},{"sha256":"91ec338774bec870b9c5c81be4330a8f0f2535124cb372e881c3703e6b65ed77","size":164462,"path":"media/videos/validation_videos_50_steps_0_91ec338774bec870b9c5.mp4","_type":"video-file"},{"_type":"video-file","sha256":"ee4e811080a619215fd7541b39203dfb69c2db4cdadbfdd7a234a2cff684f6f3","size":139435,"path":"media/videos/validation_videos_50_steps_0_ee4e811080a619215fd7.mp4"},{"sha256":"22e31e048ba5e5b9d6587d7306904d6edc6c618b8f77c3ceb05ba2d602309274","size":147072,"path":"media/videos/validation_videos_50_steps_0_22e31e048ba5e5b9d658.mp4","_type":"video-file"}]},"_timestamp":1.75118195270901e+09,"vsa_sparsity":0.05,"learning_rate":1e-05,"train_loss":0.19960195198655128,"_runtime":107.325113071}
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
SolitaryThinker	352e3c31fe	tests	2025-09-10 01:57:42 +00:00
SolitaryThinker	4f5e79c41f	update	2025-09-10 01:52:21 +00:00
SolitaryThinker	a2d303b067	fix rebase	2025-09-09 23:30:19 +00:00
SolitaryThinker	e07111b0de	update parquet handling	2025-09-09 23:29:37 +00:00
SolitaryThinker	aa49d2a5c8	fix num_inferenc_steps	2025-09-09 23:29:37 +00:00
SolitaryThinker	b337d03e82	disable trajectory deocding	2025-09-09 23:29:37 +00:00
SolitaryThinker	b92da9e912	hack to get it running	2025-09-09 23:29:36 +00:00
SolitaryThinkerandkevin314	d615271814	add kevin as coauthor Co-authored-by: kevin314 <kevin.lin.cs1@gmail.com>	2025-09-09 23:29:36 +00:00
SolitaryThinker	720cfe39ca	update	2025-09-09 23:29:36 +00:00
SolitaryThinker	03da4b1cdc	update	2025-09-09 23:29:36 +00:00
SolitaryThinker	ce52c3e87e	rename	2025-09-09 23:29:35 +00:00
SolitaryThinker	7b7a895e77	checkpoint	2025-09-09 23:29:35 +00:00
SolitaryThinker	4744ec2c0b	checkpoint	2025-09-09 23:29:33 +00:00
				`@@ -0,0 +1 @@`
				{"step_time":0.6983645600266755,"_wandb":{"runtime":107},"grad_norm":0.50390625,"avg_step_time":1.002151239803061,"_step":5,"validation_videos_50_steps":{"captions":false,"_type":"videos","count":8,"videos":[{"size":159131,"path":"media/videos/validation_videos_50_steps_0_dc447599dbe48350e9c9.mp4","_type":"video-file","sha256":"dc447599dbe48350e9c920f4971e1786bde580dd20d52b4aa147ae8d3dc564d6"},{"_type":"video-file","sha256":"4e283876ddfbf5a2cb6f8aca07a39f832b5f806fbefde8854bc73ed904ff20ee","size":160315,"path":"media/videos/validation_videos_50_steps_0_4e283876ddfbf5a2cb6f.mp4"},{"size":135225,"path":"media/videos/validation_videos_50_steps_0_78185c41e1935306e93c.mp4","_type":"video-file","sha256":"78185c41e1935306e93c2d416ee40b31d038abffb029cb5bfb11c2a634eb2fcf"},{"_type":"video-file","sha256":"27e9819d002d3f63c8918bbdc5bf2857b5effe0caf5d5b8374b9c590fc6432eb","size":197873,"path":"media/videos/validation_videos_50_steps_0_27e9819d002d3f63c891.mp4"},{"_type":"video-file","sha256":"46fe548e86144ca60a9396fcd15e8788d3a05c066c6819ccdd6a9041feaaec8f","size":170601,"path":"media/videos/validation_videos_50_steps_0_46fe548e86144ca60a93.mp4"},{"sha256":"91ec338774bec870b9c5c81be4330a8f0f2535124cb372e881c3703e6b65ed77","size":164462,"path":"media/videos/validation_videos_50_steps_0_91ec338774bec870b9c5.mp4","_type":"video-file"},{"_type":"video-file","sha256":"ee4e811080a619215fd7541b39203dfb69c2db4cdadbfdd7a234a2cff684f6f3","size":139435,"path":"media/videos/validation_videos_50_steps_0_ee4e811080a619215fd7.mp4"},{"sha256":"22e31e048ba5e5b9d6587d7306904d6edc6c618b8f77c3ceb05ba2d602309274","size":147072,"path":"media/videos/validation_videos_50_steps_0_22e31e048ba5e5b9d658.mp4","_type":"video-file"}]},"_timestamp":1.75118195270901e+09,"vsa_sparsity":0.05,"learning_rate":1e-05,"train_loss":0.19960195198655128,"_runtime":107.325113071}