[bugfix]: keep sdpa metadata masks 2D; lift to 4D only inside the torch-sdpa impl

SDPAMetadataBuilder.build() lifted 2D [batch, key_len] padding masks to 4D
[batch, 1, 1, key_len], but the metadata is cross-backend: HYWorld and
HunyuanVideo15 build SDPAMetadata while the layer selector picks FLASH_ATTN
when installed. flash_attn.py's equal-length branch padded
attn_mask.shape[1] assuming 2D, so a 4D mask produced a garbage-length pad
and an out-of-bounds gather (CUDA device-side assert, SIGABRT in the CI
transformer tests: test_hyworld crashed, test_matrixgame2/test_wanvideo
never ran).

Store the mask 2D in the builder (the convention every in-tree
producer/consumer assumes) and do the 2D -> [batch, 1, 1, key_len] lift
inside _normalize_attn_mask_for_sdpa, the torch-sdpa-only consumer, so
torch.sdpa still does not misread 2D as its [query_len, key_len]
broadcast. The int -> bool coercion is unchanged. Composes with the
consumer-side normalization in flash_attn.py (98116070): metadata carries
the shared 2D convention, and each impl reshapes for its own kernel.

Adds a CPU-only regression test for the contract and wires it into the CI
unit lane explicitly (fastvideo/tests/attention/ has no lane of its own).

.buildkite/pipeline.yml

@@ -436,7 +436,7 @@ steps:
                 - "pyproject.toml"
                 - "docker/Dockerfile"
               config:
-                command: "timeout 25m .buildkite/scripts/pr_test.sh"
+                command: "timeout 15m .buildkite/scripts/pr_test.sh"
                 label: ":test_tube: LoRA Training Tests"
                 env:
                   - TEST_TYPE=training_lora
@@ -481,7 +481,6 @@ steps:
                 - "fastvideo/layers/**"
                 - "fastvideo/worker/**"
                 - "fastvideo/entrypoints/**"
-                - "fastvideo/performance/**"
                 - "fastvideo/tests/performance/**"
                 - ".buildkite/performance-benchmarks/**"
                 - "pyproject.toml"

.buildkite/scripts/pr_test.sh

@@ -80,23 +80,6 @@ MODAL_ENV="BUILDKITE_REPO=$BUILDKITE_REPO BUILDKITE_COMMIT=$BUILDKITE_COMMIT BUI
 
 POST_RUN_HOOK=""
 
-is_truthy() {
-    case "${1:-}" in
-        1|true|TRUE|yes|YES|on|ON) return 0 ;;
-        *) return 1 ;;
-    esac
-}
-
-ssim_bootstrap_args() {
-    local title="${PR_TITLE:-}"
-    local message="${BUILDKITE_MESSAGE:-}"
-    if is_truthy "${FASTVIDEO_SSIM_BOOTSTRAP_MODE:-}" \
-        || [[ "$title" == *"[new-model]"* ]] \
-        || [[ "$message" == *"[new-model]"* ]]; then
-        printf ' --bootstrap-mode'
-    fi
-}
-
 upload_performance_artifacts() {
     SHORT_SHA=${BUILDKITE_COMMIT:0:7}
     LOCAL_DIR="downloaded_reports"
@@ -189,12 +172,7 @@ case "$TEST_TYPE" in
         ;;
     "ssim")
         log "Running SSIM tests..."
-        SSIM_BOOTSTRAP_ARGS=$(ssim_bootstrap_args)
-        if [ -n "$SSIM_BOOTSTRAP_ARGS" ]; then
-            log "SSIM bootstrap mode enabled for new-model reference draft generation"
-        fi
-        MODAL_COMMAND="$MODAL_ENV HF_API_KEY=$HF_API_KEY python3 -m modal run "
-        MODAL_COMMAND+="$MODAL_SSIM_TEST_FILE::run_ssim_tests$SSIM_BOOTSTRAP_ARGS"
+        MODAL_COMMAND="$MODAL_ENV HF_API_KEY=$HF_API_KEY python3 -m modal run $MODAL_SSIM_TEST_FILE::run_ssim_tests"
         ;;
     "training")
         log "Running training tests..."

.github/scripts/gate_full_suite.sh

@@ -1,133 +0,0 @@
-#!/usr/bin/env bash
-# Gate the expensive Buildkite full suite on the cheap GitHub checks.
-#
-# Polls the workflow runs for the PR head commit and only exits 0 once the
-# watched cheap workflows (pre-commit, docs build) have succeeded, so the
-# 'ready' label cannot burn ~20 GPU lanes on a head that a cheap check has
-# already doomed.
-#
-# Semantics:
-#   - watched run completed with a bad conclusion  -> exit 1 (fail CLOSED:
-#     no full suite; the next push re-arms via the 'synchronize' trigger)
-#   - watched run cancelled                         -> still pending: the docs
-#     workflow's repo-global 'pages' concurrency group cancels runs superseded
-#     by unrelated pushes, so 'cancelled' is not a verdict on this PR
-#   - watched runs pending                          -> poll until done
-#   - docs run absent                               -> not applicable after a
-#     short grace period ('Deploy Documentation' is path-filtered on PRs)
-#   - pre-commit run absent                         -> keep polling: pre-commit
-#     is never path-filtered, so its absence is always anomalous
-#   - 'ready' label removed while waiting           -> exit 1 (fail CLOSED:
-#     un-labeling is a deliberate maintainer action)
-#   - GitHub API unreachable or timeout             -> exit 0 (fail OPEN,
-#     loud warning: never brick CI on a GitHub outage)
-#
-# Required env: PR_SHA (PR head commit), PR_NUMBER, GITHUB_REPOSITORY, GH_TOKEN.
-set -euo pipefail
-
-: "${PR_SHA:?PR_SHA (PR head commit) is required}"
-: "${PR_NUMBER:?PR_NUMBER (pull request number) is required}"
-: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required}"
-
-# Workflow-level `name:` values that must be green before the full suite
-# may start. "Deploy Documentation" is path-filtered on PRs, so its run may
-# legitimately never exist; pre-commit always runs, so it must appear.
-WATCHED_NAMES='["pre-commit", "Deploy Documentation"]'
-WATCHED_REGEX='^(pre-commit|Deploy Documentation)$'
-POLL_SECS="${POLL_SECS:-20}"
-GRACE_SECS="${GRACE_SECS:-60}"
-MAX_WAIT_SECS="${MAX_WAIT_SECS:-1500}"
-
-# Bound each API call so a hung connection hits the 3-strike fail-open path
-# instead of pinning the loop until the job timeout (which would fail closed
-# on exactly the GitHub-outage case this script is meant to survive).
-if command -v timeout >/dev/null 2>&1; then
-  gh_api() { timeout 30 gh api "$@"; }
-else
-  gh_api() { gh api "$@"; }  # macOS dev boxes; CI always has coreutils timeout
-fi
-
-# The workflow checked the label before starting the gate, but the wait can
-# last ~25 min: re-check once before any exit 0 and fail closed if 'ready'
-# was removed in the meantime. An API error here proceeds (the label was
-# present when the gate started; never brick CI on an outage).
-recheck_ready_label() {
-  local pr_json
-  if pr_json=$(gh_api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" 2>/dev/null); then
-    if ! jq -e '[.labels[]?.name] | index("ready")' <<<"$pr_json" >/dev/null 2>&1; then
-      echo "::error::PR #${PR_NUMBER} no longer has the 'ready' label —" \
-        "NOT triggering the Buildkite full suite. Re-add the label to re-arm."
-      exit 1
-    fi
-  else
-    echo "::warning::Could not re-check the 'ready' label on PR #${PR_NUMBER}; proceeding (it was present when the gate started)."
-  fi
-}
-
-start=$(date +%s)
-api_fails=0
-missing=""
-
-while true; do
-  elapsed=$(( $(date +%s) - start ))
-
-  if runs_json=$(gh_api "repos/${GITHUB_REPOSITORY}/actions/runs?head_sha=${PR_SHA}&per_page=100" 2>/dev/null) \
-     && state=$(jq --arg re "$WATCHED_REGEX" '
-          [.workflow_runs[]? | select(.name // "" | test($re))]
-          | group_by(.name) | map(max_by(.id))
-          | map({name, status, conclusion})' <<<"$runs_json" 2>/dev/null); then
-    api_fails=0
-    echo "t+${elapsed}s watched checks: $(jq -c . <<<"$state")"
-
-    failed=$(jq -r '[.[] | select(.status == "completed"
-          and (.conclusion | IN("success", "skipped", "neutral", "cancelled") | not))]
-        | map(.name) | join(", ")' <<<"$state")
-    if [ -n "$failed" ]; then
-      echo "::error::Cheap check(s) failed on ${PR_SHA}: ${failed}." \
-        "NOT triggering the Buildkite full suite. Push a fix (the 'ready'" \
-        "label re-arms on every push), or re-run the failed check and then" \
-        "re-run this workflow."
-      exit 1
-    fi
-
-    # 'cancelled' counts as pending: wait for a re-run to reach a real verdict
-    # (bounded by MAX_WAIT, then the fail-open below).
-    pending=$(jq '[.[] | select(.status != "completed" or .conclusion == "cancelled")] | length' <<<"$state")
-    missing=$(jq -r --argjson watched "$WATCHED_NAMES" '($watched - map(.name)) | join(", ")' <<<"$state")
-    if [ "$pending" -eq 0 ]; then
-      if [ -z "$missing" ]; then
-        recheck_ready_label
-        echo "All watched cheap checks are green — full suite may proceed."
-        exit 0
-      fi
-      case "$missing" in
-        *pre-commit*)
-          echo "pre-commit run not found for ${PR_SHA} yet; waiting (pre-commit is never path-filtered, so its absence is anomalous)."
-          ;;
-        *)
-          if [ "$elapsed" -ge "$GRACE_SECS" ]; then
-            recheck_ready_label
-            echo "::warning::Watched run(s) never appeared for ${PR_SHA}: ${missing} (path-filtered, likely not applicable). Proceeding on the checks that did run."
-            exit 0
-          fi
-          echo "Waiting up to ${GRACE_SECS}s grace for path-filtered run(s) to appear: ${missing}."
-          ;;
-      esac
-    fi
-  else
-    api_fails=$(( api_fails + 1 ))
-    echo "::warning::GitHub API error querying workflow runs for ${PR_SHA} (attempt ${api_fails}/3)."
-    if [ "$api_fails" -ge 3 ]; then
-      recheck_ready_label
-      echo "::warning::FAILING OPEN: cannot query GitHub check status — triggering the full suite WITHOUT the cheap-check gate."
-      exit 0
-    fi
-  fi
-
-  if [ "$elapsed" -ge "$MAX_WAIT_SECS" ]; then
-    recheck_ready_label
-    echo "::warning::FAILING OPEN: watched checks still pending after $(( MAX_WAIT_SECS / 60 )) min${missing:+ (never appeared: ${missing})} — triggering the full suite anyway."
-    exit 0
-  fi
-  sleep "$POLL_SECS"
-done

.github/scripts/test_gate_full_suite.sh

@@ -1,122 +0,0 @@
-#!/usr/bin/env bash
-# Self-test for gate_full_suite.sh using a mocked `gh`. No network, runs on
-# any dev box: bash .github/scripts/test_gate_full_suite.sh
-set -u
-here=$(cd "$(dirname "$0")" && pwd)
-tmp=$(mktemp -d)
-trap 'rm -rf "$tmp"' EXIT
-
-# Mock gh. Asserts the exact endpoint (including head_sha) it is called
-# with — an endpoint typo in the gate script fails the test rather than
-# silently serving canned data. On the runs endpoint it serves
-# $MOCK_DIR/response_<call#>.json, sticking on the highest existing file,
-# and exits 1 if none exist (simulates a GitHub API outage). On the pulls
-# endpoint it serves $MOCK_DIR/pr.json, defaulting to a 'ready'-labeled PR.
-cat > "$tmp/gh" <<'EOF'
-#!/usr/bin/env bash
-if [ "${1:-}" != "api" ]; then
-  echo "unexpected gh invocation: $*" >> "$MOCK_DIR/endpoint_error"
-  exit 2
-fi
-case "${2:-}" in
-  "repos/o/r/actions/runs?head_sha=deadbeef&per_page=100")
-    n=$(( $(cat "$MOCK_DIR/count" 2>/dev/null || echo 0) + 1 ))
-    echo "$n" > "$MOCK_DIR/count"
-    while [ "$n" -gt 0 ]; do
-      if [ -f "$MOCK_DIR/response_$n.json" ]; then
-        cat "$MOCK_DIR/response_$n.json"
-        exit 0
-      fi
-      n=$(( n - 1 ))
-    done
-    echo "api outage" >&2
-    exit 1
-    ;;
-  "repos/o/r/pulls/42")
-    if [ -f "$MOCK_DIR/pr.json" ]; then
-      cat "$MOCK_DIR/pr.json"
-    else
-      echo '{"labels": [{"name": "ready"}]}'
-    fi
-    ;;
-  *)
-    echo "unexpected gh endpoint: $2" >> "$MOCK_DIR/endpoint_error"
-    exit 2
-    ;;
-esac
-EOF
-chmod +x "$tmp/gh"
-
-PC_OK='{"name": "pre-commit", "id": 1, "status": "completed", "conclusion": "success"}'
-PC_BAD='{"name": "pre-commit", "id": 1, "status": "completed", "conclusion": "failure"}'
-PC_PENDING='{"name": "pre-commit", "id": 1, "status": "in_progress", "conclusion": null}'
-DOCS_OK='{"name": "Deploy Documentation", "id": 2, "status": "completed", "conclusion": "success"}'
-DOCS_BAD='{"name": "Deploy Documentation", "id": 2, "status": "completed", "conclusion": "failure"}'
-DOCS_CANCELLED='{"name": "Deploy Documentation", "id": 2, "status": "completed", "conclusion": "cancelled"}'
-OTHER='{"name": "Trigger Full Suite", "id": 3, "status": "in_progress", "conclusion": null}'
-NULL_NAME='{"name": null, "id": 4, "status": "completed", "conclusion": "failure"}'
-PC_OK_RERUN='{"name": "pre-commit", "id": 5, "status": "completed", "conclusion": "success"}'
-
-fails=0
-want_log=""  # optional: expect() also greps out.log for this regex, then resets
-pr_json=""   # optional: served for the pulls (label re-check) endpoint, then resets
-raw_body=""  # optional: serve responses verbatim instead of wrapping in workflow_runs
-expect() { # <name> <expected-exit> <response json>...
-  local name=$1 want=$2 dir i=1
-  shift 2
-  dir=$(mktemp -d "$tmp/test_XXXXXX")
-  for body in "$@"; do
-    if [ -n "$raw_body" ]; then
-      printf '%s' "$body" > "$dir/response_$i.json"
-    else
-      printf '{"workflow_runs": [%s]}' "$body" > "$dir/response_$i.json"
-    fi
-    i=$(( i + 1 ))
-  done
-  [ -n "$pr_json" ] && printf '%s' "$pr_json" > "$dir/pr.json"
-  ( export PATH="$tmp:$PATH" MOCK_DIR="$dir" PR_SHA=deadbeef PR_NUMBER=42 \
-      GITHUB_REPOSITORY=o/r POLL_SECS=0 GRACE_SECS=1 MAX_WAIT_SECS=3
-    bash "$here/gate_full_suite.sh" > "$dir/out.log" 2>&1 )
-  local rc=$?
-  if [ "$rc" -ne "$want" ]; then
-    echo "FAIL: $name (exit $rc, want $want)"
-    cat "$dir/out.log"
-    fails=1
-  elif [ -f "$dir/endpoint_error" ]; then
-    echo "FAIL: $name (mock gh got an unexpected call)"
-    cat "$dir/endpoint_error"
-    fails=1
-  elif [ -n "$want_log" ] && ! grep -Eq "$want_log" "$dir/out.log"; then
-    echo "FAIL: $name (log does not match: $want_log)"
-    cat "$dir/out.log"
-    fails=1
-  else
-    echo "ok: $name"
-  fi
-  want_log="" pr_json="" raw_body=""
-}
-
-expect "both green -> proceed" 0 "$PC_OK, $DOCS_OK, $OTHER, $NULL_NAME"
-expect "docs build failed -> blocked" 1 "$PC_OK, $DOCS_BAD"
-expect "pre-commit failed -> blocked" 1 "$PC_BAD"
-expect "pending then green -> proceed" 0 "$PC_PENDING" "$PC_OK, $DOCS_OK"
-want_log="never appeared.*Deploy Documentation"
-expect "docs run absent (path-filtered) -> proceed after grace" 0 "$PC_OK"
-expect "API outage -> fail open" 0
-want_log="FAILING OPEN"
-expect "pending past MAX_WAIT -> fail open" 0 "$PC_PENDING"
-want_log="FAILING OPEN"
-expect "unrelated runs only -> no grace, fail open at MAX_WAIT" 0 "$OTHER"
-expect "cancelled docs then green -> proceed" 0 \
-  "$PC_OK, $DOCS_CANCELLED" "$PC_OK, $DOCS_OK"
-want_log="FAILING OPEN"
-expect "cancelled docs forever -> fail open at MAX_WAIT" 0 "$PC_OK, $DOCS_CANCELLED"
-want_log="FAILING OPEN"
-expect "pre-commit absent -> no grace, fail open at MAX_WAIT" 0 "$DOCS_OK"
-expect "duplicate run names -> latest wins" 0 "$PC_BAD, $PC_OK_RERUN, $DOCS_OK"
-raw_body=1
-expect "garbage response body -> fail open" 0 "this is not json"
-pr_json='{"labels": [{"name": "other"}]}'
-expect "ready label removed mid-gate -> blocked" 1 "$PC_OK, $DOCS_OK"
-
-exit "$fails"

.github/workflows/ci-precommit.yml

@@ -47,6 +47,3 @@ jobs:
     - uses: pre-commit/action@v3.0.1
       with:
         extra_args: --all-files --hook-stage manual
-    # After pre-commit so a self-test failure cannot mask lint failures.
-    - name: Full-suite gate self-test
-      run: bash .github/scripts/test_gate_full_suite.sh

.github/workflows/ci-slash-commands.yml

@@ -52,7 +52,6 @@ jobs:
             core.setOutput('pr_sha', pr.head.sha);
             core.setOutput('pr_branch', pr.head.ref);
             core.setOutput('pr_number', String(prNumber));
-            core.setOutput('pr_title', pr.title);
 
       - name: Trigger Full Suite
         if: steps.perm.outputs.has_write == 'true'
@@ -61,7 +60,6 @@ jobs:
           PR_SHA: ${{ steps.label.outputs.pr_sha }}
           PR_BRANCH: ${{ steps.label.outputs.pr_branch }}
           PR_NUMBER: ${{ steps.label.outputs.pr_number }}
-          PR_TITLE: ${{ steps.label.outputs.pr_title }}
           BK_ORG: ${{ vars.BUILDKITE_ORG_SLUG }}
           BK_PIPELINE: ${{ vars.BUILDKITE_PIPELINE_SLUG }}
         run: |
@@ -73,7 +71,6 @@ jobs:
               --arg commit "$PR_SHA" \
               --arg branch "$PR_BRANCH" \
               --arg message "Full Suite for PR #${PR_NUMBER} (via /merge)" \
-              --arg pr_title "$PR_TITLE" \
               --argjson pr_id "$PR_NUMBER" \
               '{
                 commit: $commit,
@@ -83,12 +80,11 @@ jobs:
                 pull_request_id: $pr_id,
                 pull_request_base_branch: "main",
                 env: {
-                    TEST_SCOPE: "full",
-                    FULL_SUITE: "true",
-                    PR_NUMBER: ($pr_id | tostring),
-                    PR_TITLE: $pr_title
-                  }
-                }')"
+                  TEST_SCOPE: "full",
+                  FULL_SUITE: "true",
+                  PR_NUMBER: ($pr_id | tostring)
+                }
+              }')"
 
   parse-command:
     if: >-
@@ -245,7 +241,6 @@ jobs:
           TEST_SCOPE: ${{ needs.parse-command.outputs.test_scope }}
           FULL_SUITE: ${{ needs.parse-command.outputs.full_suite }}
           TEST_TYPE: ${{ needs.parse-command.outputs.test_type }}
-          PR_TITLE: ${{ github.event.issue.title }}
           BK_ORG: ${{ vars.BUILDKITE_ORG_SLUG }}
           BK_PIPELINE: ${{ vars.BUILDKITE_PIPELINE_SLUG }}
         run: |
@@ -262,7 +257,6 @@ jobs:
               --arg full_suite "$FULL_SUITE" \
               --arg test_type "$TEST_TYPE" \
               --arg pr_number "$PR_NUMBER" \
-              --arg pr_title "$PR_TITLE" \
               '{
                 commit: $commit,
                 branch: $branch,
@@ -272,9 +266,8 @@ jobs:
                 pull_request_base_branch: "main",
                 env: {
                   TEST_SCOPE: $test_scope,
-                    FULL_SUITE: $full_suite,
-                    TEST_TYPE: $test_type,
-                    PR_NUMBER: $pr_number,
-                    PR_TITLE: $pr_title
-                  }
-                }')"
+                  FULL_SUITE: $full_suite,
+                  TEST_TYPE: $test_type,
+                  PR_NUMBER: $pr_number
+                }
+              }')"

.github/workflows/ci-trigger-full-suite.yml

@@ -7,7 +7,6 @@ on:
 permissions:
   contents: read
   pull-requests: read
-  actions: read
 
 concurrency:
   group: full-suite-${{ github.event.pull_request.number }}
@@ -19,8 +18,6 @@ jobs:
       (github.event.action == 'labeled' && github.event.label.name == 'ready')
       || github.event.action == 'synchronize'
     runs-on: ubuntu-latest
-    # Gate below may wait for cheap checks (up to MAX_WAIT_SECS = 25 min).
-    timeout-minutes: 35
     steps:
       - name: Check ready label
         id: check
@@ -52,20 +49,6 @@ jobs:
               "https://api.buildkite.com/v2/organizations/${{ vars.BUILDKITE_ORG_SLUG }}/pipelines/${{ vars.BUILDKITE_PIPELINE_SLUG }}/builds/${build_num}/cancel"
           done
 
-      # Checks out the BASE branch (default for pull_request_target), so PR
-      # authors cannot tamper with the gate script.
-      - name: Checkout gate script
-        if: steps.check.outputs.has_ready == 'true'
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-
-      - name: Wait for pre-commit and docs build
-        if: steps.check.outputs.has_ready == 'true'
-        env:
-          GH_TOKEN: ${{ github.token }}
-          PR_SHA: ${{ github.event.pull_request.head.sha }}
-          PR_NUMBER: ${{ github.event.pull_request.number }}
-        run: bash .github/scripts/gate_full_suite.sh
-
       - name: Trigger Buildkite Full Suite
         if: steps.check.outputs.has_ready == 'true'
         env:
@@ -73,7 +56,6 @@ jobs:
           PR_SHA: ${{ github.event.pull_request.head.sha }}
           PR_BRANCH: ${{ github.event.pull_request.head.ref }}
           PR_NUMBER: ${{ github.event.pull_request.number }}
-          PR_TITLE: ${{ github.event.pull_request.title }}
           BK_ORG: ${{ vars.BUILDKITE_ORG_SLUG }}
           BK_PIPELINE: ${{ vars.BUILDKITE_PIPELINE_SLUG }}
         run: |
@@ -85,7 +67,6 @@ jobs:
               --arg commit "$PR_SHA" \
               --arg branch "$PR_BRANCH" \
               --arg message "Full Suite for PR #${PR_NUMBER}" \
-              --arg pr_title "$PR_TITLE" \
               --argjson pr_id "$PR_NUMBER" \
               '{
                 commit: $commit,
@@ -97,7 +78,6 @@ jobs:
                 env: {
                   TEST_SCOPE: "full",
                   FULL_SUITE: "true",
-                  PR_NUMBER: ($pr_id | tostring),
-                  PR_TITLE: $pr_title
+                  PR_NUMBER: ($pr_id | tostring)
                 }
               }')"

.gitignore

@@ -72,7 +72,8 @@ docs/distillation/examples/
 # Python pickle files
 *.pkl
 
-# Reference videos (negations must come after the catch-all on line below)
+# Reference videos
+!fastvideo/tests/ssim/reference_videos/**/*.mp4
 
 # Static images
 !docs/assets/images/**/*.png
@@ -126,8 +127,6 @@ apps/dreamverse/web/.env.production.local
 .sisyphus/
 openspec/
 fastvideo/tests/ssim/reference_videos/**
-!fastvideo/tests/ssim/reference_videos/**/*.mp4
-!fastvideo/tests/ssim/reference_videos/**/*.png
 
 # Editor logs and local Python version pins (accidentally committed)
 *.nvimlog

assets/eval/worldmodel_synthetic_flow_calibration.json

@@ -1,22 +0,0 @@
-{
-  "alpha_yaw": 0.08734091699186919,
-  "alpha_pitch": 0.08169667696275307,
-  "alpha_turn": 5.724587470723463e-17,
-  "beta_fwd": 0.02842768078408099,
-  "beta_strafe": 0.022531015077067108,
-  "focal_length": 457.0,
-  "frame_shape": [
-    352,
-    640
-  ],
-  "calibrated_from": [
-    "1_wasd_only",
-    "camera",
-    "camera4hold_alpha1",
-    "fully_random",
-    "wasdonly_alpha1",
-    "wasd4holdrandview_simple_1key1mouse1"
-  ],
-  "residual_rms": 15.890399609478676,
-  "n_equations": 4125232
-}

docs/contributing/ci_architecture.md

@@ -99,13 +99,6 @@ status.
 Full Suite is also path-filtered. It validates broader behavior before Mergify
 can merge a PR.
 
-A `ready`-labeled PR does not hit Buildkite immediately:
-`ci-trigger-full-suite.yml` first runs `.github/scripts/gate_full_suite.sh`,
-which waits for the cheap Tier-1 checks (pre-commit, docs build) on the PR
-head. A red cheap check blocks the suite (fail closed; the next push re-arms
-it), while a GitHub outage or a >25 min wait lets it run anyway (fail open).
-`/test full` bypasses the gate.
-
 | Buildkite label | `TEST_TYPE` | Main watched paths |
 |---|---|---|
 | SSIM Tests | `ssim` | `fastvideo/**/*.py`, `pyproject.toml`, `docker/Dockerfile` |

docs/contributing/performance_benchmarks.md

@@ -79,11 +79,10 @@ fastvideo/performance/
 ```
 
 The HF dataset (`FastVideo/performance-tracking` by default) holds one
-normalized JSON per run. For v2 records, the rolling baseline is the median of
-the last 5 successful, baseline-eligible records in the same comparison cohort:
-`model_id`, `gpu_type`, `workload_id`, `variant_id`, `benchmark_version`,
-`recipe_fingerprint`, `hardware_profile_id`, and `software_profile_id`. PR and
-local records are visible in the dashboard but are not baseline eligible.
+normalized JSON per `(model_id, gpu_type, run)` tuple. The rolling baseline is
+the median of the last 5 successful, baseline-eligible records for that
+model+GPU. PR and local records are visible in the dashboard but are not
+baseline eligible.
 
 ## Planned Coverage
 
@@ -159,16 +158,13 @@ unrealistic memory growth, and optionally large component-specific slowdowns
 even when the rolling baseline is empty. They are hand-set with generous
 headroom and almost never need touching.
 
-### Rolling baseline (per comparison cohort)
+### Rolling baseline (per `(model_id, gpu_type)`)
 
 `compare_baseline.py` loads the last 5 successful, baseline-eligible records
-for the same comparison cohort from the HF dataset, computes the median for
-each available metric, and evaluates the current run with the metric's
-rolling regression policy. For v2 records, that cohort is `model_id`,
-`gpu_type`, `workload_id`, `variant_id`, `benchmark_version`,
-`recipe_fingerprint`, `hardware_profile_id`, and `software_profile_id`. For
-latency, memory, and component times, higher values are regressions. For
-throughput, lower values are regressions.
+for the same `(model_id, gpu_type)` from the HF dataset, computes the median
+for each available metric, and evaluates the current run with the metric's
+rolling regression policy. For latency, memory, and component times, higher
+values are regressions. For throughput, lower values are regressions.
 
 A metric exceeds its rolling threshold when both of these are true:
 
@@ -232,25 +228,11 @@ by the harness and is not config-declarable.)
 
 Recipe fingerprinting, hardware/software profile IDs, exact-identity
 comparison, and dashboard cohort grouping land with this change: v2 records
-compare only within their identity cohort and are never compared against v1
-records (or vice versa). Legacy records missing any comparison identity field
-still run and are normalized for reporting, but they skip rolling-baseline
-comparison entirely (`baseline_status: "skipped_missing_identity"`, comparator
-verdict `PASS`, never baseline eligible); only static thresholds gate them.
-Each comparison reports an explicit `comparator_status` on the normalized
-record and in the Markdown summary:
-
-| Status | Meaning | CI |
-|---|---|---|
-| `PASS` | Comparable baseline found with no gated regression, or comparison was skipped for a record without the v2 identity block. | passes |
-| `REGRESSION` | A gated metric regressed past both its percent and absolute floors. | fails |
-| `CALIBRATION_NEEDED` | No comparable baseline exists. Gating is inactive and the record does **not** seed a baseline — seed new cohorts explicitly via the reseed workflow. The record also carries `baseline_status: "initialized_new_cohort"`. | passes |
-| `RECIPE_MISMATCH` | Baseline history exists for the same variant/hardware/software cohort but under a different `recipe_fingerprint`. Represent recipe changes as a new `variant_id`, or reseed. | fails |
-| `INFRA_ERROR` | The comparison itself failed (for example, baseline history could not be loaded). | fails |
-| `QUALITY_BLOCKED` | Reserved for the promoted-baseline workflow; never emitted by the comparator. | n/a |
-
-Metric-specific threshold policies and promoted baselines remain separate
-follow-ups.
+compare only within their identity cohort, and a record that opens a NEW
+cohort is marked `baseline_status: "initialized_new_cohort"` (regression
+gating starts once that cohort accumulates history). Legacy v1 configs keep
+the `(model_id, gpu_type)` rolling-baseline comparison. Metric-specific
+threshold policies and promoted baselines remain separate follow-ups.
 
 ### Raw record (`results/perf_*.json`)
 
@@ -259,7 +241,7 @@ Written by `test_inference_performance.py`. One file per benchmark run.
 ```jsonc
 {
   "benchmark_id": "wan-t2v-1.3b-2gpu",
-  "result_schema_version": 2,
+  "config_schema_version": 2,
   "workload_id": "wan-t2v",
   "variant_id": "1.3b-sp2",
   "benchmark_version": 2,
@@ -288,15 +270,8 @@ Written by `test_inference_performance.py`. One file per benchmark run.
     }
   },
   "commit": "<full sha>",
-  "run_source": "pr",
-  "branch": "feature/perf-change",
   "pr_number": "1234",
-  "test_scope": "direct",
-  "build_url": "https://buildkite.example/build",
-  "build_id": "<buildkite-build-id>",
-  "job_id": "<buildkite-job-id>",
   "timestamp": "2026-05-08T22:00:00+00:00",
-  "quality_metadata": { "quality_status": "canonical" },
   "text_encoder_time_s": 2.141,
   "dit_time_s": 8.437,
   "vae_decode_time_s": 3.208,
@@ -347,7 +322,6 @@ result, used as the rolling-baseline source of truth.
 ```jsonc
 {
   "model_id": "wan-t2v-1.3b-2gpu",
-  "result_schema_version": 2,
   "workload_id": "wan-t2v",
   "variant_id": "1.3b-sp2",
   "benchmark_version": 2,
@@ -371,32 +345,17 @@ result, used as the rolling-baseline source of truth.
   "hardware_profile_id": "hw-<sha256-prefix>",
   "software_profile_id": "sw-<sha256-prefix>",
   "environment_fingerprint": "env-<sha256-prefix>",
-  "run_source": "pr",
-  "branch": "feature/perf-change",
-  "pr_number": "1234",
-  "test_scope": "direct",
-  "build_url": "https://buildkite.example/build",
-  "build_id": "<buildkite-build-id>",
-  "job_id": "<buildkite-job-id>",
-  "quality_metadata": { "quality_status": "canonical" },
-  "baseline_status": "compared",
-  "comparator_status": "PASS",
   "success": true
 }
 ```
 
 ### Compatibility with legacy records
 
-Older records in the HF dataset may not have `result_schema_version`,
-component timing fields, or v2 identity/profile fields. Records without
-`result_schema_version` are treated as v1. The comparator ignores missing or
-`null` metrics when computing a median, and the dashboard lists skipped plots
-for metric series that have no non-null values. Records missing both
-`run_source` and `baseline_eligible` are treated as legacy successful
-main/full-suite uploads and remain eligible for rolling baselines.
-Current `perf_*.json` artifacts that lack the v2 comparison identity are
-normalized for reporting but skip rolling-baseline comparison and are not marked
-baseline eligible.
+Older records in the HF dataset may not have component timing fields. The
+comparator ignores missing or `null` metrics when computing a median, and the
+dashboard lists skipped plots for metric series that have no non-null values.
+Records missing both `run_source` and `baseline_eligible` are treated as legacy
+successful main/full-suite uploads and remain eligible for rolling baselines.
 
 New records compare only against the same `model_id`, `gpu_type`,
 `workload_id`, `variant_id`, `benchmark_version`, `recipe_fingerprint`,
@@ -418,7 +377,7 @@ FlashInfer, Cutlass DSL, SageAttention, Triton, and xFormers when installed.
 | `PERF_REPORTS_DIR` | `/root/data/perf_reports` | `compare_baseline.py`, `dashboard.py` | Where the Markdown summary and Plotly HTML get written for Buildkite to pick up. |
 | `HF_REPO_ID` | `FastVideo/performance-tracking` | `fastvideo/performance/hf_store.py` | HF dataset repo holding rolling-baseline records. |
 | `HF_API_KEY`, `HUGGINGFACE_HUB_TOKEN`, `HF_TOKEN` | unset | `fastvideo/performance/hf_store.py` | Required for upload or private dataset reads. |
-| `PERF_RUN_SOURCE` | inferred | `compare_baseline.py`, `test_inference_performance.py` | Source metadata for uploaded records: `pr`, `local`, `scheduled_main`, or `unknown`. |
+| `PERF_RUN_SOURCE` | inferred | `compare_baseline.py` | Source metadata for uploaded records: `pr`, `local`, `scheduled_main`, or `unknown`. |
 | `PERF_UPLOAD_POLICY` | `never` | `compare_baseline.py` | Upload policy: `never`, `pass`, or `always`. |
 | `PERF_PYTEST_RC` | unset | `compare_baseline.py` | Static-threshold pytest exit code, used so scheduled-main failures can be uploaded with `success=false`. |
 | `TEST_SCOPE` | unset | `compare_baseline.py` | CI context used to infer scheduled-main runs together with `BUILDKITE_BRANCH=main`. |
@@ -435,14 +394,11 @@ point is `fastvideo/tests/modal/pr_test.py:run_performance_tests` and the
 Buildkite artifact upload is in
 `.buildkite/scripts/pr_test.sh:upload_performance_artifacts`.
 
-Each performance build runs pytest first. PR and direct runs only continue to
-`compare_baseline.py` when that fixed-threshold phase passes; if pytest fails,
-Markdown summaries and normalized JSON artifacts are not emitted. Scheduled
-main runs set `PERF_UPLOAD_POLICY=always`, so they still run
-`compare_baseline.py` (with `PERF_PYTEST_RC` set) after a fixed-threshold
-failure. Those failed scheduled main runs emit summaries and normalized
-records, upload records with `success=false`, and are excluded from future
-rolling baselines. The dashboard still runs best-effort for observability.
+Each performance build runs pytest first. If that fixed-threshold phase fails,
+PR/direct runs skip `compare_baseline.py` because they only upload passing
+records. Scheduled-main runs still execute `compare_baseline.py` with
+`PERF_PYTEST_RC` set so the failed canonical attempt is visible in normalized
+JSON and dashboard history. The dashboard runs best-effort for observability.
 When the rolling-baseline phase runs, it emits:
 
 * **Markdown summary** — appended to `$GITHUB_STEP_SUMMARY` when that variable
@@ -450,7 +406,7 @@ When the rolling-baseline phase runs, it emits:
   per-benchmark row with current vs. baseline values for latency, throughput,
   memory, text encoder time, DiT time, and VAE decode time.
 * **Plotly dashboard** — `dashboard_<sha>_<ts>.html` showing time-series for
-  each metric grouped by comparison cohort.
+  each metric grouped by `(model_id, gpu_type)`.
 * **Normalized records** — `normalized_perf_*.json`, one per benchmark.
   Useful as input to the
   [`reseed-performance-baseline`](https://github.com/hao-ai-lab/FastVideo/blob/main/.agents/skills/reseed-performance-baseline/SKILL.md)
@@ -501,10 +457,10 @@ When the rolling-baseline phase runs, it emits:
 2. The pytest test auto-discovers all configs — no test code needed. CI
    picks it up on the next `/test performance` run.
 
-3. The first run with no HF history reports `CALIBRATION_NEEDED`: it passes,
-   but it does not seed the shared baseline. Seed the cohort explicitly with
-   the `reseed-performance-baseline` skill; subsequent scheduled-main runs
-   then compare against it and keep the rolling baseline advancing.
+3. The first persisted main-branch run with no HF history initializes the
+   baseline (passes automatically). Subsequent runs compare against it. Local
+   and pull-request runs with no HF history also pass, but they do not seed the
+   shared baseline.
 
 4. If the benchmark targets a GPU not currently in `thresholds`, either add
    that GPU as a key or rely on the `default` block. Note that `default` is
@@ -524,13 +480,8 @@ When the rolling-baseline phase runs, it emits:
 
 ## Troubleshooting
 
-**`CALIBRATION_NEEDED: no comparable baseline`** — first run for this cohort.
-The run passes but does not seed a baseline; seed it explicitly with the
-`reseed-performance-baseline` skill.
-
-**`RECIPE_MISMATCH`** — the benchmark recipe changed without a new
-`variant_id`. Either bump `variant_id` to open a new cohort, or reseed the
-baseline if the existing variant should adopt the new recipe.
+**"No baseline for ... Initializing"** — first run for this `(model_id,
+gpu_type)`. Run will pass and (if persisting) seed the first record.
 
 **Persistent failure right after a torch / kernel / image upgrade** —
 genuine regression *or* baseline drift. Compare the failing normalized record

docs/contributing/testing.md

@@ -180,30 +180,6 @@ python fastvideo/tests/ssim/reference_videos_cli.py copy-local \
   --device-folder L40S_reference_videos
 ```
 
-### SSIM Bootstrap Mode
-
-Normal SSIM runs are strict: if a reference video or latent is missing, the
-test fails. For new-model PRs, CI can run SSIM in bootstrap mode so missing
-references are uploaded as draft artifacts for review instead of immediately
-blocking on a missing canonical reference.
-
-Buildkite enables SSIM bootstrap mode when either condition is true:
-
-- the PR title or Buildkite message contains `[new-model]`;
-- `FASTVIDEO_SSIM_BOOTSTRAP_MODE=1` is set for the Buildkite job.
-
-Bootstrap mode passes `--ssim-bootstrap-mode` to pytest. When a generated
-artifact is available, the test uploads it under the `drafts/...` namespace in
-the SSIM reference repo and marks that case as expected-failed. After reviewing
-the draft, promote it into the canonical reference layout:
-
-```bash
-python fastvideo/tests/ssim/reference_videos_cli.py promote-draft \
-  --quality-tier default \
-  --device-folder L40S_reference_videos \
-  --model-id <model_id>
-```
-
 ## CI Integration
 
 FastVideo CI tests are orchestrated by Buildkite and run on Modal GPU

docs/design/inference_schema_parity_inventory.yaml

@@ -458,8 +458,6 @@ surfaces:
       guidance_scale: request.sampling.guidance_scale
       guidance_scale_2: request.sampling.guidance_scale_2
       guidance_rescale: request.sampling.guidance_rescale
-      use_embedded_guidance: request.sampling.use_embedded_guidance
-      true_cfg_scale: request.sampling.true_cfg_scale
       boundary_ratio: request.sampling.boundary_ratio
       sigmas: request.sampling.sigmas
       enable_teacache: request.runtime.enable_teacache

docs/distillation/anyflow.md

@@ -1,116 +0,0 @@
-# 🌊 AnyFlow Any-Step Video Distillation
-
-**AnyFlow** ([paper](https://arxiv.org/abs/2605.13724), [project page](https://nvlabs.github.io/AnyFlow/), [official code](https://github.com/NVlabs/AnyFlow), [model weights](https://huggingface.co/collections/nvidia/anyflow)) is an any-step video diffusion framework built on flow maps. A single distilled checkpoint can be evaluated at NFE ∈ {1, 2, 4, 8, 16, 32} without retraining, and quality scales **monotonically** with steps — unlike consistency-based distillation, which often degrades as NFE grows.
-
-The student network ``u_θ(x_t, t, r)`` predicts the *average velocity* from time ``t`` back to time ``r``, so one Euler step is
-
-```
-x_r = x_t - ((t - r) / N) · u_θ(x_t, t, r)
-```
-
-for any ``t > r``.
-
-## 📊 Model Overview
-
-NVIDIA publishes four checkpoints under [`nvidia/anyflow`](https://huggingface.co/collections/nvidia/anyflow):
-
-- `nvidia/AnyFlow-Wan2.1-T2V-1.3B-Diffusers` — bidirectional T2V, Wan2.1 1.3B base
-- `nvidia/AnyFlow-Wan2.1-T2V-14B-Diffusers` — bidirectional T2V, Wan2.1 14B base
-- `nvidia/AnyFlow-FAR-Wan2.1-1.3B-Diffusers` — frame-autoregressive variant, 1.3B
-- `nvidia/AnyFlow-FAR-Wan2.1-14B-Diffusers` — frame-autoregressive variant, 14B
-
-FastVideo currently supports the bidirectional T2V variants for training; the FAR variants can be loaded for inference through the diffusers integration.
-
-## ⚙️ Inference
-
-For inference, load the published checkpoint directly through diffusers; FastVideo's training-side ``WanModel`` config maps the HF AnyFlow ``delta_embedder`` weights onto its internal layout via ``param_names_mapping`` so the same checkpoint can be used as the ``init_from`` for the on-policy YAML below.
-
-## 🧠 Algorithm
-
-Training runs in two stages. Both use the dual-timestep Wan backbone — enabled by ``pipeline.dit_config.r_embedder: true`` in the YAML, which allocates a sibling ``condition_embedder.delta_embedder`` and fuses its embedding with the standard timestep embedding via either an additive or a gated mixer.
-
-### Stage 1 — Pretrain (flow-map central-difference)
-
-Method: ``AnyFlowPretrainMethod`` (``fastvideo/train/methods/distribution_matching/anyflow_pretrain.py``)
-
-For each batch, sample ``(t, r) ∈ [0, 1]`` as ``(max, min)`` of two uniform draws, then:
-
-- a ``diffusion_ratio`` fraction (default 0.5) gets ``r = t`` — recovers plain flow matching;
-- a ``consistency_ratio`` fraction (default 0.25) gets ``r = 0`` — forces consistency to clean data;
-- the remainder is free.
-
-The student forward at ``(t, r)`` is trained against the central-difference target
-
-```
-target = (eps - x_0) - (t - r) · dF/dt
-```
-
-where ``dF/dt`` is estimated from the student's own forward at ``(t ± δ, r)`` with the sample also moved along the flow trajectory by ``v_pred · (δ / N)``. Per-timestep weighting uses ``beta08`` (``w(t) = t · sqrt(1 - t)``, renormalized). A stop-gradient scale-balance keeps the non-diffusion branches' loss magnitude aligned with the diffusion branch.
-
-### Stage 2 — On-policy DMD
-
-Method: ``AnyFlowMethod`` (``fastvideo/train/methods/distribution_matching/anyflow.py``)
-
-Inherits ``DMD2Method``. The student is rolled out for ``student_sample_steps`` Euler-flow steps from pure noise; one randomly-chosen step is gradient-enabled (broadcast from rank 0 so every worker agrees), the rest run under ``torch.no_grad``. With ``use_mean_velocity: true`` (default) the rollout uses ``r = t_next`` at each step, matching AnyFlow's ``WanAnyFlowPipeline.training_rollout``.
-
-The inherited ``_dmd_loss`` (VSD with fake-score critic) consumes the rollout output and the teacher's CFG prediction. The optional pinned ``t_list_override`` lets configs reproduce the paper's hand-tuned 4-step schedule ``[999, 937, 833, 624, 0]``.
-
-## 🚀 Training Scripts
-
-### Stage 1 — pretrain
-
-```bash
-bash examples/train/run.sh \
-    examples/train/configs/distribution_matching/wan/anyflow_pretrain_t2v.yaml
-```
-
-**Key configuration** (in ``examples/train/configs/distribution_matching/wan/anyflow_pretrain_t2v.yaml``):
-
-- Global batch size: 32 (8 GPUs × 4 per-GPU)
-- Learning rate: 5e-5
-- Flow shift: 5.0
-- ``diffusion_ratio`` / ``consistency_ratio``: 0.5 / 0.25
-- ``epsilon`` (finite-difference step): 5 (absolute train-timestep units)
-- ``weight_type``: ``beta08``
-- ``fuse_guidance_scale``: 3.0
-- Training steps: 6000
-
-### Stage 2 — on-policy
-
-```bash
-bash examples/train/run.sh \
-    examples/train/configs/distribution_matching/wan/anyflow_onpolicy_t2v.yaml \
-    --models.student.init_from outputs/wan2.1_anyflow_pretrain/checkpoint-final
-```
-
-(Or point ``models.student.init_from`` directly at ``nvidia/AnyFlow-Wan2.1-T2V-1.3B-Diffusers`` to bootstrap from the paper weights and skip Stage 1.)
-
-**Key configuration**:
-
-- Global batch size: 8 (8 GPUs × 1 per-GPU)
-- Learning rate: 2e-6
-- Flow shift: 5.0
-- ``student_sample_steps``: 4
-- ``t_list_override``: ``[999, 937, 833, 624, 0]``
-- ``use_mean_velocity``: ``true`` (i.e. ``r = t_next`` during rollout)
-- ``real_score_guidance_scale``: 3.0
-- ``generator_update_interval``: 5 (DMD2 alternation)
-- Training steps: 4000
-
-## 🔌 Loading published AnyFlow checkpoints
-
-The HF AnyFlow checkpoints expose ``condition_embedder.delta_embedder.*`` weights that FastVideo internally maps onto its ``condition_embedder.delta_embedder.mlp.*`` layout. This rename happens automatically through the regex in ``WanVideoArchConfig.param_names_mapping`` — no separate adapter is needed. The same regex is a no-op on plain Wan checkpoints (which don't contain any ``delta_embedder`` keys).
-
-Set the YAML's ``pipeline.dit_config.r_embedder: true`` to allocate the ``delta_embedder`` module on the FastVideo side; when initializing from a plain Wan checkpoint the delta weights are deep-copied from ``time_embedder`` (matching AnyFlow's ``setup_flowmap_model()`` behavior).
-
-## 🧭 Note on ``fuse_guidance_scale``
-
-Stage 1 optionally fuses classifier-free guidance into the training target so the resulting checkpoint can be sampled at ``guidance_scale=1.0`` (no extra forward pass at inference time). The transformation is
-
-```
-noise_pred ← (noise_pred - (1 - g) · noise_pred_uncond) / g
-```
-
-with ``g = fuse_guidance_scale``. The negative prompt embedding comes from ``WanModel``'s ``ensure_negative_conditioning()`` — i.e. the dataset's configured ``sampling_param.negative_prompt``. Setting ``fuse_guidance_scale: 1.0`` skips the extra unconditional forward entirely.
-
-The on-policy stage's ``real_score_guidance_scale`` (inherited from DMD2) follows the same parameterization conventions documented in [``dmd.md``](dmd.md#-note-on-real_score_guidance_scale).

docs/training/train_infra.md

@@ -161,18 +161,6 @@ training:
     decay_interval_steps: 0
 ```
 
-`training.data.data_path` can also mix multiple preprocessed datasets by using a mapping from dataset path to repeat count:
-
-```yaml
-training:
-  data:
-    data_path:
-      data/zeldam2-clean: 1
-      data/multi3d_games: 2
-```
-
-The repeat count duplicates that dataset's parquet file list before shuffling/sampling, so the example above trains with roughly twice as much `multi3d_games` exposure as `zeldam2-clean`. Paths are just suggested locations; use any local path that contains a FastVideo preprocessed parquet dataset.
-
 See [Training Trackers](trackers.md) to configure Weights & Biases or SwanLab,
 including SwanLab installation and authentication.
 
@@ -338,40 +326,6 @@ Self-Forcing inherits all DMD2 parameters, plus:
 | `enable_gradient_in_rollout` | `true` | Enable backprop through rollout |
 | `start_gradient_frame` | `0` | Frame index where gradients begin |
 
-### Streaming Long Tuning
-
-`StreamingLongTuningMethod` extends Self-Forcing for LongLive-style rollouts. It
-keeps a streaming state, generates overlapping chunks, and trains only the new
-frames while preserving context from earlier chunks.
-
-For the MatrixGame2/Zelda world-model example, self-forcing and long tuning are
-separate runs: first train or load the 1k-step self-forcing checkpoint using
-`examples/train/scenario/worldmodel/zelda/self_forcing_causal_i2v.yaml`,
-then run
-`examples/train/scenario/worldmodel/zelda/streaming_long_tuning_causal_i2v.yaml`
-from that checkpoint for the 3k-step streaming long-tuning stage.
-
-```yaml
-method:
-  _target_: fastvideo.train.methods.distribution_matching.streaming_long_tuning.StreamingLongTuningMethod
-  streaming_chunk_size: 9
-  streaming_max_length: 39
-  streaming_fixed_overlap_latents: 3
-  streaming_reencode_overlap_anchor: true
-  streaming_anchor_inject_k: 1
-  streaming_require_full_blocks: true
-  multi_phased_distill_schedule:
-    - stage: streaming_long
-      start_step: 0
-      end_step: 3000
-      num_latent_t: 39
-      streaming_training: true
-```
-
-See
-`examples/train/scenario/worldmodel/zelda/streaming_long_tuning_causal_i2v.yaml`
-for a complete MatrixGame2/Zelda configuration.
-
 ---
 
 ## Callbacks

examples/inference/basic/basic_flux_dev.py

@@ -1,140 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-
-from __future__ import annotations
-
-import argparse
-import contextlib
-import os
-import re
-
-DEFAULT_PROMPTS = [
-    "a photo of a cat",
-    (
-        "a cinematic photo of a red panda wearing a tiny backpack, standing on a "
-        "rainy neon-lit street at night, shallow depth of field, sharp focus, "
-        "35mm, bokeh"
-    ),
-]
-
-
-def _safe_filename(text: str, max_len: int = 100) -> str:
-    """Make a stable, filesystem-friendly filename base."""
-    s = text[:max_len].strip()
-    s = s.replace(os.sep, "_")
-    if os.altsep:
-        s = s.replace(os.altsep, "_")
-    s = re.sub(r"\s+", " ", s)
-    s = re.sub(r"[^A-Za-z0-9 .,_-]", "_", s)
-    s = s.strip(" .")
-    return s or "prompt"
-
-
-def _remove_existing_outputs(out_dir: str, filename_base: str) -> None:
-    """Delete prior outputs so reruns do not get _1, _2 suffixes."""
-    if not os.path.isdir(out_dir):
-        return
-
-    pattern = re.compile(rf"^{re.escape(filename_base)}(_\d+)?\.(mp4|png)$")
-    for fn in os.listdir(out_dir):
-        if pattern.match(fn):
-            with contextlib.suppress(FileNotFoundError):
-                os.remove(os.path.join(out_dir, fn))
-
-
-def parse_args() -> argparse.Namespace:
-    p = argparse.ArgumentParser(
-        description="Run FLUX.1-dev text-to-image with FastVideo VideoGenerator.",
-    )
-    p.add_argument(
-        "--model-path",
-        default="official_weights/FLUX.1-dev",
-        help="Local Diffusers checkpoint dir or HF repo id.",
-    )
-    p.add_argument(
-        "--out-dir",
-        "--outdir",
-        default="outputs/flux_dev/samples",
-        help="Directory for saved PNG outputs.",
-    )
-    p.add_argument(
-        "--prompt",
-        action="append",
-        default=None,
-        help="Prompt. Repeat for multiple images.",
-    )
-    p.add_argument(
-        "--backend",
-        default=None,
-        help="Set FASTVIDEO_ATTENTION_BACKEND (e.g. TORCH_SDPA).",
-    )
-    p.add_argument("--seed", type=int, default=42, help="Base seed; each prompt uses seed + index.")
-    p.add_argument("--height", type=int, default=1024, help="Output height.")
-    p.add_argument("--width", type=int, default=1024, help="Output width.")
-    p.add_argument("--steps", type=int, default=28, help="Number of inference steps.")
-    p.add_argument("--guidance", type=float, default=3.5, help="Guidance scale.")
-    p.add_argument("--num-gpus", type=int, default=1, help="GPU count.")
-    return p.parse_args()
-
-
-def main() -> None:
-    args = parse_args()
-    prompts: list[str] = args.prompt if args.prompt else DEFAULT_PROMPTS
-
-    if args.backend:
-        os.environ["FASTVIDEO_ATTENTION_BACKEND"] = args.backend
-
-    from fastvideo import VideoGenerator
-
-    os.makedirs(args.out_dir, exist_ok=True)
-
-    init_kwargs = {
-        "num_gpus": args.num_gpus,
-        "workload_type": "t2i",
-        "sp_size": 1,
-        "tp_size": 1,
-        "dit_cpu_offload": False,
-        "dit_layerwise_offload": False,
-        "text_encoder_cpu_offload": False,
-        "vae_cpu_offload": False,
-        "image_encoder_cpu_offload": False,
-        "pin_cpu_memory": False,
-        "use_fsdp_inference": False,
-    }
-
-    generator = VideoGenerator.from_pretrained(
-        model_path=args.model_path,
-        **init_kwargs,
-    )
-    try:
-        for i, prompt in enumerate(prompts):
-            seed = args.seed + i
-            filename_base = (
-                f"flux_dev_{i:02d}_seed{seed}_{_safe_filename(prompt, max_len=80)}"
-            )
-            _remove_existing_outputs(args.out_dir, filename_base)
-            output_path = os.path.join(args.out_dir, f"{filename_base}.png")
-            print(f"[flux] prompt_idx={i} seed={seed} output_path={output_path}")
-
-            generation_kwargs = {
-                "output_path": output_path,
-                "height": args.height,
-                "width": args.width,
-                "num_frames": 1,
-                "fps": 1,
-                "num_inference_steps": args.steps,
-                "guidance_scale": args.guidance,
-                "use_embedded_guidance": True,
-                "true_cfg_scale": 1.0,
-                "seed": seed,
-                "save_video": True,
-            }
-
-            generator.generate_video(prompt, **generation_kwargs)
-
-        print(f"[flux] done. outputs written to: {args.out_dir}")
-    finally:
-        generator.shutdown()
-
-
-if __name__ == "__main__":
-    main()

examples/inference/basic/basic_glm_image.py

@@ -1,107 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-"""Run GLM-Image text-to-image generation through FastVideo.
-
-User story:
-    "I have the HF `zai-org/GLM-Image` checkpoint and want a minimal
-    text-to-image generation command, saved as a PNG."
-"""
-import argparse
-from pathlib import Path
-
-from PIL import Image
-
-from fastvideo import VideoGenerator
-from fastvideo.api import (
-    EngineConfig,
-    GenerationRequest,
-    GeneratorConfig,
-    OutputConfig,
-    ParallelismConfig,
-    PipelineSelection,
-    SamplingConfig,
-)
-
-
-def parse_args() -> argparse.Namespace:
-    parser = argparse.ArgumentParser(description="Run GLM-Image text-to-image generation.")
-    parser.add_argument(
-        "--model-path",
-        default="zai-org/GLM-Image",
-        help="HF id or local diffusers-format GLM-Image weights directory.",
-    )
-    parser.add_argument(
-        "--output",
-        default="image_output/landscape.png",
-        help="Output PNG path.",
-    )
-    parser.add_argument(
-        "--prompt",
-        default=("A beautiful landscape photography with rolling hills, "
-                 "a winding river, and a vibrant sunset in the background. "
-                 "Warm golden light, photorealistic style."),
-        help="Text prompt.",
-    )
-    parser.add_argument("--height", type=int, default=1024)
-    parser.add_argument("--width", type=int, default=1024)
-    parser.add_argument("--steps", type=int, default=50)
-    parser.add_argument("--guidance-scale", type=float, default=1.5)
-    parser.add_argument("--seed", type=int, default=1024)
-    parser.add_argument("--num-gpus", type=int, default=1)
-    parser.add_argument("--tp-size", type=int, default=None)
-    parser.add_argument("--sp-size", type=int, default=None)
-    return parser.parse_args()
-
-
-def main() -> None:
-    args = parse_args()
-
-    output = Path(args.output)
-    output.parent.mkdir(parents=True, exist_ok=True)
-    tp_size = args.tp_size if args.tp_size is not None else (args.num_gpus if args.num_gpus > 1 else 1)
-    sp_size = args.sp_size if args.sp_size is not None else (1 if args.num_gpus > 1 else args.num_gpus)
-
-    # GLM-Image needs trust_remote_code for its AR encoder; offload and the
-    # pipeline class come from the model's registered defaults — don't override.
-    generator_config = GeneratorConfig(
-        model_path=args.model_path,
-        trust_remote_code=True,
-        engine=EngineConfig(
-            num_gpus=args.num_gpus,
-            parallelism=ParallelismConfig(tp_size=tp_size, sp_size=sp_size),
-        ),
-        pipeline=PipelineSelection(workload_type="t2i"),
-    )
-
-    generator = VideoGenerator.from_config(generator_config)
-    try:
-        request = GenerationRequest(
-            prompt=args.prompt,
-            sampling=SamplingConfig(
-                height=args.height,
-                width=args.width,
-                num_frames=1,
-                fps=1,
-                num_inference_steps=args.steps,
-                guidance_scale=args.guidance_scale,
-                seed=args.seed,
-            ),
-            output=OutputConfig(
-                output_path=str(output.parent),
-                save_video=False,
-                return_frames=True,
-            ),
-        )
-        result = generator.generate(request)
-        if isinstance(result, list):
-            result = result[0]
-
-        frames = result.frames
-        if frames is not None and len(frames):
-            Image.fromarray(frames[0]).save(output)
-            print(f"Saved image to {output}")
-    finally:
-        generator.shutdown()
-
-
-if __name__ == "__main__":
-    main()

examples/inference/basic/edit_glm_image.py

@@ -1,120 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-"""Run GLM-Image image-to-image (edit) generation through FastVideo.
-
-User story:
-    "I have the HF `zai-org/GLM-Image` checkpoint and a condition image, and
-    want a minimal edit command (text + image -> edited image), saved as a PNG."
-
-GLM-Image is a single unified pipeline: passing a condition image switches it
-from text-to-image to the edit path (the condition enters the DiT via a KV-cache
-write pass), so the generator config is identical to `basic_glm_image.py` — the
-`inputs.pil_image` on the request is what selects the edit mode.
-"""
-import argparse
-from pathlib import Path
-
-from PIL import Image
-
-from fastvideo import VideoGenerator
-from fastvideo.api import (
-    EngineConfig,
-    GenerationRequest,
-    GeneratorConfig,
-    InputConfig,
-    OutputConfig,
-    ParallelismConfig,
-    PipelineSelection,
-    SamplingConfig,
-)
-
-
-def parse_args() -> argparse.Namespace:
-    parser = argparse.ArgumentParser(description="Run GLM-Image image-to-image (edit) generation.")
-    parser.add_argument(
-        "--model-path",
-        default="zai-org/GLM-Image",
-        help="HF id or local diffusers-format GLM-Image weights directory.",
-    )
-    parser.add_argument(
-        "--image",
-        default="assets/images/couple.jpg",
-        help="Condition image to edit.",
-    )
-    parser.add_argument(
-        "--output",
-        default="image_output/edited.png",
-        help="Output PNG path.",
-    )
-    parser.add_argument(
-        "--prompt",
-        default="Change the background to a snowy mountain landscape at golden hour.",
-        help="Edit instruction.",
-    )
-    parser.add_argument("--height", type=int, default=1024)
-    parser.add_argument("--width", type=int, default=1024)
-    parser.add_argument("--steps", type=int, default=50)
-    parser.add_argument("--guidance-scale", type=float, default=1.5)
-    parser.add_argument("--seed", type=int, default=1024)
-    parser.add_argument("--num-gpus", type=int, default=1)
-    parser.add_argument("--tp-size", type=int, default=None)
-    parser.add_argument("--sp-size", type=int, default=None)
-    return parser.parse_args()
-
-
-def main() -> None:
-    args = parse_args()
-
-    output = Path(args.output)
-    output.parent.mkdir(parents=True, exist_ok=True)
-    condition = Image.open(args.image).convert("RGB")
-    tp_size = args.tp_size if args.tp_size is not None else (args.num_gpus if args.num_gpus > 1 else 1)
-    sp_size = args.sp_size if args.sp_size is not None else (1 if args.num_gpus > 1 else args.num_gpus)
-
-    # GLM-Image needs trust_remote_code for its AR encoder; offload and the
-    # pipeline class come from the model's registered defaults — don't override.
-    # The pipeline is registered as t2i; passing inputs.pil_image below switches
-    # it to the edit path.
-    generator_config = GeneratorConfig(
-        model_path=args.model_path,
-        trust_remote_code=True,
-        engine=EngineConfig(
-            num_gpus=args.num_gpus,
-            parallelism=ParallelismConfig(tp_size=tp_size, sp_size=sp_size),
-        ),
-        pipeline=PipelineSelection(workload_type="t2i"),
-    )
-
-    generator = VideoGenerator.from_config(generator_config)
-    try:
-        request = GenerationRequest(
-            prompt=args.prompt,
-            inputs=InputConfig(pil_image=condition),
-            sampling=SamplingConfig(
-                height=args.height,
-                width=args.width,
-                num_frames=1,
-                fps=1,
-                num_inference_steps=args.steps,
-                guidance_scale=args.guidance_scale,
-                seed=args.seed,
-            ),
-            output=OutputConfig(
-                output_path=str(output.parent),
-                save_video=False,
-                return_frames=True,
-            ),
-        )
-        result = generator.generate(request)
-        if isinstance(result, list):
-            result = result[0]
-
-        frames = result.frames
-        if frames is not None and len(frames):
-            Image.fromarray(frames[0]).save(output)
-            print(f"Saved image to {output}")
-    finally:
-        generator.shutdown()
-
-
-if __name__ == "__main__":
-    main()

examples/inference/ltx2_3/README.md

@@ -1,69 +0,0 @@
-# LTX-2.3 distilled inference configs
-
-Ready-to-run `fastvideo generate` run configs for the LTX-2.3
-distilled model (`FastVideo/LTX-2.3-Distilled-Diffusers`), covering both
-workloads (t2v / i2v), both two-stage step schedules (`5+2`, `8+3` = denoise
-+ refine), and four resolutions.
-
-```bash
-fastvideo generate --config examples/inference/ltx2_3/t2v_8s3_1280x832.yaml
-```
-
-Each config is self-contained (no preset registry needed): the two-stage
-refine is wired via `generator.pipeline.preset_overrides.refine`, and the
-base sampling knobs live under `request.sampling`. The refine upsampler
-auto-resolves from the model's `spatial_upscaler`.
-
-## Configs
-
-| workload | schedule | resolution (HxW) | file |
-|---|---|---|---|
-| t2v | 5+2 | 1280x832 | `t2v_5s2_1280x832.yaml` |
-| t2v | 5+2 | 1024x1536 | `t2v_5s2_1024x1536.yaml` |
-| t2v | 5+2 | 768x1280 | `t2v_5s2_768x1280.yaml` |
-| t2v | 5+2 | 512x768 | `t2v_5s2_512x768.yaml` |
-| t2v | 8+3 | 1280x832 | `t2v_8s3_1280x832.yaml` |
-| t2v | 8+3 | 1024x1536 | `t2v_8s3_1024x1536.yaml` |
-| t2v | 8+3 | 768x1280 | `t2v_8s3_768x1280.yaml` |
-| t2v | 8+3 | 512x768 | `t2v_8s3_512x768.yaml` |
-| i2v | 5+2 | 1280x832 | `i2v_5s2_1280x832.yaml` |
-| i2v | 5+2 | 1024x1536 | `i2v_5s2_1024x1536.yaml` |
-| i2v | 5+2 | 768x1280 | `i2v_5s2_768x1280.yaml` |
-| i2v | 5+2 | 512x768 | `i2v_5s2_512x768.yaml` |
-| i2v | 8+3 | 1280x832 | `i2v_8s3_1280x832.yaml` |
-| i2v | 8+3 | 1024x1536 | `i2v_8s3_1024x1536.yaml` |
-| i2v | 8+3 | 768x1280 | `i2v_8s3_768x1280.yaml` |
-| i2v | 8+3 | 512x768 | `i2v_8s3_512x768.yaml` |
-
-## Overriding without editing a file
-
-Dotted overrides (prefixes `generator.` / `request.`) let you tweak any field:
-
-```bash
-# swap prompt
-fastvideo generate --config examples/inference/ltx2_3/t2v_8s3_1280x832.yaml \
-  --request.prompt "a red fox running through fresh snow"
-
-# change output path / gpu count
-fastvideo generate --config examples/inference/ltx2_3/t2v_5s2_512x768.yaml \
-  --request.output.output_path outputs/preview.mp4 \
-  --generator.engine.num_gpus 4
-```
-
-## i2v
-
-The `i2v_*` configs take a first-frame image via
-`request.extensions.ltx2_images` (`[[path, frame_offset, weight]]`). Edit the
-path in the file, or override it:
-
-```bash
-fastvideo generate --config examples/inference/ltx2_3/i2v_8s3_1280x832.yaml \
-  --request.extensions.ltx2_images '[["/data/portrait.jpg", 0, 1.0]]'
-```
-
-## Schedules
-
-`5+2` is the fast preview schedule; `8+3` is the higher-quality distilled
-recipe. Refine (`preset_overrides.refine.num_inference_steps`) only accepts 2
-or 3 steps.
-

examples/inference/ltx2_3/i2v_5s2_1024x1536.yaml

@@ -1,38 +0,0 @@
-# LTX-2.3 distilled i2v — 5+2 two-stage at 1024x1536.
-#
-# Run:
-#   fastvideo generate --config examples/inference/ltx2_3/i2v_5s2_1024x1536.yaml
-#
-# Stage 1 denoises for 5 steps at half resolution; the latents are then
-# spatially upsampled and refined for 2 steps (refine only supports 2 or 3).
-# The refine upsampler auto-resolves from the model's `spatial_upscaler`.
-generator:
-  model_path: FastVideo/LTX-2.3-Distilled-Diffusers
-  engine:
-    num_gpus: 1
-  pipeline:
-    workload_type: i2v
-    preset_overrides:
-      refine:
-        enabled: true
-        num_inference_steps: 2
-        guidance_scale: 1.0
-        add_noise: true
-request:
-  prompt: >-
-    The subject slowly turns toward the camera with a soft, natural expression, hair and clothing swaying gently, shallow depth of field, subtle cinematic motion.
-  sampling:
-    height: 1024
-    width: 1536
-    num_frames: 121
-    fps: 24
-    guidance_scale: 1.0
-    num_inference_steps: 5
-  # i2v conditioning — replace the path with your own first-frame image.
-  extensions:
-    ltx2_images:
-      - ["/path/to/your/first_frame.jpg", 0, 1.0]
-    ltx2_image_crf: 0.0
-  output:
-    output_path: outputs/ltx2_3_i2v_5s2_1024x1536.mp4
-    save_video: true

examples/inference/ltx2_3/i2v_5s2_1280x832.yaml

@@ -1,38 +0,0 @@
-# LTX-2.3 distilled i2v — 5+2 two-stage at 1280x832.
-#
-# Run:
-#   fastvideo generate --config examples/inference/ltx2_3/i2v_5s2_1280x832.yaml
-#
-# Stage 1 denoises for 5 steps at half resolution; the latents are then
-# spatially upsampled and refined for 2 steps (refine only supports 2 or 3).
-# The refine upsampler auto-resolves from the model's `spatial_upscaler`.
-generator:
-  model_path: FastVideo/LTX-2.3-Distilled-Diffusers
-  engine:
-    num_gpus: 1
-  pipeline:
-    workload_type: i2v
-    preset_overrides:
-      refine:
-        enabled: true
-        num_inference_steps: 2
-        guidance_scale: 1.0
-        add_noise: true
-request:
-  prompt: >-
-    The subject slowly turns toward the camera with a soft, natural expression, hair and clothing swaying gently, shallow depth of field, subtle cinematic motion.
-  sampling:
-    height: 1280
-    width: 832
-    num_frames: 121
-    fps: 24
-    guidance_scale: 1.0
-    num_inference_steps: 5
-  # i2v conditioning — replace the path with your own first-frame image.
-  extensions:
-    ltx2_images:
-      - ["/path/to/your/first_frame.jpg", 0, 1.0]
-    ltx2_image_crf: 0.0
-  output:
-    output_path: outputs/ltx2_3_i2v_5s2_1280x832.mp4
-    save_video: true

examples/inference/ltx2_3/i2v_5s2_512x768.yaml

@@ -1,38 +0,0 @@
-# LTX-2.3 distilled i2v — 5+2 two-stage at 512x768.
-#
-# Run:
-#   fastvideo generate --config examples/inference/ltx2_3/i2v_5s2_512x768.yaml
-#
-# Stage 1 denoises for 5 steps at half resolution; the latents are then
-# spatially upsampled and refined for 2 steps (refine only supports 2 or 3).
-# The refine upsampler auto-resolves from the model's `spatial_upscaler`.
-generator:
-  model_path: FastVideo/LTX-2.3-Distilled-Diffusers
-  engine:
-    num_gpus: 1
-  pipeline:
-    workload_type: i2v
-    preset_overrides:
-      refine:
-        enabled: true
-        num_inference_steps: 2
-        guidance_scale: 1.0
-        add_noise: true
-request:
-  prompt: >-
-    The subject slowly turns toward the camera with a soft, natural expression, hair and clothing swaying gently, shallow depth of field, subtle cinematic motion.
-  sampling:
-    height: 512
-    width: 768
-    num_frames: 121
-    fps: 24
-    guidance_scale: 1.0
-    num_inference_steps: 5
-  # i2v conditioning — replace the path with your own first-frame image.
-  extensions:
-    ltx2_images:
-      - ["/path/to/your/first_frame.jpg", 0, 1.0]
-    ltx2_image_crf: 0.0
-  output:
-    output_path: outputs/ltx2_3_i2v_5s2_512x768.mp4
-    save_video: true

examples/inference/ltx2_3/i2v_5s2_768x1280.yaml

@@ -1,38 +0,0 @@
-# LTX-2.3 distilled i2v — 5+2 two-stage at 768x1280.
-#
-# Run:
-#   fastvideo generate --config examples/inference/ltx2_3/i2v_5s2_768x1280.yaml
-#
-# Stage 1 denoises for 5 steps at half resolution; the latents are then
-# spatially upsampled and refined for 2 steps (refine only supports 2 or 3).
-# The refine upsampler auto-resolves from the model's `spatial_upscaler`.
-generator:
-  model_path: FastVideo/LTX-2.3-Distilled-Diffusers
-  engine:
-    num_gpus: 1
-  pipeline:
-    workload_type: i2v
-    preset_overrides:
-      refine:
-        enabled: true
-        num_inference_steps: 2
-        guidance_scale: 1.0
-        add_noise: true
-request:
-  prompt: >-
-    The subject slowly turns toward the camera with a soft, natural expression, hair and clothing swaying gently, shallow depth of field, subtle cinematic motion.
-  sampling:
-    height: 768
-    width: 1280
-    num_frames: 121
-    fps: 24
-    guidance_scale: 1.0
-    num_inference_steps: 5
-  # i2v conditioning — replace the path with your own first-frame image.
-  extensions:
-    ltx2_images:
-      - ["/path/to/your/first_frame.jpg", 0, 1.0]
-    ltx2_image_crf: 0.0
-  output:
-    output_path: outputs/ltx2_3_i2v_5s2_768x1280.mp4
-    save_video: true

examples/inference/ltx2_3/i2v_8s3_1024x1536.yaml

@@ -1,38 +0,0 @@
-# LTX-2.3 distilled i2v — 8+3 two-stage at 1024x1536.
-#
-# Run:
-#   fastvideo generate --config examples/inference/ltx2_3/i2v_8s3_1024x1536.yaml
-#
-# Stage 1 denoises for 8 steps at half resolution; the latents are then
-# spatially upsampled and refined for 3 steps (refine only supports 2 or 3).
-# The refine upsampler auto-resolves from the model's `spatial_upscaler`.
-generator:
-  model_path: FastVideo/LTX-2.3-Distilled-Diffusers
-  engine:
-    num_gpus: 1
-  pipeline:
-    workload_type: i2v
-    preset_overrides:
-      refine:
-        enabled: true
-        num_inference_steps: 3
-        guidance_scale: 1.0
-        add_noise: true
-request:
-  prompt: >-
-    The subject slowly turns toward the camera with a soft, natural expression, hair and clothing swaying gently, shallow depth of field, subtle cinematic motion.
-  sampling:
-    height: 1024
-    width: 1536
-    num_frames: 121
-    fps: 24
-    guidance_scale: 1.0
-    num_inference_steps: 8
-  # i2v conditioning — replace the path with your own first-frame image.
-  extensions:
-    ltx2_images:
-      - ["/path/to/your/first_frame.jpg", 0, 1.0]
-    ltx2_image_crf: 0.0
-  output:
-    output_path: outputs/ltx2_3_i2v_8s3_1024x1536.mp4
-    save_video: true

examples/inference/ltx2_3/i2v_8s3_1280x832.yaml

@@ -1,38 +0,0 @@
-# LTX-2.3 distilled i2v — 8+3 two-stage at 1280x832.
-#
-# Run:
-#   fastvideo generate --config examples/inference/ltx2_3/i2v_8s3_1280x832.yaml
-#
-# Stage 1 denoises for 8 steps at half resolution; the latents are then
-# spatially upsampled and refined for 3 steps (refine only supports 2 or 3).
-# The refine upsampler auto-resolves from the model's `spatial_upscaler`.
-generator:
-  model_path: FastVideo/LTX-2.3-Distilled-Diffusers
-  engine:
-    num_gpus: 1
-  pipeline:
-    workload_type: i2v
-    preset_overrides:
-      refine:
-        enabled: true
-        num_inference_steps: 3
-        guidance_scale: 1.0
-        add_noise: true
-request:
-  prompt: >-
-    The subject slowly turns toward the camera with a soft, natural expression, hair and clothing swaying gently, shallow depth of field, subtle cinematic motion.
-  sampling:
-    height: 1280
-    width: 832
-    num_frames: 121
-    fps: 24
-    guidance_scale: 1.0
-    num_inference_steps: 8
-  # i2v conditioning — replace the path with your own first-frame image.
-  extensions:
-    ltx2_images:
-      - ["/path/to/your/first_frame.jpg", 0, 1.0]
-    ltx2_image_crf: 0.0
-  output:
-    output_path: outputs/ltx2_3_i2v_8s3_1280x832.mp4
-    save_video: true

examples/inference/ltx2_3/i2v_8s3_512x768.yaml

@@ -1,38 +0,0 @@
-# LTX-2.3 distilled i2v — 8+3 two-stage at 512x768.
-#
-# Run:
-#   fastvideo generate --config examples/inference/ltx2_3/i2v_8s3_512x768.yaml
-#
-# Stage 1 denoises for 8 steps at half resolution; the latents are then
-# spatially upsampled and refined for 3 steps (refine only supports 2 or 3).
-# The refine upsampler auto-resolves from the model's `spatial_upscaler`.
-generator:
-  model_path: FastVideo/LTX-2.3-Distilled-Diffusers
-  engine:
-    num_gpus: 1
-  pipeline:
-    workload_type: i2v
-    preset_overrides:
-      refine:
-        enabled: true
-        num_inference_steps: 3
-        guidance_scale: 1.0
-        add_noise: true
-request:
-  prompt: >-
-    The subject slowly turns toward the camera with a soft, natural expression, hair and clothing swaying gently, shallow depth of field, subtle cinematic motion.
-  sampling:
-    height: 512
-    width: 768
-    num_frames: 121
-    fps: 24
-    guidance_scale: 1.0
-    num_inference_steps: 8
-  # i2v conditioning — replace the path with your own first-frame image.
-  extensions:
-    ltx2_images:
-      - ["/path/to/your/first_frame.jpg", 0, 1.0]
-    ltx2_image_crf: 0.0
-  output:
-    output_path: outputs/ltx2_3_i2v_8s3_512x768.mp4
-    save_video: true

examples/inference/ltx2_3/i2v_8s3_768x1280.yaml

@@ -1,38 +0,0 @@
-# LTX-2.3 distilled i2v — 8+3 two-stage at 768x1280.
-#
-# Run:
-#   fastvideo generate --config examples/inference/ltx2_3/i2v_8s3_768x1280.yaml
-#
-# Stage 1 denoises for 8 steps at half resolution; the latents are then
-# spatially upsampled and refined for 3 steps (refine only supports 2 or 3).
-# The refine upsampler auto-resolves from the model's `spatial_upscaler`.
-generator:
-  model_path: FastVideo/LTX-2.3-Distilled-Diffusers
-  engine:
-    num_gpus: 1
-  pipeline:
-    workload_type: i2v
-    preset_overrides:
-      refine:
-        enabled: true
-        num_inference_steps: 3
-        guidance_scale: 1.0
-        add_noise: true
-request:
-  prompt: >-
-    The subject slowly turns toward the camera with a soft, natural expression, hair and clothing swaying gently, shallow depth of field, subtle cinematic motion.
-  sampling:
-    height: 768
-    width: 1280
-    num_frames: 121
-    fps: 24
-    guidance_scale: 1.0
-    num_inference_steps: 8
-  # i2v conditioning — replace the path with your own first-frame image.
-  extensions:
-    ltx2_images:
-      - ["/path/to/your/first_frame.jpg", 0, 1.0]
-    ltx2_image_crf: 0.0
-  output:
-    output_path: outputs/ltx2_3_i2v_8s3_768x1280.mp4
-    save_video: true

examples/inference/ltx2_3/t2v_5s2_1024x1536.yaml

@@ -1,33 +0,0 @@
-# LTX-2.3 distilled t2v — 5+2 two-stage at 1024x1536.
-#
-# Run:
-#   fastvideo generate --config examples/inference/ltx2_3/t2v_5s2_1024x1536.yaml
-#
-# Stage 1 denoises for 5 steps at half resolution; the latents are then
-# spatially upsampled and refined for 2 steps (refine only supports 2 or 3).
-# The refine upsampler auto-resolves from the model's `spatial_upscaler`.
-generator:
-  model_path: FastVideo/LTX-2.3-Distilled-Diffusers
-  engine:
-    num_gpus: 1
-  pipeline:
-    workload_type: t2v
-    preset_overrides:
-      refine:
-        enabled: true
-        num_inference_steps: 2
-        guidance_scale: 1.0
-        add_noise: true
-request:
-  prompt: >-
-    A cinematic drone shot flying over dramatic coastal cliffs at sunrise, golden light spilling across the water, gentle waves breaking on the rocks below, ultra-detailed, smooth camera motion.
-  sampling:
-    height: 1024
-    width: 1536
-    num_frames: 121
-    fps: 24
-    guidance_scale: 1.0
-    num_inference_steps: 5
-  output:
-    output_path: outputs/ltx2_3_t2v_5s2_1024x1536.mp4
-    save_video: true

examples/inference/ltx2_3/t2v_5s2_1280x832.yaml

@@ -1,33 +0,0 @@
-# LTX-2.3 distilled t2v — 5+2 two-stage at 1280x832.
-#
-# Run:
-#   fastvideo generate --config examples/inference/ltx2_3/t2v_5s2_1280x832.yaml
-#
-# Stage 1 denoises for 5 steps at half resolution; the latents are then
-# spatially upsampled and refined for 2 steps (refine only supports 2 or 3).
-# The refine upsampler auto-resolves from the model's `spatial_upscaler`.
-generator:
-  model_path: FastVideo/LTX-2.3-Distilled-Diffusers
-  engine:
-    num_gpus: 1
-  pipeline:
-    workload_type: t2v
-    preset_overrides:
-      refine:
-        enabled: true
-        num_inference_steps: 2
-        guidance_scale: 1.0
-        add_noise: true
-request:
-  prompt: >-
-    A cinematic drone shot flying over dramatic coastal cliffs at sunrise, golden light spilling across the water, gentle waves breaking on the rocks below, ultra-detailed, smooth camera motion.
-  sampling:
-    height: 1280
-    width: 832
-    num_frames: 121
-    fps: 24
-    guidance_scale: 1.0
-    num_inference_steps: 5
-  output:
-    output_path: outputs/ltx2_3_t2v_5s2_1280x832.mp4
-    save_video: true

examples/inference/ltx2_3/t2v_5s2_512x768.yaml

@@ -1,33 +0,0 @@
-# LTX-2.3 distilled t2v — 5+2 two-stage at 512x768.
-#
-# Run:
-#   fastvideo generate --config examples/inference/ltx2_3/t2v_5s2_512x768.yaml
-#
-# Stage 1 denoises for 5 steps at half resolution; the latents are then
-# spatially upsampled and refined for 2 steps (refine only supports 2 or 3).
-# The refine upsampler auto-resolves from the model's `spatial_upscaler`.
-generator:
-  model_path: FastVideo/LTX-2.3-Distilled-Diffusers
-  engine:
-    num_gpus: 1
-  pipeline:
-    workload_type: t2v
-    preset_overrides:
-      refine:
-        enabled: true
-        num_inference_steps: 2
-        guidance_scale: 1.0
-        add_noise: true
-request:
-  prompt: >-
-    A cinematic drone shot flying over dramatic coastal cliffs at sunrise, golden light spilling across the water, gentle waves breaking on the rocks below, ultra-detailed, smooth camera motion.
-  sampling:
-    height: 512
-    width: 768
-    num_frames: 121
-    fps: 24
-    guidance_scale: 1.0
-    num_inference_steps: 5
-  output:
-    output_path: outputs/ltx2_3_t2v_5s2_512x768.mp4
-    save_video: true

examples/inference/ltx2_3/t2v_5s2_768x1280.yaml

@@ -1,33 +0,0 @@
-# LTX-2.3 distilled t2v — 5+2 two-stage at 768x1280.
-#
-# Run:
-#   fastvideo generate --config examples/inference/ltx2_3/t2v_5s2_768x1280.yaml
-#
-# Stage 1 denoises for 5 steps at half resolution; the latents are then
-# spatially upsampled and refined for 2 steps (refine only supports 2 or 3).
-# The refine upsampler auto-resolves from the model's `spatial_upscaler`.
-generator:
-  model_path: FastVideo/LTX-2.3-Distilled-Diffusers
-  engine:
-    num_gpus: 1
-  pipeline:
-    workload_type: t2v
-    preset_overrides:
-      refine:
-        enabled: true
-        num_inference_steps: 2
-        guidance_scale: 1.0
-        add_noise: true
-request:
-  prompt: >-
-    A cinematic drone shot flying over dramatic coastal cliffs at sunrise, golden light spilling across the water, gentle waves breaking on the rocks below, ultra-detailed, smooth camera motion.
-  sampling:
-    height: 768
-    width: 1280
-    num_frames: 121
-    fps: 24
-    guidance_scale: 1.0
-    num_inference_steps: 5
-  output:
-    output_path: outputs/ltx2_3_t2v_5s2_768x1280.mp4
-    save_video: true

examples/inference/ltx2_3/t2v_8s3_1024x1536.yaml

@@ -1,33 +0,0 @@
-# LTX-2.3 distilled t2v — 8+3 two-stage at 1024x1536.
-#
-# Run:
-#   fastvideo generate --config examples/inference/ltx2_3/t2v_8s3_1024x1536.yaml
-#
-# Stage 1 denoises for 8 steps at half resolution; the latents are then
-# spatially upsampled and refined for 3 steps (refine only supports 2 or 3).
-# The refine upsampler auto-resolves from the model's `spatial_upscaler`.
-generator:
-  model_path: FastVideo/LTX-2.3-Distilled-Diffusers
-  engine:
-    num_gpus: 1
-  pipeline:
-    workload_type: t2v
-    preset_overrides:
-      refine:
-        enabled: true
-        num_inference_steps: 3
-        guidance_scale: 1.0
-        add_noise: true
-request:
-  prompt: >-
-    A cinematic drone shot flying over dramatic coastal cliffs at sunrise, golden light spilling across the water, gentle waves breaking on the rocks below, ultra-detailed, smooth camera motion.
-  sampling:
-    height: 1024
-    width: 1536
-    num_frames: 121
-    fps: 24
-    guidance_scale: 1.0
-    num_inference_steps: 8
-  output:
-    output_path: outputs/ltx2_3_t2v_8s3_1024x1536.mp4
-    save_video: true

examples/inference/ltx2_3/t2v_8s3_1280x832.yaml

@@ -1,33 +0,0 @@
-# LTX-2.3 distilled t2v — 8+3 two-stage at 1280x832.
-#
-# Run:
-#   fastvideo generate --config examples/inference/ltx2_3/t2v_8s3_1280x832.yaml
-#
-# Stage 1 denoises for 8 steps at half resolution; the latents are then
-# spatially upsampled and refined for 3 steps (refine only supports 2 or 3).
-# The refine upsampler auto-resolves from the model's `spatial_upscaler`.
-generator:
-  model_path: FastVideo/LTX-2.3-Distilled-Diffusers
-  engine:
-    num_gpus: 1
-  pipeline:
-    workload_type: t2v
-    preset_overrides:
-      refine:
-        enabled: true
-        num_inference_steps: 3
-        guidance_scale: 1.0
-        add_noise: true
-request:
-  prompt: >-
-    A cinematic drone shot flying over dramatic coastal cliffs at sunrise, golden light spilling across the water, gentle waves breaking on the rocks below, ultra-detailed, smooth camera motion.
-  sampling:
-    height: 1280
-    width: 832
-    num_frames: 121
-    fps: 24
-    guidance_scale: 1.0
-    num_inference_steps: 8
-  output:
-    output_path: outputs/ltx2_3_t2v_8s3_1280x832.mp4
-    save_video: true

examples/inference/ltx2_3/t2v_8s3_512x768.yaml

@@ -1,33 +0,0 @@
-# LTX-2.3 distilled t2v — 8+3 two-stage at 512x768.
-#
-# Run:
-#   fastvideo generate --config examples/inference/ltx2_3/t2v_8s3_512x768.yaml
-#
-# Stage 1 denoises for 8 steps at half resolution; the latents are then
-# spatially upsampled and refined for 3 steps (refine only supports 2 or 3).
-# The refine upsampler auto-resolves from the model's `spatial_upscaler`.
-generator:
-  model_path: FastVideo/LTX-2.3-Distilled-Diffusers
-  engine:
-    num_gpus: 1
-  pipeline:
-    workload_type: t2v
-    preset_overrides:
-      refine:
-        enabled: true
-        num_inference_steps: 3
-        guidance_scale: 1.0
-        add_noise: true
-request:
-  prompt: >-
-    A cinematic drone shot flying over dramatic coastal cliffs at sunrise, golden light spilling across the water, gentle waves breaking on the rocks below, ultra-detailed, smooth camera motion.
-  sampling:
-    height: 512
-    width: 768
-    num_frames: 121
-    fps: 24
-    guidance_scale: 1.0
-    num_inference_steps: 8
-  output:
-    output_path: outputs/ltx2_3_t2v_8s3_512x768.mp4
-    save_video: true

examples/inference/ltx2_3/t2v_8s3_768x1280.yaml

@@ -1,33 +0,0 @@
-# LTX-2.3 distilled t2v — 8+3 two-stage at 768x1280.
-#
-# Run:
-#   fastvideo generate --config examples/inference/ltx2_3/t2v_8s3_768x1280.yaml
-#
-# Stage 1 denoises for 8 steps at half resolution; the latents are then
-# spatially upsampled and refined for 3 steps (refine only supports 2 or 3).
-# The refine upsampler auto-resolves from the model's `spatial_upscaler`.
-generator:
-  model_path: FastVideo/LTX-2.3-Distilled-Diffusers
-  engine:
-    num_gpus: 1
-  pipeline:
-    workload_type: t2v
-    preset_overrides:
-      refine:
-        enabled: true
-        num_inference_steps: 3
-        guidance_scale: 1.0
-        add_noise: true
-request:
-  prompt: >-
-    A cinematic drone shot flying over dramatic coastal cliffs at sunrise, golden light spilling across the water, gentle waves breaking on the rocks below, ultra-detailed, smooth camera motion.
-  sampling:
-    height: 768
-    width: 1280
-    num_frames: 121
-    fps: 24
-    guidance_scale: 1.0
-    num_inference_steps: 8
-  output:
-    output_path: outputs/ltx2_3_t2v_8s3_768x1280.mp4
-    save_video: true

examples/train/configs/distribution_matching/wan/anyflow_onpolicy_t2v.yaml

@@ -1,105 +0,0 @@
-# AnyFlow on-policy DMD — Wan 2.1 T2V 1.3B.
-#
-# Stage 2 of the AnyFlow two-stage recipe. Continues from the pretrain
-# checkpoint; refines the student via DMD2 with a multi-step Euler-flow
-# rollout from pure noise. Teacher provides the real score, critic
-# learns the fake score; both inherited from DMD2Method.
-#
-# Replace <PATH_TO_PRETRAIN_CKPT> with the output of the pretrain stage,
-# or with the NVIDIA-released checkpoint
-# nvidia/AnyFlow-Wan2.1-T2V-1.3B-Diffusers to bootstrap directly from
-# the paper weights (the delta_embedder rename is handled by the
-# param_names_mapping in WanVideoArchConfig).
-
-models:
-  student:
-    _target_: fastvideo.train.models.wan.WanModel
-    init_from: <PATH_TO_PRETRAIN_CKPT>
-    trainable: true
-  teacher:
-    _target_: fastvideo.train.models.wan.WanModel
-    init_from: Wan-AI/Wan2.1-T2V-14B-Diffusers
-    trainable: false
-    disable_custom_init_weights: true
-  critic:
-    _target_: fastvideo.train.models.wan.WanModel
-    init_from: Wan-AI/Wan2.1-T2V-1.3B-Diffusers
-    trainable: true
-    disable_custom_init_weights: true
-
-method:
-  _target_: fastvideo.train.methods.distribution_matching.anyflow.AnyFlowMethod
-  rollout_mode: simulate
-  generator_update_interval: 5
-  real_score_guidance_scale: 3.0
-  dmd_denoising_steps: [999, 937, 833, 624]
-  warp_denoising_step: false
-
-  # AnyFlow rollout knobs.
-  student_sample_steps: 4
-  use_mean_velocity: true
-  t_list_override: [999.0, 937.0, 833.0, 624.0, 0.0]
-  dmd_score_r_value: 0.0  # DMD scoring conditioning is at r=0 (consistency target).
-
-  # Critic optimizer (DMD2 inherited).
-  fake_score_learning_rate: 8.0e-6
-  fake_score_betas: [0.0, 0.999]
-  fake_score_lr_scheduler: constant
-
-  attn_kind: vsa
-
-training:
-  distributed:
-    num_gpus: 8
-    sp_size: 1
-    tp_size: 1
-    hsdp_replicate_dim: 1
-    hsdp_shard_dim: 8
-
-  data:
-    data_path: data/preprocessed
-    dataloader_num_workers: 4
-    train_batch_size: 1
-    training_cfg_rate: 0.0
-    seed: 1000
-    num_latent_t: 21
-    num_height: 480
-    num_width: 832
-    num_frames: 81
-
-  optimizer:
-    learning_rate: 2.0e-6
-    betas: [0.0, 0.999]
-    weight_decay: 0.01
-    lr_scheduler: constant
-    lr_warmup_steps: 0
-
-  loop:
-    max_train_steps: 4000
-    gradient_accumulation_steps: 1
-
-  checkpoint:
-    output_dir: outputs/wan2.1_anyflow_onpolicy
-    training_state_checkpointing_steps: 500
-    checkpoints_total_limit: 3
-    resume_from_checkpoint: latest
-
-  tracker:
-    project_name: anyflow-wan
-    run_name: wan2.1_t2v_anyflow_onpolicy
-
-  model:
-    enable_gradient_checkpointing_type: full
-
-callbacks:
-  grad_clip:
-    _target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback
-    max_grad_norm: 1.0
-
-pipeline:
-  flow_shift: 5.0
-  dit_config:
-    r_embedder: true
-    r_embedder_fusion: gated
-    r_embedder_gate_value: 0.25
-    r_embedder_deltatime_type: r

examples/train/configs/distribution_matching/wan/anyflow_pretrain_t2v.yaml

@@ -1,83 +0,0 @@
-# AnyFlow pretrain (flow-map central-difference) — Wan 2.1 T2V 1.3B.
-#
-# Stage 1 of the AnyFlow two-stage recipe. Trains the dual-timestep
-# u_θ(x_t, t, r) on the central-difference target so the same checkpoint
-# can be sampled at arbitrary NFE in the on-policy stage.
-#
-# Initialize from base Wan 2.1 T2V 1.3B. No teacher or critic at this
-# stage; AnyFlowPretrainMethod owns a single student + one optimizer.
-
-models:
-  student:
-    _target_: fastvideo.train.models.wan.WanModel
-    init_from: Wan-AI/Wan2.1-T2V-1.3B-Diffusers
-    trainable: true
-
-method:
-  _target_: fastvideo.train.methods.distribution_matching.anyflow_pretrain.AnyFlowPretrainMethod
-  diffusion_ratio: 0.5
-  consistency_ratio: 0.25
-  epsilon: 5            # finite-difference step in absolute train-timestep units
-  weight_type: beta08   # per-timestep loss weight = t * sqrt(1 - t), renormalized
-  fuse_guidance_scale: 3.0
-  # shift is taken from pipeline.flow_shift below.
-
-training:
-  distributed:
-    num_gpus: 8
-    sp_size: 1
-    tp_size: 1
-    hsdp_replicate_dim: 1
-    hsdp_shard_dim: 8
-
-  data:
-    data_path: data/preprocessed
-    dataloader_num_workers: 4
-    train_batch_size: 4
-    training_cfg_rate: 0.0
-    seed: 1000
-    num_latent_t: 21
-    num_height: 480
-    num_width: 832
-    num_frames: 81
-
-  optimizer:
-    learning_rate: 5.0e-5
-    betas: [0.9, 0.999]
-    weight_decay: 0.0
-    lr_scheduler: constant
-    lr_warmup_steps: 0
-
-  loop:
-    max_train_steps: 6000
-    gradient_accumulation_steps: 1
-
-  checkpoint:
-    output_dir: outputs/wan2.1_anyflow_pretrain
-    training_state_checkpointing_steps: 500
-    checkpoints_total_limit: 3
-    resume_from_checkpoint: latest
-
-  tracker:
-    project_name: anyflow-wan
-    run_name: wan2.1_t2v_anyflow_pretrain
-
-  model:
-    enable_gradient_checkpointing_type: full
-
-callbacks:
-  grad_clip:
-    _target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback
-    max_grad_norm: 1.0
-
-pipeline:
-  flow_shift: 5.0
-  dit_config:
-    # Enable AnyFlow dual-timestep conditioning. The student loads from
-    # base Wan 2.1 — its checkpoint has no delta_embedder weights, so they
-    # get initialized identically to time_embedder via deep-copy in
-    # WanTimeTextImageEmbedding.__init__.
-    r_embedder: true
-    r_embedder_fusion: gated
-    r_embedder_gate_value: 0.25
-    r_embedder_deltatime_type: r

examples/train/configs/distribution_matching/wan/self_forcing_causal_t2v.yaml

@@ -100,6 +100,9 @@ callbacks:
     sampling_steps: [4]
     sampling_timesteps: [1000, 750, 500, 250]
     num_frames: 81
+    # Validation/inference uses standard CFG in both clean and Self-Forcing,
+    # so this directly matches Self-Forcing guidance_scale=3.0.
+    guidance_scale: 3.0
 
 pipeline:
   flow_shift: 5

examples/train/scenario/worldmodel/README.md

@@ -1,119 +1,32 @@
 # World-Model: Matrix-Game 2.0 I2V
 
-Training scenarios for the Matrix-Game 2.0 I2V world model on Solaris (Minecraft)
-data and Zelda data, using the new YAML-driven trainer
-(`fastvideo/train/entrypoint/train.py`).
-
-## Solaris Configs
+Three training scenarios for the Matrix-Game 2.0 I2V world model on the
+new YAML-driven trainer (`fastvideo/train/entrypoint/train.py`).
 
 | Config | Method | Student | Notes |
 |---|---|---|---|
-| `solaris/finetune_i2v.yaml` | `FineTuneMethod` | `MatrixGame2Model` (bidirectional) | Multi-step SFT from `mg_bidirectional_Solaris`. |
-| `solaris/dfsft_causal_i2v.yaml` | `DiffusionForcingSFTMethod` | `MatrixGame2CausalModel` | Diffusion-Forcing SFT with chunkwise timesteps. |
-| `solaris/self_forcing_causal_i2v.yaml` | `SelfForcingMethod` | `MatrixGame2CausalModel` | Matrix-Game 2.0 DMD/Self-Forcing distillation; teacher = bidirectional, critic = bidirectional. |
-
-## Zelda Configs
-
-| Config | Method | Student | Notes |
-|---|---|---|---|
-| `zelda/finetune_i2v.yaml` | `FineTuneMethod` | `MatrixGame2Model` (bidirectional) | Zelda bidirectional I2V finetuning from `FastVideo/Matrix-Game-2.0-Base-Diffusers`. Uses 33-frame clips and Zelda validation with action overlays. |
-| `zelda/dfsft_causal_i2v.yaml` | `DiffusionForcingSFTMethod` | `MatrixGame2CausalModel` | Zelda causal Diffusion-Forcing SFT from `mignonjia/mg_bidirectional_zelda`. Uses the same Zelda data, resolution, optimizer, and validation defaults as the Zelda finetune config. |
-| `zelda/self_forcing_causal_i2v.yaml` | `SelfForcingMethod` | `MatrixGame2CausalModel` | Zelda DMD/Self-Forcing distillation; student init = `mignonjia/mg_causal_zelda`, teacher = bidirectional (`mignonjia/mg_bidirectional_zelda`), critic = bidirectional. |
-| `zelda/streaming_long_tuning_causal_i2v.yaml` | `StreamingLongTuningMethod` | `MatrixGame2CausalModel` | LongLive-style streaming long tuning from the 1k-step Zelda self-forcing checkpoint. |
-
-Zelda world-model distillation is a two-run workflow: first run
-`zelda/self_forcing_causal_i2v.yaml` to train or load the 1k-step
-self-forcing checkpoint (`mignonjia/mg_sf_distilled_zelda_1k_steps`), then run
-`zelda/streaming_long_tuning_causal_i2v.yaml` for the 3k-step streaming
-long-tuning stage. The long-tuning YAML starts from that 1k-step checkpoint; it
-does not run the short self-forcing stage inside the same config.
-
-## Zelda Training Data
-
-The Zelda training configs use `data/zeldam2-clean` as a suggested local path.
-Download the dataset from Hugging Face before running those configs:
-
-```bash
-python scripts/huggingface/download_hf.py \
-    --repo_id mignonjia/zeldam2-clean \
-    --local_dir data/zeldam2-clean \
-    --repo_type dataset
-```
-
-You can store the dataset elsewhere; update `training.data.data_path` in the
-YAML to point at that location.
-
-## Multi3D Training Data
-
-`zelda/finetune_i2v.yaml` includes an optional, commented-out Multi3D entry.
-Enable it only when you want to mix Zelda with multi-game data from
-`data/multi3d_games`. You can store this dataset anywhere; before enabling it,
-update the matching commented `training.data.data_path` key in the YAML to the
-correct location.
-
-To mix datasets in a training YAML, set `training.data.data_path` to a
-path-to-repeat-count mapping. For example, `zelda/finetune_i2v.yaml` can use
-`data/zeldam2-clean: 1` and `# data/multi3d_games: 10`; uncommenting the
-Multi3D entry repeats the multi-game parquet list ten times before training
-samples are shuffled.
-
-## World Model Validation Data
-
-The Zelda validation configs expect a small public validation bundle under
-`data/zelda_validation_data`.
-
-Download it from Hugging Face before running the Zelda scenarios:
-
-```bash
-python scripts/huggingface/download_hf.py \
-    --repo_id mignonjia/zelda_validation_data \
-    --local_dir data/zelda_validation_data \
-    --repo_type dataset
-```
-
-The bundle contains `validation_zelda.json`, `images/`, and `actions/`.
-The Zelda configs point
-`callbacks.validation.dataset_file` at
-`data/zelda_validation_data/validation_zelda.json`.
+| `finetune_i2v.yaml` | `FineTuneMethod` | `MatrixGame2Model` (bidirectional) | Multi-step SFT from `mg_bidirectional_Solaris`. |
+| `dfsft_causal_i2v.yaml` | `DiffusionForcingSFTMethod` | `MatrixGame2CausalModel` | Diffusion-Forcing SFT with chunkwise timesteps. |
+| `self_forcing_causal_i2v.yaml` | `SelfForcingMethod` | `MatrixGame2CausalModel` | DMD/Self-Forcing distillation; teacher = bidirectional, critic = bidirectional. |
 
 ## Usage
 
-### Solaris
-
 ```bash
 bash examples/train/run.sh \
-    examples/train/scenario/worldmodel/solaris/finetune_i2v.yaml
+    examples/train/scenario/worldmodel/finetune_i2v.yaml
 
 bash examples/train/run.sh \
-    examples/train/scenario/worldmodel/solaris/dfsft_causal_i2v.yaml
+    examples/train/scenario/worldmodel/dfsft_causal_i2v.yaml
 
 bash examples/train/run.sh \
-    examples/train/scenario/worldmodel/solaris/self_forcing_causal_i2v.yaml
-```
-
-### Zelda
-
-```bash
-# Finetuning / DFSFT
-bash examples/train/run.sh \
-    examples/train/scenario/worldmodel/zelda/finetune_i2v.yaml
-
-bash examples/train/run.sh \
-    examples/train/scenario/worldmodel/zelda/dfsft_causal_i2v.yaml
-
-# Distillation / long tuning
-bash examples/train/run.sh \
-    examples/train/scenario/worldmodel/zelda/self_forcing_causal_i2v.yaml
-
-bash examples/train/run.sh \
-    examples/train/scenario/worldmodel/zelda/streaming_long_tuning_causal_i2v.yaml
+    examples/train/scenario/worldmodel/self_forcing_causal_i2v.yaml
 ```
 
 Override any field on the command line:
 
 ```bash
 bash examples/train/run.sh \
-    examples/train/scenario/worldmodel/solaris/dfsft_causal_i2v.yaml \
+    examples/train/scenario/worldmodel/dfsft_causal_i2v.yaml \
     --training.distributed.num_gpus 8 \
     --training.optimizer.learning_rate 1e-5
 ```

examples/train/scenario/worldmodel/self_forcing_causal_i2v.yaml

@@ -97,4 +97,4 @@ callbacks:
     guidance_scale: 6.0
 
 pipeline:
-  flow_shift: 5
+  flow_shift: 5

examples/train/scenario/worldmodel/zelda/dfsft_causal_i2v.yaml

@@ -1,94 +0,0 @@
-# Diffusion-Forcing SFT: Zelda world model I2V Causal
-
-models:
-  student:
-    _target_: fastvideo.train.models.matrixgame2.matrixgame2_causal.MatrixGame2CausalModel
-    init_from: mignonjia/mg_bidirectional_zelda
-    trainable: true
-
-method:
-  _target_: fastvideo.train.methods.fine_tuning.dfsft.DiffusionForcingSFTMethod
-  chunk_size: 3
-
-training:
-  distributed:
-    num_gpus: 8
-    sp_size: 1
-    tp_size: 1
-    hsdp_replicate_dim: 1
-    hsdp_shard_dim: 8
-
-  data:
-    data_path:
-      data/zeldam2-clean: 1
-    dataloader_num_workers: 1
-    train_batch_size: 1
-    training_cfg_rate: 0.0
-    seed: 42
-    num_latent_t: 9
-    num_height: 480
-    num_width: 832
-    num_frames: 33
-
-  optimizer:
-    learning_rate: 2.0e-5
-    betas: [0.9, 0.95]
-    weight_decay: 0.0
-    lr_scheduler: constant
-    lr_warmup_steps: 0
-
-  loop:
-    max_train_steps: 60000
-    gradient_accumulation_steps: 1
-
-  checkpoint:
-    output_dir: outputs/matrixgame_finetune/checkpoints/zelda_causal_dfsft
-    training_state_checkpointing_steps: 5000
-    checkpoints_total_limit: 3
-
-  tracker:
-    entity: hapo-exp
-    project_name: mg_1.3b_zelda
-    run_name: zelda_causal_dfsft
-
-  model:
-    enable_gradient_checkpointing_type: full
-
-  dit_precision: fp32
-
-callbacks:
-  grad_clip:
-    max_grad_norm: 1.0
-  validation:
-    pipeline_target: fastvideo.pipelines.basic.matrixgame2.matrixgame2_causal_dmd_pipeline.MatrixGame2CausalDMDPipeline
-    dataset_file: data/zelda_validation_data/validation_zelda.json
-    every_steps: 200
-    sampling_steps: [40]
-    sampling_timesteps: [1000, 975, 950, 925, 900, 875, 850, 825, 800, 775,
-                         750, 725, 700, 675, 650, 625, 600, 575, 550, 525,
-                         500, 475, 450, 425, 400, 375, 350, 325, 300, 275,
-                         250, 225, 200, 175, 150, 125, 100, 75, 50, 25]
-    num_frames: 33
-    overlay_actions: true
-    guidance_scale: 6.0
-    metrics:
-      enabled: true
-      names:
-        - vbench.imaging_quality
-        - vbench.aesthetic_quality
-        - vbench.temporal_flickering
-        - vbench.motion_smoothness
-        - vbench.subject_consistency
-        - vbench.background_consistency
-        - vbench.dynamic_degree
-        - optical_flow.synthetic_optical_flow
-      calibration_path: assets/eval/worldmodel_synthetic_flow_calibration.json
-      skip_missing_deps: true
-      strict: false
-      unload_after_validation: true
-
-pipeline:
-  flow_shift: 5
-  dit_config:
-    local_attn_size: 6
-    sink_size: 1

examples/train/scenario/worldmodel/zelda/finetune_i2v.yaml

@@ -1,88 +0,0 @@
-# Matrix-Game 2.0 Zelda + multi-game I2V finetune.
-
-models:
-  student:
-    _target_: fastvideo.train.models.matrixgame2.matrixgame2.MatrixGame2Model
-    init_from: FastVideo/Matrix-Game-2.0-Base-Diffusers
-    trainable: true
-
-method:
-  _target_: fastvideo.train.methods.fine_tuning.finetune.FineTuneMethod
-
-training:
-  distributed:
-    num_gpus: 8
-    sp_size: 1
-    tp_size: 1
-    hsdp_replicate_dim: 1
-    hsdp_shard_dim: 8
-
-  data:
-    data_path:
-      data/zeldam2-clean: 1
-      # data/multi3d_games: 10
-    dataloader_num_workers: 1
-    train_batch_size: 1
-    training_cfg_rate: 0.0  # unused for MatrixGame2 I2V; no text_embedding CFG dropout
-    seed: 42
-    num_latent_t: 9
-    num_height: 480
-    num_width: 832
-    num_frames: 33
-
-  optimizer:
-    learning_rate: 2.0e-5
-    betas: [0.9, 0.95]
-    weight_decay: 0.0
-    lr_scheduler: constant
-    lr_warmup_steps: 0
-
-  loop:
-    max_train_steps: 60000
-    gradient_accumulation_steps: 1
-
-  checkpoint:
-    output_dir: outputs/matrixgame_finetune/checkpoints/zelda_with_mg_init
-    training_state_checkpointing_steps: 5000
-    checkpoints_total_limit: 3
-
-  tracker:
-    project_name: mg_1.3b_zelda
-    run_name: zelda_with_mg_init
-
-  model:
-    enable_gradient_checkpointing_type: full
-
-  dit_precision: fp32
-
-callbacks:
-  grad_clip:
-    _target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback
-    max_grad_norm: 1.0
-  validation:
-    _target_: fastvideo.train.callbacks.validation.ValidationCallback
-    pipeline_target: fastvideo.pipelines.basic.matrixgame2.matrixgame2_i2v_pipeline.MatrixGame2I2VPipeline
-    dataset_file: data/zelda_validation_data/validation_zelda.json
-    every_steps: 200
-    sampling_steps: [40]
-    num_frames: 33
-    overlay_actions: true
-    guidance_scale: 6.0
-    metrics:
-      enabled: true
-      names:
-        - vbench.imaging_quality
-        - vbench.aesthetic_quality
-        - vbench.temporal_flickering
-        - vbench.motion_smoothness
-        - vbench.subject_consistency
-        - vbench.background_consistency
-        - vbench.dynamic_degree
-        - optical_flow.synthetic_optical_flow
-      calibration_path: assets/eval/worldmodel_synthetic_flow_calibration.json
-      skip_missing_deps: true
-      strict: false
-      unload_after_validation: true
-
-pipeline:
-  flow_shift: 5

examples/train/scenario/worldmodel/zelda/self_forcing_causal_i2v.yaml

@@ -1,120 +0,0 @@
-# Self-forcing distillation: Zelda world model I2V Causal
-
-models:
-  student:
-    _target_: fastvideo.train.models.matrixgame2.matrixgame2_causal.MatrixGame2CausalModel
-    init_from: mignonjia/mg_causal_zelda
-    trainable: true
-  teacher:
-    _target_: fastvideo.train.models.matrixgame2.matrixgame2.MatrixGame2Model
-    init_from: mignonjia/mg_bidirectional_zelda
-    trainable: false
-    disable_custom_init_weights: true
-  critic:
-    _target_: fastvideo.train.models.matrixgame2.matrixgame2.MatrixGame2Model
-    init_from: mignonjia/mg_bidirectional_zelda
-    trainable: true
-    disable_custom_init_weights: true
-
-method:
-  _target_: fastvideo.train.methods.distribution_matching.self_forcing.SelfForcingMethod
-  rollout_mode: simulate
-  generator_update_interval: 5
-  dmd_denoising_steps: [1000, 750, 500, 250]
-  warp_denoising_step: true
-  min_timestep_ratio: 0.02
-  max_timestep_ratio: 0.98
-
-  chunk_size: 3
-  student_sample_type: sde
-  same_step_across_blocks: true
-  last_step_only: false
-  context_noise: 0.0
-  enable_gradient_in_rollout: true
-  start_gradient_frame: 0
-
-  # Critic optimizer
-  fake_score_learning_rate: 3.0e-7
-  fake_score_betas: [0.9, 0.95]
-  fake_score_lr_scheduler: constant
-
-training:
-  distributed:
-    num_gpus: 8
-    sp_size: 1
-    tp_size: 1
-    hsdp_replicate_dim: 1
-    hsdp_shard_dim: 8
-
-  data:
-    data_path: data/zeldam2-clean
-    dataloader_num_workers: 4
-    train_batch_size: 1
-    training_cfg_rate: 0.0
-    seed: 1001
-    num_latent_t: 9
-    num_height: 480
-    num_width: 832
-    num_frames: 33
-
-  optimizer:
-    learning_rate: 3.0e-6
-    betas: [0.9, 0.95]
-    weight_decay: 0.0
-    lr_scheduler: constant
-    lr_warmup_steps: 0
-
-  loop:
-    max_train_steps: 1000
-    gradient_accumulation_steps: 1
-
-  checkpoint:
-    output_dir: outputs/zelda_causal_self_forcing
-    training_state_checkpointing_steps: 1000
-    checkpoints_total_limit: 1
-
-  tracker:
-    project_name: wangame_sf
-    run_name: mg2_self_forcing_9_latents
-
-  model:
-    enable_gradient_checkpointing_type: full
-
-  dit_precision: fp32
-
-callbacks:
-  ema:
-    decay: 0.99
-    start_iter: 200
-  grad_clip:
-    max_grad_norm: 1.0
-  validation:
-    pipeline_target: fastvideo.pipelines.basic.matrixgame2.matrixgame2_causal_dmd_pipeline.MatrixGame2CausalDMDPipeline
-    dataset_file: data/zelda_validation_data/validation_zelda.json
-    every_steps: 100
-    sampling_steps: [4]
-    sampling_timesteps: [1000, 750, 500, 250]
-    num_frames: 153
-    overlay_actions: true
-    keyboard_value_scale: 1.0
-    metrics:
-      enabled: true
-      names:
-        - vbench.imaging_quality
-        - vbench.aesthetic_quality
-        - vbench.temporal_flickering
-        - vbench.motion_smoothness
-        - vbench.subject_consistency
-        - vbench.background_consistency
-        - vbench.dynamic_degree
-        - optical_flow.synthetic_optical_flow
-      calibration_path: assets/eval/worldmodel_synthetic_flow_calibration.json
-      skip_missing_deps: true
-      strict: false
-      unload_after_validation: true
-
-pipeline:
-  flow_shift: 5
-  dit_config:
-    local_attn_size: 6
-    sink_size: 1

examples/train/scenario/worldmodel/zelda/streaming_long_tuning_causal_i2v.yaml

@@ -1,140 +0,0 @@
-# MatrixGame2 I2V LongLive-style streaming distillation: Zelda world model I2V Causal
-# Student init from self forcing after 1k steps
-
-models:
-  student:
-    _target_: fastvideo.train.models.matrixgame2.matrixgame2_causal.MatrixGame2CausalModel
-    init_from: mignonjia/mg_sf_distilled_zelda_1k_steps 
-    trainable: true
-    # transformer_override_safetensor: outputs/matrixgame_dmd/checkpoints/mg_zelda_sf_m2/checkpoint-1000_weight_only/ema/generator_ema.safetensors
-  teacher:
-    _target_: fastvideo.train.models.matrixgame2.matrixgame2.MatrixGame2Model
-    init_from: mignonjia/mg_bidirectional_zelda
-    trainable: false
-    disable_custom_init_weights: true
-  critic:
-    _target_: fastvideo.train.models.matrixgame2.matrixgame2.MatrixGame2Model
-    init_from: mignonjia/mg_bidirectional_zelda
-    trainable: true
-    disable_custom_init_weights: true
-
-method:
-  _target_: fastvideo.train.methods.distribution_matching.streaming_long_tuning.StreamingLongTuningMethod
-  rollout_mode: simulate
-  generator_update_interval: 5
-  dmd_denoising_steps: [1000, 750, 500, 250]
-  warp_denoising_step: true
-  min_timestep_ratio: 0.02
-  max_timestep_ratio: 0.98
-
-  chunk_size: 3
-  student_sample_type: sde
-  same_step_across_blocks: true
-  last_step_only: false
-  context_noise: 0.0
-  enable_gradient_in_rollout: true
-  start_gradient_frame: 0
-
-  streaming_training: true
-  streaming_chunk_size: 9
-  streaming_max_length: 39
-  streaming_fixed_overlap_latents: 3
-  streaming_reencode_overlap_anchor: true
-  streaming_anchor_inject_k: 1
-  streaming_require_full_blocks: true
-
-  multi_phased_distill_schedule:
-    - stage: streaming_long
-      start_step: 0
-      end_step: 3000
-      num_latent_t: 39
-      streaming_training: true
-      streaming_chunk_size: 9
-      streaming_max_length: 39
-      streaming_fixed_overlap_latents: 3
-
-  # Critic optimizer
-  fake_score_learning_rate: 3.0e-7
-  fake_score_betas: [0.9, 0.95]
-  fake_score_lr_scheduler: constant
-
-training:
-  distributed:
-    num_gpus: 8
-    sp_size: 1
-    tp_size: 1
-    hsdp_replicate_dim: 1
-    hsdp_shard_dim: 8
-
-  data:
-    data_path: data/zeldam2-clean
-    dataloader_num_workers: 4
-    train_batch_size: 1
-    training_cfg_rate: 0.0
-    seed: 1001
-    num_latent_t: 39
-    num_height: 480
-    num_width: 832
-    num_frames: 153
-
-  optimizer:
-    learning_rate: 3.0e-6
-    betas: [0.9, 0.95]
-    weight_decay: 0.0
-    lr_scheduler: constant
-    lr_warmup_steps: 0
-
-  loop:
-    max_train_steps: 3000
-    gradient_accumulation_steps: 1
-
-  checkpoint:
-    output_dir: outputs/zelda_causal_long_tuning
-    training_state_checkpointing_steps: 1000
-    checkpoints_total_limit: 2
-
-  tracker:
-    project_name: wangame_sf
-    run_name: mg2_39only_streaming_long
-
-  model:
-    enable_gradient_checkpointing_type: full
-
-  dit_precision: fp32
-
-callbacks:
-  ema:
-    decay: 0.99
-    start_iter: 200
-  grad_clip:
-    max_grad_norm: 1.0
-  validation:
-    pipeline_target: fastvideo.pipelines.basic.matrixgame2.matrixgame2_causal_dmd_pipeline.MatrixGame2CausalDMDPipeline
-    dataset_file: data/zelda_validation_data/validation_zelda.json
-    every_steps: 100
-    sampling_steps: [4]
-    sampling_timesteps: [1000, 750, 500, 250]
-    num_frames: 153
-    overlay_actions: true
-    keyboard_value_scale: 1.0
-    metrics:
-      enabled: true
-      names:
-        - vbench.imaging_quality
-        - vbench.aesthetic_quality
-        - vbench.temporal_flickering
-        - vbench.motion_smoothness
-        - vbench.subject_consistency
-        - vbench.background_consistency
-        - vbench.dynamic_degree
-        - optical_flow.synthetic_optical_flow
-      calibration_path: assets/eval/worldmodel_synthetic_flow_calibration.json
-      skip_missing_deps: true
-      strict: false
-      unload_after_validation: true
-
-pipeline:
-  flow_shift: 5
-  dit_config:
-    local_attn_size: 6
-    sink_size: 1

fastvideo/api/flux.py

@@ -1,27 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-
-from __future__ import annotations
-
-from dataclasses import dataclass
-
-from fastvideo.api.sampling_param import SamplingParam
-
-
-@dataclass
-class FluxSamplingParam(SamplingParam):
-
-    prompt: str | None = "a photo of a cat"
-    negative_prompt: str = ""
-
-    num_videos_per_prompt: int = 1
-    seed: int = 0
-
-    num_frames: int = 1
-    height: int = 1024
-    width: int = 1024
-    fps: int = 1
-
-    num_inference_steps: int = 28
-    guidance_scale: float = 3.5
-    use_embedded_guidance: bool = True
-    true_cfg_scale: float = 1.0

fastvideo/api/sampling_param.py

@@ -90,10 +90,6 @@ class SamplingParam:
     num_inference_steps_sr: int = 50
     guidance_scale: float = 1.0
     guidance_scale_2: float | None = None
-    # Embedded guidance (FLUX): do not treat ``guidance_scale > 1`` as classic CFG.
-    use_embedded_guidance: bool = False
-    # Diffusers-style true CFG for FLUX when > 1 (requires negative prompt encoding).
-    true_cfg_scale: float = 1.0
     guidance_rescale: float = 0.0
     boundary_ratio: float | None = None
     sigmas: list[float] | None = None
@@ -329,18 +325,6 @@ class SamplingParam:
             default=SamplingParam.guidance_rescale,
             help="Guidance rescale factor",
         )
-        parser.add_argument(
-            "--use-embedded-guidance",
-            action="store_true",
-            default=SamplingParam.use_embedded_guidance,
-            help="Use embedded guidance scale (FLUX-style) instead of classic CFG",
-        )
-        parser.add_argument(
-            "--true-cfg-scale",
-            type=float,
-            default=SamplingParam.true_cfg_scale,
-            help="True CFG scale for FLUX when > 1 (requires negative prompt encoding)",
-        )
         parser.add_argument(
             "--boundary-ratio",
             type=float,

fastvideo/api/schema.py

@@ -150,7 +150,6 @@ class SamplingConfig:
     guidance_scale_2: float | None = None
     guidance_rescale: float = 0.0
     true_cfg_scale: float | None = None
-    use_embedded_guidance: bool | None = None
     boundary_ratio: float | None = None
     sigmas: list[float] | None = None
 

fastvideo/attention/backends/flash_attn.py

@@ -1,15 +1,13 @@
 # SPDX-License-Identifier: Apache-2.0
 
+import importlib.util
 import os
+
 import torch
 import torch.nn.functional as F
 from dataclasses import dataclass
 
-from fastvideo.attention.utils.flash_attn_default import (
-    fa_version,
-    flash_attn_func_compilable,
-)
-
+from fastvideo import envs
 from fastvideo.attention.backends.abstract import (
     AttentionBackend,
     AttentionImpl,
@@ -19,6 +17,119 @@ from fastvideo.attention.backends.abstract import (
 from fastvideo.logger import init_logger
 
 logger = init_logger(__name__)
+
+# FA4 (flash_attn.cute) is explicit opt-in via FASTVIDEO_FA4=1, mirroring the
+# kernel package's FASTVIDEO_VSA_CUTEDSL: its CuTeDSL kernels JIT-compile per
+# shape family and can fail at runtime on some arch/shape combinations, so it
+# is never auto-selected just because it is installed. Below sm90 a capability
+# gate in flash_attn_cute routes to FA2 the calls FA4 cannot serve there:
+# grad-enabled (its backward asserts sm90+) and GQA (pack_gqa fails CuTeDSL
+# JIT, observed on sm_89).
+if envs.FASTVIDEO_FA4:
+    try:
+        from fastvideo.attention.utils.flash_attn_cute import flash_attn_func
+    except ImportError as e:
+        raise RuntimeError(f"FASTVIDEO_FA4=1 but flash_attn.cute (FA4) is not usable ({e}); "
+                           "fix the FA4 install (see the flash-attn-4 pin in pyproject.toml) "
+                           "or unset FASTVIDEO_FA4.") from e
+    fa_version = "4"
+else:
+    try:
+        from flash_attn_interface import flash_attn_func as flash_attn_3_func
+
+        # flash_attn 3 no longer have a different API, see following commit:
+        # https://github.com/Dao-AILab/flash-attention/commit/ed209409acedbb2379f870bbd03abce31a7a51b7
+        flash_attn_func = flash_attn_3_func
+        fa_version = "3"
+    except ImportError:
+        from flash_attn import flash_attn_func as flash_attn_2_func
+        flash_attn_func = flash_attn_2_func
+        fa_version = "2"
+    try:
+        if importlib.util.find_spec("flash_attn.cute") is not None:
+            logger.info("flash_attn.cute (FA4) is installed but not enabled; "
+                        "set FASTVIDEO_FA4=1 to use it for inference.")
+    except ImportError:
+        pass
+
+# torch.compile traceability: the FA4/cute path (fa_version=="4") is
+# already a registered torch.library custom op, so dynamo treats it as a
+# graph node. The external FA2/FA3 `flash_attn_func` is NOT — dynamo
+# breaks the graph at the call site (observed: wanvideo.py self-attn,
+# once per layer every step), which fragments the compiled region and
+# blocks CUDA-graph capture. Wrap the FA2/FA3 default call in a custom
+# op (mirrors the FP4 `flash_attn_cute` template) so it becomes an
+# opaque-but-traceable node. The kernel still runs eager inside the op
+# (correct — flash-attn must run eager); only dynamo's treatment of the
+# boundary changes, so numerics are unchanged (SSIM-gate to confirm).
+if fa_version in ("2", "3"):
+    _fa_default = flash_attn_func
+
+    # Scope: this op covers exactly the q/k/v + softmax_scale + causal
+    # call shape used by FlashAttentionImpl.forward's default branch
+    # (see `flash_attn_func_compilable(...)` call site below). The
+    # masked/no-pad and varlen / cross-attn paths use different
+    # entry points (`flash_attn_no_pad`, `flash_attn_varlen_*`) which
+    # are intentionally out of scope for this PR — wrapping them is a
+    # natural follow-up. The wrapper's signature is the contract: any
+    # extra kwarg (dropout_p, window_size, alibi_slopes, deterministic,
+    # return_attn_probs, ...) raises TypeError at the call site, so
+    # silent loss of kwargs is not a failure mode.
+    @torch.library.custom_op(
+        "fastvideo::_flash_attn_default_forward",
+        mutates_args=(),
+        device_types="cuda",
+    )
+    def _flash_attn_default_forward(
+        q: torch.Tensor,
+        k: torch.Tensor,
+        v: torch.Tensor,
+        softmax_scale: float | None,
+        causal: bool,
+    ) -> torch.Tensor:
+        return _fa_default(q, k, v, softmax_scale=softmax_scale, causal=causal)
+
+    @torch.library.register_fake("fastvideo::_flash_attn_default_forward")
+    def _flash_attn_default_forward_fake(
+        q: torch.Tensor,
+        k: torch.Tensor,
+        v: torch.Tensor,
+        softmax_scale: float | None,
+        causal: bool,
+    ) -> torch.Tensor:
+        del softmax_scale, causal
+        # FA2/FA3 default path: [batch, seqlen_q, nheads, head_dim_v],
+        # same dtype/device as q (head dim taken from v).
+        return q.new_empty(q.shape[0], q.shape[1], q.shape[2], v.shape[-1])
+
+    def flash_attn_func_compilable(q, k, v, softmax_scale=None, causal=False):
+        # Autograd carve-out. The custom op above registers a forward + fake
+        # kernel but NO backward (register_autograd), so it is opaque to
+        # autograd. Inference runs under no_grad / inference_mode and routes
+        # through the traceable custom op — that is the torch.compile win, and
+        # the only path this PR claims. Training backprops through attention,
+        # so route grad-enabled calls to the original FA2/FA3 `flash_attn_func`
+        # (itself an autograd.Function, so backward is correct) at the cost of a
+        # dynamo graph break on the training path — i.e. pre-PR behavior, no
+        # regression. Full autograd parity for the custom op (mirroring the FP4
+        # cute template) is a tracked follow-up.
+        if torch.is_grad_enabled() and (q.requires_grad or k.requires_grad or v.requires_grad):
+            return _fa_default(q, k, v, softmax_scale=softmax_scale, causal=causal)
+        return torch.ops.fastvideo._flash_attn_default_forward(q, k, v, softmax_scale, causal)
+elif fa_version == "4":
+    # FA4 path: `flash_attn_func` (from `flash_attn_cute`) goes through a
+    # registered torch.library custom op (with an FA4 backward on sm90+;
+    # grad-enabled and GQA calls below sm90 route to FA2), so a passthrough
+    # is enough — no extra registration needed.
+    def flash_attn_func_compilable(q, k, v, softmax_scale=None, causal=False):
+        return flash_attn_func(q, k, v, softmax_scale=softmax_scale, causal=causal)
+else:
+    # Defensive: the probe above only ever sets fa_version to "2", "3",
+    # or "4"; an unexpected value means an import/probe regression and
+    # we want a loud error at import, not a silent NameError later.
+    raise RuntimeError(f"Unsupported FlashAttention version: {fa_version!r} — expected "
+                       f"'2', '3', or '4' from the import probe above.")
+
 logger.info("Using FlashAttention-%s backend", fa_version)
 
 # FP4 FA4 support: quantize Q/K to NVFP4 E2M1 for block-scaled MMA on Blackwell.
@@ -198,17 +309,9 @@ class FlashAttentionImpl(AttentionImpl):
         attn_metadata: FlashAttnMetadata,
     ):
         if (attn_metadata is not None and hasattr(attn_metadata, "attn_mask") and attn_metadata.attn_mask is not None):
-            # Route through the *_compilable wrappers so dynamo sees one
-            # traceable node for each masked entry point (the unpad/pad
-            # bookkeeping runs eager inside the custom op). On FA2 these
-            # wrappers go through ops with full register_autograd, so
-            # training also backprops through the op (no graph break on
-            # the training path); on FA3/FA4 they carve out to the
-            # autograd.Function for grad-enabled calls — see
-            # fastvideo/attention/utils/flash_attn_no_pad.py.
             from fastvideo.attention.utils.flash_attn_no_pad import (
-                flash_attn_no_pad_compilable as flash_attn_no_pad,
-                flash_attn_varlen_qk_no_pad_compilable as flash_attn_varlen_qk_no_pad,
+                flash_attn_no_pad,
+                flash_attn_varlen_qk_no_pad,
             )
 
             attn_mask = attn_metadata.attn_mask

fastvideo/attention/backends/sdpa.py

@@ -15,14 +15,8 @@ class SDPABackend(AttentionBackend):
     accept_output_buffer: bool = True
 
     @staticmethod
-    def get_supported_head_sizes() -> list[int] | None:
-        # torch.nn.functional.scaled_dot_product_attention is head-size
-        # agnostic: the math backend handles any size and the fused kernels
-        # fall back internally when they cannot. The previous list
-        # ([32, 64, ..., 256]) was copied from FlashAttentionBackend in the
-        # v1 refactor (#270) and wrongly excluded sizes such as 80 (e.g.
-        # CLIP vision encoders). None means "no restriction".
-        return None
+    def get_supported_head_sizes() -> list[int]:
+        return [32, 64, 96, 128, 160, 192, 224, 256]
 
     @staticmethod
     def get_name() -> str:

fastvideo/attention/utils/flash_attn_default.py

@@ -1,251 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-"""torch.compile-traceable wrapper for the FA2/FA3/FA4 default attention path.
-
-The FA4/cute path (`fa_version == "4"`) is already a registered
-`torch.library.custom_op` in `fastvideo.attention.utils.flash_attn_cute`, so
-dynamo treats it as a graph node. The external FA2/FA3 ``flash_attn_func`` is
-NOT — dynamo breaks the graph at the call site (observed: wanvideo.py
-self-attn, once per layer every step), which fragments the compiled region
-and blocks CUDA-graph capture. Wrap the FA2/FA3 default call in a custom op
-(mirrors the FP4 `flash_attn_cute` template) so it becomes an
-opaque-but-traceable node. The kernel still runs eager inside the op
-(correct — flash-attn must run eager); only dynamo's treatment of the
-boundary changes, so numerics are unchanged (SSIM-gated).
-
-Autograd: FA2 has full ``register_autograd`` parity — the custom op's
-backward calls flash_attn's ``_flash_attn_backward`` directly, so training
-backprops *through* the op (no graph break on the training path either).
-FA3 currently keeps the no-backward + carve-out pattern from PR #1373
-because FA3's private backward signature wants validation on a real Hopper
-box (gated on Kuan-Hao's Modal FA3 setup PR). Once that lands the FA3 path
-can mirror FA2.
-
-Lives in `attention/utils/` (sibling of `flash_attn_cute.py` and
-`flash_attn_no_pad.py`) so it can be imported by any backend that wants the
-traceable FA default call without pulling in backend dispatch logic. The
-backend (`attention/backends/flash_attn.py`) just imports
-`flash_attn_func_compilable` and `fa_version` from here.
-"""
-
-import importlib.util
-
-import torch
-
-from fastvideo import envs
-from fastvideo.logger import init_logger
-
-logger = init_logger(__name__)
-
-# Pick the same backend the rest of FastVideo picked for `flash_attn_func`
-# (FA4/cute → FA3 → FA2). Mirror the precedence used in
-# `attention/utils/flash_attn_no_pad.py` so the two probes always agree.
-#
-# FA4 (flash_attn.cute) is explicit opt-in via FASTVIDEO_FA4=1: its CuTeDSL
-# kernels JIT-compile per shape family and can fail at runtime on some
-# arch/shape combinations, so it is never auto-selected just because it is
-# installed.
-if envs.FASTVIDEO_FA4:
-    try:
-        from fastvideo.attention.utils.flash_attn_cute import flash_attn_func
-    except ImportError as e:
-        raise RuntimeError(f"FASTVIDEO_FA4=1 but flash_attn.cute (FA4) is not usable ({e}); "
-                           "fix the FA4 install (see the flash-attn-4 pin in pyproject.toml) "
-                           "or unset FASTVIDEO_FA4.") from e
-    fa_version = "4"
-else:
-    try:
-        from flash_attn_interface import flash_attn_func as flash_attn_3_func
-
-        # flash_attn 3 no longer has a different API, see following commit:
-        # https://github.com/Dao-AILab/flash-attention/commit/ed209409acedbb2379f870bbd03abce31a7a51b7
-        flash_attn_func = flash_attn_3_func
-        fa_version = "3"
-    except ImportError:
-        from flash_attn import flash_attn_func as flash_attn_2_func
-        flash_attn_func = flash_attn_2_func
-        fa_version = "2"
-    try:
-        if importlib.util.find_spec("flash_attn.cute") is not None:
-            logger.info("flash_attn.cute (FA4) is installed but not enabled; "
-                        "set FASTVIDEO_FA4=1 to use it for inference.")
-    except ImportError:
-        pass
-
-if fa_version == "2":
-    # Scope: this op covers exactly the q/k/v + softmax_scale + causal call
-    # shape used by FlashAttentionImpl.forward's default branch (see
-    # `flash_attn_func_compilable(...)` call site in
-    # `attention/backends/flash_attn.py`). The masked/no-pad and varlen /
-    # cross-attn paths use different entry points
-    # (`flash_attn_no_pad`, `flash_attn_varlen_*`) which live in
-    # `attention/utils/flash_attn_no_pad.py`. The wrapper's signature is the
-    # contract: any extra kwarg (dropout_p, window_size, alibi_slopes,
-    # deterministic, return_attn_probs, ...) raises TypeError at the call
-    # site, so silent loss of kwargs is not a failure mode.
-    from flash_attn.flash_attn_interface import _flash_attn_backward as _fa2_backward
-    _fa_default = flash_attn_func
-
-    @torch.library.custom_op(
-        "fastvideo::_flash_attn_default_forward",
-        mutates_args=(),
-        device_types="cuda",
-    )
-    def _flash_attn_default_forward(
-        q: torch.Tensor,
-        k: torch.Tensor,
-        v: torch.Tensor,
-        softmax_scale: float | None,
-        causal: bool,
-    ) -> tuple[torch.Tensor, torch.Tensor]:
-        # `return_attn_probs=True` asks FA2 to also return softmax_lse +
-        # S_dmask. We need softmax_lse to feed the backward; S_dmask is the
-        # dropout mask (always None here since dropout_p is fixed at 0).
-        out, softmax_lse, _ = _fa_default(q, k, v, softmax_scale=softmax_scale, causal=causal, return_attn_probs=True)
-        return out, softmax_lse
-
-    @torch.library.register_fake("fastvideo::_flash_attn_default_forward")
-    def _flash_attn_default_forward_fake(
-        q: torch.Tensor,
-        k: torch.Tensor,
-        v: torch.Tensor,
-        softmax_scale: float | None,
-        causal: bool,
-    ) -> tuple[torch.Tensor, torch.Tensor]:
-        del softmax_scale, causal
-        # FA2 default path: out = [batch, seqlen_q, nheads, head_dim_v],
-        # softmax_lse = [batch, nheads, seqlen_q], fp32 regardless of q dtype.
-        b, sq, hq = q.shape[0], q.shape[1], q.shape[2]
-        out = q.new_empty(b, sq, hq, v.shape[-1])
-        lse = q.new_empty(b, hq, sq, dtype=torch.float32)
-        return out, lse
-
-    def _flash_attn_default_setup_context(ctx, inputs, output):
-        q, k, v, softmax_scale, causal = inputs
-        out, lse = output
-        ctx.save_for_backward(q, k, v, out, lse)
-        # `lse` is an auxiliary output we save to feed FA2's backward; nobody
-        # should differentiate through it. Mark it non-differentiable so
-        # autograd errors loudly if a caller wires it into a loss, rather
-        # than silently producing zero/None grads through the `del grad_lse`
-        # in our backward.
-        ctx.mark_non_differentiable(lse)
-        # FA2's *forward* substitutes `1 / sqrt(head_dim)` for `softmax_scale=None`
-        # internally; FA2's *backward* (`_flash_attn_backward`) demands a concrete
-        # float in its C++ schema and rejects None at the binding boundary. Resolve
-        # the default here so the value saved on ctx (and passed to backward) is
-        # always a real float — matches what FA2's own autograd.Function does.
-        if softmax_scale is None:
-            softmax_scale = q.shape[-1]**-0.5
-        ctx.softmax_scale = softmax_scale
-        ctx.causal = causal
-
-    def _flash_attn_default_backward(ctx, grad_out, grad_lse):
-        # We only differentiate `out`; softmax_lse is saved-for-backward, not
-        # a real differentiable output. (Mirrors the FP4 cute template.)
-        del grad_lse
-        q, k, v, out, lse = ctx.saved_tensors
-        dq = torch.empty_like(q)
-        dk = torch.empty_like(k)
-        dv = torch.empty_like(v)
-        # FA2's `_flash_attn_backward` writes into dq/dk/dv in place. The
-        # extra kwargs (window_size_*, softcap, alibi_slopes, deterministic,
-        # rng_state) are pinned to the same defaults the forward wrapper
-        # uses — flash-attn==2.8.1 (the version FastVideo pins) requires
-        # all of them explicitly. `rng_state=None` is correct for our
-        # `dropout_p=0` configuration.
-        _fa2_backward(
-            grad_out,
-            q,
-            k,
-            v,
-            out,
-            lse,
-            dq,
-            dk,
-            dv,
-            dropout_p=0.0,
-            softmax_scale=ctx.softmax_scale,
-            causal=ctx.causal,
-            window_size_left=-1,
-            window_size_right=-1,
-            softcap=0.0,
-            alibi_slopes=None,
-            deterministic=False,
-            rng_state=None,
-        )
-        return dq, dk, dv, None, None
-
-    torch.library.register_autograd(
-        "fastvideo::_flash_attn_default_forward",
-        _flash_attn_default_backward,
-        setup_context=_flash_attn_default_setup_context,
-    )
-
-    def flash_attn_func_compilable(q, k, v, softmax_scale=None, causal=False):
-        # Backward is registered: autograd flows through the op (training
-        # path is also traceable; no carve-out needed). Public API matches
-        # `flash_attn_func` — returns just `out`; we drop the saved-for-
-        # backward `lse` here so callers see the original single-tensor
-        # contract.
-        out, _ = torch.ops.fastvideo._flash_attn_default_forward(q, k, v, softmax_scale, causal)
-        return out
-elif fa_version == "3":
-    # FA3 path: same forward+fake custom op as the original PR #1373, with
-    # the autograd carve-out kept. The full backward (mirroring the FA2 leg
-    # above) wants a Hopper box for grad-check validation, which we don't
-    # have until Kuan-Hao's Modal FA3 setup PR lands. Until then this keeps
-    # inference traceable + training correct (via the original
-    # autograd.Function path + a pre-PR-style graph break on training).
-    _fa_default = flash_attn_func
-
-    @torch.library.custom_op(
-        "fastvideo::_flash_attn_default_forward",
-        mutates_args=(),
-        device_types="cuda",
-    )
-    def _flash_attn_default_forward(
-        q: torch.Tensor,
-        k: torch.Tensor,
-        v: torch.Tensor,
-        softmax_scale: float | None,
-        causal: bool,
-    ) -> torch.Tensor:
-        return _fa_default(q, k, v, softmax_scale=softmax_scale, causal=causal)
-
-    @torch.library.register_fake("fastvideo::_flash_attn_default_forward")
-    def _flash_attn_default_forward_fake(
-        q: torch.Tensor,
-        k: torch.Tensor,
-        v: torch.Tensor,
-        softmax_scale: float | None,
-        causal: bool,
-    ) -> torch.Tensor:
-        del softmax_scale, causal
-        return q.new_empty(q.shape[0], q.shape[1], q.shape[2], v.shape[-1])
-
-    def flash_attn_func_compilable(q, k, v, softmax_scale=None, causal=False):
-        # Autograd carve-out. The custom op above registers a forward + fake
-        # kernel but NO backward (register_autograd), so it is opaque to
-        # autograd. Inference runs under no_grad / inference_mode and routes
-        # through the traceable custom op — that is the torch.compile win, and
-        # the only path this PR claims. Training backprops through attention,
-        # so route grad-enabled calls to the original FA2/FA3 `flash_attn_func`
-        # (itself an autograd.Function, so backward is correct) at the cost of a
-        # dynamo graph break on the training path — i.e. pre-PR behavior, no
-        # regression. Full autograd parity for the custom op (mirroring the FP4
-        # cute template) is a tracked follow-up.
-        if torch.is_grad_enabled() and (q.requires_grad or k.requires_grad or v.requires_grad):
-            return _fa_default(q, k, v, softmax_scale=softmax_scale, causal=causal)
-        return torch.ops.fastvideo._flash_attn_default_forward(q, k, v, softmax_scale, causal)
-elif fa_version == "4":
-    # FA4 path: `flash_attn_func` is already a torch.library custom op
-    # (registered in `fastvideo.attention.utils.flash_attn_cute`), so a
-    # passthrough is enough — no extra registration needed.
-    def flash_attn_func_compilable(q, k, v, softmax_scale=None, causal=False):
-        return flash_attn_func(q, k, v, softmax_scale=softmax_scale, causal=causal)
-else:
-    # Defensive: the probe above only ever sets fa_version to "2", "3",
-    # or "4"; an unexpected value means an import/probe regression and
-    # we want a loud error at import, not a silent NameError later.
-    raise RuntimeError(f"Unsupported FlashAttention version: {fa_version!r} — expected "
-                       f"'2', '3', or '4' from the import probe above.")

fastvideo/attention/utils/flash_attn_no_pad.py

@@ -24,7 +24,7 @@ from flash_attn.bert_padding import pad_input, unpad_input
 from fastvideo import envs
 
 
-def _resolve_flash_attn_varlen_func() -> tuple[Any, str]:
+def _resolve_flash_attn_varlen_func() -> Any:
     if envs.FASTVIDEO_FA4:
         # FA4 cute is explicit opt-in (see fastvideo/attention/backends/
         # flash_attn.py); with FASTVIDEO_FA4=1 an unimportable FA4 build must
@@ -39,28 +39,20 @@ def _resolve_flash_attn_varlen_func() -> tuple[Any, str]:
                                "fix the FA4 install (see the flash-attn-4 pin in pyproject.toml) "
                                "or unset FASTVIDEO_FA4.") from e
 
-        return flash_attn_varlen_func_cute, "4"
+        return flash_attn_varlen_func_cute
     try:
         from flash_attn_interface import (
             flash_attn_varlen_func as flash_attn_varlen_func_interface, )
 
-        return flash_attn_varlen_func_interface, "3"
+        return flash_attn_varlen_func_interface
     except ImportError:
         from flash_attn import (
             flash_attn_varlen_func as flash_attn_varlen_func_flash, )
 
-        return flash_attn_varlen_func_flash, "2"
+        return flash_attn_varlen_func_flash
 
 
-flash_attn_varlen_func_impl, _FA_VARLEN_VERSION = _resolve_flash_attn_varlen_func()
-
-# FA2-only: the private varlen backward we register against the custom ops
-# below. FA3 / FA4 have different private signatures and validation paths
-# (Hopper / Blackwell boxes) — those legs keep the autograd carve-out
-# pattern from PR #1373 until their setup PRs land.
-if _FA_VARLEN_VERSION == "2":
-    from flash_attn.flash_attn_interface import (
-        _flash_attn_varlen_backward as _fa2_varlen_backward, )
+flash_attn_varlen_func_impl = _resolve_flash_attn_varlen_func()
 
 
 def flash_attn_no_pad(
@@ -199,473 +191,3 @@ def flash_attn_varlen_qk_no_pad(
         h=nheads,
     )
     return output
-
-
-# ---------------------------------------------------------------------------
-# torch.compile traceability + register_autograd parity for the masked /
-# varlen attention paths.
-#
-# Wraps the two entry points `FlashAttentionImpl.forward` calls
-# (`flash_attn_no_pad`, `flash_attn_varlen_qk_no_pad`) as
-# `torch.library.custom_op`s so dynamo sees one traceable node — the
-# internal unpad / pad bookkeeping (data-dependent `nnz` shapes) runs
-# eager inside the op, and the op's outputs are the statically-shaped
-# padded tensors. This mirrors the FA2 default-path wrapper in
-# `fastvideo/attention/backends/flash_attn.py`.
-#
-# Autograd: on FA2 we register a real backward (`register_autograd`)
-# that calls FA2's `_flash_attn_varlen_backward` on the unpadded form
-# — re-unpadding the saved padded tensors using the saved mask. The
-# `softmax_lse` from the varlen forward is naturally unpadded
-# (`[nheads, total_q]`); we pad it to `[batch, nheads, seqlen]` on
-# the way out (statically shaped) and re-unpad in backward. So
-# training backprops *through* the op (no graph break on the training
-# path either).
-#
-# FA3 / FA4 keep the autograd carve-out pattern from PR #1373: the
-# custom op has forward + fake only, and `*_compilable` falls back to
-# the original autograd.Function for grad-enabled calls. Those legs
-# are gated on Hopper-class / Blackwell-class boxes for backward
-# validation and ship as separate follow-ups.
-
-if _FA_VARLEN_VERSION == "2":
-    # ---------- masked self-attention: flash_attn_no_pad (FA2) ----------
-
-    @torch.library.custom_op(
-        "fastvideo::_flash_attn_no_pad_forward",
-        mutates_args=(),
-        device_types="cuda",
-    )
-    def _flash_attn_no_pad_forward(
-        qkv: torch.Tensor,
-        key_padding_mask: torch.Tensor,
-        causal: bool,
-        dropout_p: float,
-        softmax_scale: float | None,
-        deterministic: bool,
-    ) -> tuple[torch.Tensor, torch.Tensor]:
-        b, s, _three, h, d = qkv.shape
-        x = rearrange(qkv, "b s three h d -> b s (three h d)")
-        x_unpad, indices, cu_seqlens, max_s, _ = unpad_input(x, key_padding_mask)
-        x_unpad = rearrange(x_unpad, "nnz (three h d) -> nnz three h d", three=3, h=h)
-        out_unpad, lse_unpad, _ = flash_attn_varlen_qkvpacked_func(x_unpad,
-                                                                   cu_seqlens,
-                                                                   max_s,
-                                                                   dropout_p,
-                                                                   softmax_scale=softmax_scale,
-                                                                   causal=causal,
-                                                                   deterministic=deterministic,
-                                                                   return_attn_probs=True)
-        # Pad out: [nnz, h, d] -> [b, s, h, d]
-        out_padded = rearrange(pad_input(rearrange(out_unpad, "nnz h d -> nnz (h d)"), indices, b, s),
-                               "b s (h d) -> b s h d",
-                               h=h)
-        # Pad lse: FA2 varlen returns [nheads, total_q]. Transpose to [total_q,
-        # nheads], pad to [b, s, nheads], permute to [b, nheads, s] — statically
-        # shaped so register_fake matches.
-        lse_padded = pad_input(lse_unpad.t().contiguous(), indices, b, s).permute(0, 2, 1).contiguous()
-        return out_padded, lse_padded
-
-    @torch.library.register_fake("fastvideo::_flash_attn_no_pad_forward")
-    def _flash_attn_no_pad_forward_fake(
-        qkv: torch.Tensor,
-        key_padding_mask: torch.Tensor,
-        causal: bool,
-        dropout_p: float,
-        softmax_scale: float | None,
-        deterministic: bool,
-    ) -> tuple[torch.Tensor, torch.Tensor]:
-        del key_padding_mask, causal, dropout_p, softmax_scale, deterministic
-        b, s, _three, h, d = qkv.shape
-        out = qkv.new_empty(b, s, h, d)
-        lse = qkv.new_empty(b, h, s, dtype=torch.float32)
-        return out, lse
-
-    def _flash_attn_no_pad_setup_context(ctx, inputs, output):
-        qkv, key_padding_mask, causal, dropout_p, softmax_scale, deterministic = inputs
-        out, lse = output
-        ctx.save_for_backward(qkv, out, lse, key_padding_mask)
-        # Auxiliary output, not differentiable — see default-path note.
-        ctx.mark_non_differentiable(lse)
-        # FA2's varlen backward requires a concrete float for softmax_scale.
-        if softmax_scale is None:
-            softmax_scale = qkv.shape[-1]**-0.5  # head_dim from qkv's last dim
-        ctx.softmax_scale = softmax_scale
-        ctx.causal = causal
-        ctx.dropout_p = dropout_p
-        ctx.deterministic = deterministic
-
-    def _flash_attn_no_pad_backward(ctx, grad_out, grad_lse):
-        # lse is saved-for-backward, not differentiated.
-        del grad_lse
-        qkv, out_padded, lse_padded, key_padding_mask = ctx.saved_tensors
-        b, s, _three, h, d = qkv.shape
-
-        # One `unpad_input` call (on qkv) gives us indices + cu_seqlens + max_s;
-        # reuse those for out / dout / lse below via direct indexing instead
-        # of redundant `unpad_input` calls (each of which would re-run
-        # `nonzero` + `cumsum` + a `.max().item()` GPU→CPU sync).
-        x = rearrange(qkv, "b s three h d -> b s (three h d)")
-        x_unpad, indices, cu_seqlens, max_s, _ = unpad_input(x, key_padding_mask)
-        x_unpad = rearrange(x_unpad, "nnz (three h d) -> nnz three h d", three=3, h=h)
-        q_unpad, k_unpad, v_unpad = (t.contiguous() for t in x_unpad.unbind(dim=1))
-
-        # Direct-index variants reuse `indices` (computed above).
-        out_unpad = out_padded.flatten(0, 1)[indices].view(-1, h, d).contiguous()
-        dout_unpad = grad_out.flatten(0, 1)[indices].view(-1, h, d).contiguous()
-        # lse_padded [b, h, s] -> [b, s, h] -> [nnz, h] -> [h, nnz].
-        lse_unpad = lse_padded.permute(0, 2, 1).contiguous().flatten(0, 1)[indices].t().contiguous()
-
-        dq_unpad = torch.empty_like(q_unpad)
-        dk_unpad = torch.empty_like(k_unpad)
-        dv_unpad = torch.empty_like(v_unpad)
-        _fa2_varlen_backward(
-            dout_unpad,
-            q_unpad,
-            k_unpad,
-            v_unpad,
-            out_unpad,
-            lse_unpad,
-            dq_unpad,
-            dk_unpad,
-            dv_unpad,
-            cu_seqlens_q=cu_seqlens,
-            cu_seqlens_k=cu_seqlens,
-            max_seqlen_q=max_s,
-            max_seqlen_k=max_s,
-            dropout_p=ctx.dropout_p,
-            softmax_scale=ctx.softmax_scale,
-            causal=ctx.causal,
-            window_size_left=-1,
-            window_size_right=-1,
-            softcap=0.0,
-            alibi_slopes=None,
-            deterministic=ctx.deterministic,
-            rng_state=None,
-        )
-
-        # Re-pad each grad and stack into dqkv.
-        def _repad(dt_unpad: torch.Tensor) -> torch.Tensor:
-            padded = pad_input(rearrange(dt_unpad, "nnz h d -> nnz (h d)"), indices, b, s)
-            return rearrange(padded, "b s (h d) -> b s h d", h=h)
-
-        dqkv = torch.stack([_repad(dq_unpad), _repad(dk_unpad), _repad(dv_unpad)], dim=2)
-        # 6 inputs total: qkv, key_padding_mask, causal, dropout_p, softmax_scale, deterministic.
-        return dqkv, None, None, None, None, None
-
-    torch.library.register_autograd(
-        "fastvideo::_flash_attn_no_pad_forward",
-        _flash_attn_no_pad_backward,
-        setup_context=_flash_attn_no_pad_setup_context,
-    )
-
-    # ---------- cross-attention: flash_attn_varlen_qk_no_pad (FA2) ----------
-
-    @torch.library.custom_op(
-        "fastvideo::_flash_attn_varlen_qk_no_pad_forward",
-        mutates_args=(),
-        device_types="cuda",
-    )
-    def _flash_attn_varlen_qk_no_pad_forward(
-        query: torch.Tensor,
-        key: torch.Tensor,
-        value: torch.Tensor,
-        query_padding_mask: torch.Tensor,
-        key_padding_mask: torch.Tensor,
-        causal: bool,
-        dropout_p: float,
-        softmax_scale: float | None,
-        deterministic: bool,
-    ) -> tuple[torch.Tensor, torch.Tensor]:
-        b, sq, h, d = query.shape
-        q_unpad, q_indices, cu_seqlens_q, max_seqlen_q, _ = unpad_input(rearrange(query, "b s h d -> b s (h d)"),
-                                                                        query_padding_mask)
-        k_unpad, _, cu_seqlens_k, max_seqlen_k, _ = unpad_input(rearrange(key, "b s h d -> b s (h d)"),
-                                                                key_padding_mask)
-        v_unpad, _, _, _, _ = unpad_input(rearrange(value, "b s h d -> b s (h d)"), key_padding_mask)
-        q_unpad = rearrange(q_unpad, "nnz (h d) -> nnz h d", h=h)
-        k_unpad = rearrange(k_unpad, "nnz (h d) -> nnz h d", h=h)
-        v_unpad = rearrange(v_unpad, "nnz (h d) -> nnz h d", h=h)
-        out_unpad, lse_unpad, _ = flash_attn_varlen_func_impl(q_unpad,
-                                                              k_unpad,
-                                                              v_unpad,
-                                                              cu_seqlens_q,
-                                                              cu_seqlens_k,
-                                                              max_seqlen_q,
-                                                              max_seqlen_k,
-                                                              dropout_p=dropout_p,
-                                                              softmax_scale=softmax_scale,
-                                                              causal=causal,
-                                                              deterministic=deterministic,
-                                                              return_attn_probs=True)
-        # Pad out: [nnz_q, h, d] -> [b, sq, h, d]
-        out_padded = rearrange(pad_input(rearrange(out_unpad, "nnz h d -> nnz (h d)"), q_indices, b, sq),
-                               "b s (h d) -> b s h d",
-                               h=h)
-        # Pad lse: [h, nnz_q] -> [b, h, sq]
-        lse_padded = pad_input(lse_unpad.t().contiguous(), q_indices, b, sq).permute(0, 2, 1).contiguous()
-        return out_padded, lse_padded
-
-    @torch.library.register_fake("fastvideo::_flash_attn_varlen_qk_no_pad_forward")
-    def _flash_attn_varlen_qk_no_pad_forward_fake(
-        query: torch.Tensor,
-        key: torch.Tensor,
-        value: torch.Tensor,
-        query_padding_mask: torch.Tensor,
-        key_padding_mask: torch.Tensor,
-        causal: bool,
-        dropout_p: float,
-        softmax_scale: float | None,
-        deterministic: bool,
-    ) -> tuple[torch.Tensor, torch.Tensor]:
-        del key, query_padding_mask, key_padding_mask
-        del causal, dropout_p, softmax_scale, deterministic
-        b, sq, h, _ = query.shape
-        # `out`'s head_dim comes from value (d_v), matching the real forward's
-        # out_padded ([b, sq, h, d_v]); it can differ from query's d_q.
-        out = query.new_empty(b, sq, h, value.shape[-1])
-        lse = query.new_empty(b, h, sq, dtype=torch.float32)
-        return out, lse
-
-    def _flash_attn_varlen_qk_no_pad_setup_context(ctx, inputs, output):
-        (query, key, value, query_padding_mask, key_padding_mask, causal, dropout_p, softmax_scale,
-         deterministic) = inputs
-        out, lse = output
-        ctx.save_for_backward(query, key, value, out, lse, query_padding_mask, key_padding_mask)
-        # Auxiliary output, not differentiable — see default-path note.
-        ctx.mark_non_differentiable(lse)
-        if softmax_scale is None:
-            softmax_scale = query.shape[-1]**-0.5
-        ctx.softmax_scale = softmax_scale
-        ctx.causal = causal
-        ctx.dropout_p = dropout_p
-        ctx.deterministic = deterministic
-
-    def _flash_attn_varlen_qk_no_pad_backward(ctx, grad_out, grad_lse):
-        del grad_lse
-        (query, key, value, out_padded, lse_padded, query_padding_mask, key_padding_mask) = ctx.saved_tensors
-        b, sq, h, d = query.shape
-        sk = key.shape[1]
-
-        # One `unpad_input` call per distinct mask; reuse the returned
-        # indices via direct indexing for everything else that shares
-        # the same mask (v with k_mask; out/dout/lse with q_mask; the
-        # final repad of dk/dv also reuses k_indices). Avoids ~4
-        # redundant `unpad_input` calls + their GPU→CPU `.max().item()`
-        # syncs.
-        q_unpad, q_indices, cu_seqlens_q, max_seqlen_q, _ = unpad_input(rearrange(query, "b s h d -> b s (h d)"),
-                                                                        query_padding_mask)
-        k_unpad, k_indices, cu_seqlens_k, max_seqlen_k, _ = unpad_input(rearrange(key, "b s h d -> b s (h d)"),
-                                                                        key_padding_mask)
-        q_unpad = rearrange(q_unpad, "nnz (h d) -> nnz h d", h=h).contiguous()
-        k_unpad = rearrange(k_unpad, "nnz (h d) -> nnz h d", h=h).contiguous()
-        v_unpad = value.flatten(0, 1)[k_indices].view(-1, h, d).contiguous()
-
-        # out / dout / lse follow q's shape, so index with q_indices.
-        out_unpad = out_padded.flatten(0, 1)[q_indices].view(-1, h, d).contiguous()
-        dout_unpad = grad_out.flatten(0, 1)[q_indices].view(-1, h, d).contiguous()
-        lse_unpad = lse_padded.permute(0, 2, 1).contiguous().flatten(0, 1)[q_indices].t().contiguous()
-
-        dq_unpad = torch.empty_like(q_unpad)
-        dk_unpad = torch.empty_like(k_unpad)
-        dv_unpad = torch.empty_like(v_unpad)
-        _fa2_varlen_backward(
-            dout_unpad,
-            q_unpad,
-            k_unpad,
-            v_unpad,
-            out_unpad,
-            lse_unpad,
-            dq_unpad,
-            dk_unpad,
-            dv_unpad,
-            cu_seqlens_q=cu_seqlens_q,
-            cu_seqlens_k=cu_seqlens_k,
-            max_seqlen_q=max_seqlen_q,
-            max_seqlen_k=max_seqlen_k,
-            dropout_p=ctx.dropout_p,
-            softmax_scale=ctx.softmax_scale,
-            causal=ctx.causal,
-            window_size_left=-1,
-            window_size_right=-1,
-            softcap=0.0,
-            alibi_slopes=None,
-            deterministic=ctx.deterministic,
-            rng_state=None,
-        )
-
-        # k_indices is already available from the unpad_input above —
-        # no need to recompute it for the dk/dv repad.
-        def _repad(dt_unpad: torch.Tensor, indices: torch.Tensor, batch: int, seqlen: int) -> torch.Tensor:
-            padded = pad_input(rearrange(dt_unpad, "nnz h d -> nnz (h d)"), indices, batch, seqlen)
-            return rearrange(padded, "b s (h d) -> b s h d", h=h)
-
-        dq_padded = _repad(dq_unpad, q_indices, b, sq)
-        dk_padded = _repad(dk_unpad, k_indices, b, sk)
-        dv_padded = _repad(dv_unpad, k_indices, b, sk)
-        # 9 inputs total: query, key, value, q_mask, k_mask, causal, dropout_p,
-        # softmax_scale, deterministic.
-        return dq_padded, dk_padded, dv_padded, None, None, None, None, None, None
-
-    torch.library.register_autograd(
-        "fastvideo::_flash_attn_varlen_qk_no_pad_forward",
-        _flash_attn_varlen_qk_no_pad_backward,
-        setup_context=_flash_attn_varlen_qk_no_pad_setup_context,
-    )
-
-    # ---------- public dispatchers (FA2: autograd flows through the op) -----
-
-
-    def flash_attn_no_pad_compilable(qkv,
-                                     key_padding_mask,
-                                     causal=False,
-                                     dropout_p=0.0,
-                                     softmax_scale=None,
-                                     deterministic=False):
-        """dynamo-traceable wrapper around ``flash_attn_no_pad`` (registered op,
-        full register_autograd on FA2 — both inference and training go through
-        the op, no graph break on either)."""
-        out, _ = torch.ops.fastvideo._flash_attn_no_pad_forward(qkv, key_padding_mask, causal, dropout_p, softmax_scale,
-                                                                deterministic)
-        return out
-
-    def flash_attn_varlen_qk_no_pad_compilable(query,
-                                               key,
-                                               value,
-                                               query_padding_mask,
-                                               key_padding_mask,
-                                               causal=False,
-                                               dropout_p=0.0,
-                                               softmax_scale=None,
-                                               deterministic=False):
-        """dynamo-traceable wrapper around ``flash_attn_varlen_qk_no_pad`` (registered
-        op, full register_autograd on FA2)."""
-        out, _ = torch.ops.fastvideo._flash_attn_varlen_qk_no_pad_forward(query, key, value, query_padding_mask,
-                                                                          key_padding_mask, causal, dropout_p,
-                                                                          softmax_scale, deterministic)
-        return out
-
-else:
-    # ---------- FA3 / FA4: carve-out (forward+fake only, no real backward) ---
-    # Same pattern as the parked varlen-extension and the FA3 default leg in
-    # `fastvideo/attention/backends/flash_attn.py`. Real backward for these
-    # versions is a follow-up gated on Hopper / Blackwell box validation.
-
-    @torch.library.custom_op(
-        "fastvideo::_flash_attn_no_pad_forward",
-        mutates_args=(),
-        device_types="cuda",
-    )
-    def _flash_attn_no_pad_forward(
-        qkv: torch.Tensor,
-        key_padding_mask: torch.Tensor,
-        causal: bool,
-        dropout_p: float,
-        softmax_scale: float | None,
-        deterministic: bool,
-    ) -> torch.Tensor:
-        return flash_attn_no_pad(  # type: ignore[no-untyped-call]
-            qkv,
-            key_padding_mask,
-            causal=causal,
-            dropout_p=dropout_p,
-            softmax_scale=softmax_scale,
-            deterministic=deterministic)
-
-    @torch.library.register_fake("fastvideo::_flash_attn_no_pad_forward")
-    def _flash_attn_no_pad_forward_fake(
-        qkv: torch.Tensor,
-        key_padding_mask: torch.Tensor,
-        causal: bool,
-        dropout_p: float,
-        softmax_scale: float | None,
-        deterministic: bool,
-    ) -> torch.Tensor:
-        del key_padding_mask, causal, dropout_p, softmax_scale, deterministic
-        b, s, _three, h, d = qkv.shape
-        return qkv.new_empty(b, s, h, d)
-
-    @torch.library.custom_op(
-        "fastvideo::_flash_attn_varlen_qk_no_pad_forward",
-        mutates_args=(),
-        device_types="cuda",
-    )
-    def _flash_attn_varlen_qk_no_pad_forward(
-        query: torch.Tensor,
-        key: torch.Tensor,
-        value: torch.Tensor,
-        query_padding_mask: torch.Tensor,
-        key_padding_mask: torch.Tensor,
-        causal: bool,
-        dropout_p: float,
-        softmax_scale: float | None,
-        deterministic: bool,
-    ) -> torch.Tensor:
-        return flash_attn_varlen_qk_no_pad(  # type: ignore[no-untyped-call]
-            query,
-            key,
-            value,
-            query_padding_mask=query_padding_mask,
-            key_padding_mask=key_padding_mask,
-            causal=causal,
-            dropout_p=dropout_p,
-            softmax_scale=softmax_scale,
-            deterministic=deterministic)
-
-    @torch.library.register_fake("fastvideo::_flash_attn_varlen_qk_no_pad_forward")
-    def _flash_attn_varlen_qk_no_pad_forward_fake(
-        query: torch.Tensor,
-        key: torch.Tensor,
-        value: torch.Tensor,
-        query_padding_mask: torch.Tensor,
-        key_padding_mask: torch.Tensor,
-        causal: bool,
-        dropout_p: float,
-        softmax_scale: float | None,
-        deterministic: bool,
-    ) -> torch.Tensor:
-        del key, query_padding_mask, key_padding_mask
-        del causal, dropout_p, softmax_scale, deterministic
-        b, sq, h, _ = query.shape
-        # `out`'s head_dim comes from value (d_v), matching the real forward's
-        # output ([b, sq, h, d_v]); it can differ from query's d_q.
-        return query.new_empty(b, sq, h, value.shape[-1])
-
-    def flash_attn_no_pad_compilable(qkv,
-                                     key_padding_mask,
-                                     causal=False,
-                                     dropout_p=0.0,
-                                     softmax_scale=None,
-                                     deterministic=False):
-        if torch.is_grad_enabled() and qkv.requires_grad:
-            return flash_attn_no_pad(qkv,
-                                     key_padding_mask,
-                                     causal=causal,
-                                     dropout_p=dropout_p,
-                                     softmax_scale=softmax_scale,
-                                     deterministic=deterministic)
-        return torch.ops.fastvideo._flash_attn_no_pad_forward(qkv, key_padding_mask, causal, dropout_p, softmax_scale,
-                                                              deterministic)
-
-    def flash_attn_varlen_qk_no_pad_compilable(query,
-                                               key,
-                                               value,
-                                               query_padding_mask,
-                                               key_padding_mask,
-                                               causal=False,
-                                               dropout_p=0.0,
-                                               softmax_scale=None,
-                                               deterministic=False):
-        if torch.is_grad_enabled() and (query.requires_grad or key.requires_grad or value.requires_grad):
-            return flash_attn_varlen_qk_no_pad(query,
-                                               key,
-                                               value,
-                                               query_padding_mask=query_padding_mask,
-                                               key_padding_mask=key_padding_mask,
-                                               causal=causal,
-                                               dropout_p=dropout_p,
-                                               softmax_scale=softmax_scale,
-                                               deterministic=deterministic)
-        return torch.ops.fastvideo._flash_attn_varlen_qk_no_pad_forward(query, key, value, query_padding_mask,
-                                                                        key_padding_mask, causal, dropout_p,
-                                                                        softmax_scale, deterministic)

fastvideo/configs/models/dits/__init__.py

@@ -1,9 +1,7 @@
 from fastvideo.configs.models.dits.cosmos import CosmosVideoConfig
 from fastvideo.configs.models.dits.cosmos2_5 import Cosmos25VideoConfig
 from fastvideo.configs.models.dits.dreamx_world import DreamXWorldARConfig, DreamXWorldConfig
-from fastvideo.configs.models.dits.flux import FluxDiTConfig
 from fastvideo.configs.models.dits.flux_2 import Flux2Config
-from fastvideo.configs.models.dits.glm_image import GlmImageDiTConfig
 from fastvideo.configs.models.dits.hunyuangamecraft import HunyuanGameCraftConfig
 from fastvideo.configs.models.dits.hunyuanvideo import HunyuanVideoConfig
 from fastvideo.configs.models.dits.hunyuanvideo15 import HunyuanVideo15Config
@@ -17,7 +15,6 @@ from fastvideo.configs.models.dits.kandinsky5 import Kandinsky5VideoConfig
 
 __all__ = [
     "HunyuanVideoConfig", "HunyuanVideo15Config", "HunyuanGameCraftConfig", "WanVideoConfig", "DreamXWorldConfig",
-    "DreamXWorldARConfig", "CosmosVideoConfig", "Cosmos25VideoConfig", "FluxDiTConfig", "Flux2Config",
-    "LongCatVideoConfig", "LTX2VideoConfig", "HYWorldConfig", "Kandinsky5VideoConfig", "MagiHumanVideoConfig",
-    "StableAudioConfig", "GlmImageDiTConfig"
+    "DreamXWorldARConfig", "CosmosVideoConfig", "Cosmos25VideoConfig", "LongCatVideoConfig", "LTX2VideoConfig",
+    "HYWorldConfig", "Kandinsky5VideoConfig", "MagiHumanVideoConfig", "StableAudioConfig", "Flux2Config"
 ]

fastvideo/configs/models/dits/flux.py

@@ -1,27 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-
-from dataclasses import dataclass, field
-
-from fastvideo.configs.models.dits.base import DiTArchConfig, DiTConfig
-
-
-@dataclass
-class FluxTransformer2DArchConfig(DiTArchConfig):
-
-    patch_size: int = 1
-    in_channels: int = 64
-    out_channels: int | None = None
-    num_layers: int = 19
-    num_single_layers: int = 38
-    attention_head_dim: int = 128
-    num_attention_heads: int = 24
-    joint_attention_dim: int = 4096
-    pooled_projection_dim: int = 768
-    guidance_embeds: bool = True
-    axes_dims_rope: tuple[int, int, int] = (16, 56, 56)
-
-
-@dataclass
-class FluxDiTConfig(DiTConfig):
-    arch_config: DiTArchConfig = field(default_factory=FluxTransformer2DArchConfig)
-    prefix: str = "flux"

fastvideo/configs/models/dits/glm_image.py

@@ -1,61 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-from dataclasses import dataclass, field
-
-from fastvideo.configs.models.dits.base import DiTArchConfig, DiTConfig
-
-
-def is_blocks(n: str, m) -> bool:
-    return "transformer_blocks" in n and str.isdigit(n.split(".")[-1])
-
-
-@dataclass
-class GlmImageDiTArchConfig(DiTArchConfig):
-
-    _fsdp_shard_conditions: list = field(default_factory=lambda: [is_blocks])
-
-    hidden_size: int = 4096
-    num_attention_heads: int = 32
-    attention_head_dim: int = 128
-    in_channels: int = 16
-    out_channels: int = 16
-    num_layers: int = 30
-
-    text_embed_dim: int = 1472
-    time_embed_dim: int = 512
-    condition_dim: int = 256
-
-    prior_vq_quantizer_codebook_size: int = 16384
-
-    patch_size: int = 2
-
-    max_height: int = 2048
-    max_width: int = 2048
-
-    qk_norm: str = "layer_norm"
-    eps: float = 1e-5
-
-    exclude_lora_layers: list[str] = field(
-        default_factory=lambda: ["image_projector", "glyph_projector", "prior_token_embedding"])
-
-    param_names_mapping: dict = field(
-        default_factory=lambda: {
-            r"^glyph_projector\.net\.0\.proj\.(.*)$": r"glyph_projector.fc_in.\1",
-            r"^glyph_projector\.net\.2\.(.*)$": r"glyph_projector.fc_out.\1",
-            r"^prior_projector\.net\.0\.proj\.(.*)$": r"prior_projector.fc_in.\1",
-            r"^prior_projector\.net\.2\.(.*)$": r"prior_projector.fc_out.\1",
-            r"^transformer_blocks\.(\d+)\.ff\.net\.0\.proj\.(.*)$": r"transformer_blocks.\1.ff.fc_in.\2",
-            r"^transformer_blocks\.(\d+)\.ff\.net\.2\.(.*)$": r"transformer_blocks.\1.ff.fc_out.\2",
-        })
-
-    reverse_param_names_mapping: dict = field(default_factory=dict)
-    lora_param_names_mapping: dict = field(default_factory=dict)
-
-    def __post_init__(self):
-        super().__post_init__()
-        self.num_channels_latents = self.out_channels
-
-
-@dataclass
-class GlmImageDiTConfig(DiTConfig):
-    arch_config: DiTArchConfig = field(default_factory=GlmImageDiTArchConfig)
-    prefix: str = "GlmImage"

fastvideo/configs/models/dits/wanvideo.py

@@ -1,6 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
 from dataclasses import dataclass, field
-from typing import Literal
 
 from fastvideo.configs.models.dits.base import DiTArchConfig, DiTConfig
 
@@ -20,11 +19,6 @@ class WanVideoArchConfig(DiTArchConfig):
             r"^condition_embedder\.text_embedder\.linear_2\.(.*)$": r"condition_embedder.text_embedder.fc_out.\1",
             r"^condition_embedder\.time_embedder\.linear_1\.(.*)$": r"condition_embedder.time_embedder.mlp.fc_in.\1",
             r"^condition_embedder\.time_embedder\.linear_2\.(.*)$": r"condition_embedder.time_embedder.mlp.fc_out.\1",
-            # AnyFlow dual-timestep checkpoints expose delta_embedder weights with the
-            # same internal layout as time_embedder. The regex is harmless on plain
-            # Wan checkpoints (no delta_embedder keys to match).
-            r"^condition_embedder\.delta_embedder\.linear_1\.(.*)$": r"condition_embedder.delta_embedder.mlp.fc_in.\1",
-            r"^condition_embedder\.delta_embedder\.linear_2\.(.*)$": r"condition_embedder.delta_embedder.mlp.fc_out.\1",
             r"^condition_embedder\.time_proj\.(.*)$": r"condition_embedder.time_modulation.linear.\1",
             r"^condition_embedder\.image_embedder\.ff\.net\.0\.proj\.(.*)$":
             r"condition_embedder.image_embedder.ff.fc_in.\1",
@@ -92,14 +86,6 @@ class WanVideoArchConfig(DiTArchConfig):
     # "relativistic" keeps long rollouts in-distribution; a no-op unless sink_size > 0 and local_attn_size > 0.
     rope_cache_policy: str = "absolute"
 
-    # AnyFlow dual-timestep conditioning. Defaults preserve bit-identity with
-    # the legacy single-timestep forward (no delta_embedder allocated, no
-    # extra computation on the embedder forward path).
-    r_embedder: bool = False
-    r_embedder_fusion: Literal["additive", "gated"] = "additive"
-    r_embedder_gate_value: float = 0.25
-    r_embedder_deltatime_type: Literal["r", "t-r"] = "r"
-
     def __post_init__(self):
         super().__post_init__()
         self.out_channels = self.out_channels or self.in_channels

fastvideo/configs/models/vaes/__init__.py

@@ -2,7 +2,6 @@ from fastvideo.configs.models.vaes.cosmosvae import CosmosVAEConfig
 from fastvideo.configs.models.vaes.cosmos2_5vae import Cosmos25VAEConfig
 from fastvideo.configs.models.vaes.gamecraftvae import GameCraftVAEConfig
 from fastvideo.configs.models.vaes.gen3cvae import Gen3CVAEConfig
-from fastvideo.configs.models.vaes.glm_image import GlmImageVAEConfig
 from fastvideo.configs.models.vaes.hunyuanvae import HunyuanVAEConfig
 from fastvideo.configs.models.vaes.hunyuan15vae import Hunyuan15VAEConfig
 from fastvideo.configs.models.vaes.ltx2vae import LTX2VAEConfig
@@ -22,5 +21,4 @@ __all__ = [
     "OobleckVAEArchConfig",
     "OobleckVAEConfig",
     "Flux2VAEConfig",
-    "GlmImageVAEConfig",
 ]

fastvideo/configs/models/vaes/glm_image.py

@@ -1,95 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-from __future__ import annotations
-
-from dataclasses import dataclass, field
-
-from fastvideo.configs.models.vaes.autoencoder_kl import (AutoencoderKLArchConfig, AutoencoderKLVAEConfig)
-
-_GLM_IMAGE_LATENTS_MEAN: tuple[float, ...] = (
-    -0.2080078125,
-    1.875,
-    -0.470703125,
-    -1.265625,
-    -1.421875,
-    0.77734375,
-    -0.3671875,
-    -0.9453125,
-    0.318359375,
-    0.7734375,
-    -0.1884765625,
-    -0.022216796875,
-    -0.220703125,
-    -1.59375,
-    -0.81640625,
-    -0.255859375,
-)
-_GLM_IMAGE_LATENTS_STD: tuple[float, ...] = (
-    3.0625,
-    2.203125,
-    2.265625,
-    4.84375,
-    2.5,
-    3.9375,
-    2.203125,
-    3.03125,
-    2.1875,
-    2.046875,
-    2.71875,
-    2.390625,
-    2.390625,
-    2.453125,
-    2.25,
-    2.15625,
-)
-
-
-@dataclass
-class GlmImageVAEArchConfig(AutoencoderKLArchConfig):
-    act_fn: str = "silu"
-    block_out_channels: tuple[int, ...] = (128, 512, 1024, 1024)
-    down_block_types: tuple[str, ...] = (
-        "DownEncoderBlock2D",
-        "DownEncoderBlock2D",
-        "DownEncoderBlock2D",
-        "DownEncoderBlock2D",
-    )
-    up_block_types: tuple[str, ...] = (
-        "UpDecoderBlock2D",
-        "UpDecoderBlock2D",
-        "UpDecoderBlock2D",
-        "UpDecoderBlock2D",
-    )
-    force_upcast: bool = True
-    in_channels: int = 3
-    out_channels: int = 3
-    latent_channels: int = 16
-    latents_mean: tuple[float, ...] = _GLM_IMAGE_LATENTS_MEAN
-    latents_std: tuple[float, ...] = _GLM_IMAGE_LATENTS_STD
-    layers_per_block: int = 3
-    mid_block_add_attention: bool = False
-    norm_num_groups: int = 32
-    sample_size: int = 1024
-    scaling_factor: float = 0.18215
-    shift_factor: float | None = None
-    use_quant_conv: bool = False
-    use_post_quant_conv: bool = False
-
-    temporal_compression_ratio: int = 1
-    spatial_compression_ratio: int = 8
-
-
-@dataclass
-class GlmImageVAEConfig(AutoencoderKLVAEConfig):
-    arch_config: GlmImageVAEArchConfig = field(default_factory=GlmImageVAEArchConfig)
-
-    use_tiling: bool = True
-    use_temporal_tiling: bool = False
-    use_parallel_tiling: bool = False
-
-    tile_sample_min_height: int = 512
-    tile_sample_min_width: int = 512
-    tile_sample_stride_height: int = 384
-    tile_sample_stride_width: int = 384
-
-    load_encoder: bool = True
-    load_decoder: bool = True

fastvideo/configs/pipelines/flux.py

@@ -1,74 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-
-from __future__ import annotations
-
-from collections.abc import Callable
-from dataclasses import dataclass, field
-
-import torch
-
-from fastvideo.configs.models import EncoderConfig
-from fastvideo.configs.models.dits.flux import FluxDiTConfig
-from fastvideo.configs.models.encoders import (
-    BaseEncoderOutput,
-    CLIPTextConfig,
-    T5LargeConfig,
-)
-from fastvideo.configs.models.vaes.autoencoder_kl import AutoencoderKLVAEConfig
-from fastvideo.configs.pipelines.base import PipelineConfig, preprocess_text
-
-
-def _flux_clip_pooled_postprocess(outputs: BaseEncoderOutput) -> torch.Tensor:
-    """CLIP branch for FLUX: Diffusers uses pooled prompt embeddings only."""
-    if outputs.pooler_output is None:
-        raise RuntimeError(
-            "FLUX CLIP conditioning requires pooler_output. Ensure the CLIP text encoder returns pooled features.")
-    return outputs.pooler_output
-
-
-def _flux_t5_sequence_postprocess(outputs: BaseEncoderOutput) -> torch.Tensor:
-    if outputs.last_hidden_state is None:
-        raise RuntimeError("FLUX T5 conditioning requires last_hidden_state.")
-    return outputs.last_hidden_state
-
-
-@dataclass
-class FluxPipelineConfig(PipelineConfig):
-    """Pipeline layout for Diffusers FLUX.1-dev (CLIP + T5 + packed DiT + FlowMatch)."""
-
-    scheduler_arch: str = "FlowMatchEulerDiscreteScheduler"
-    transformer_arch: str = "FluxTransformer2DModel"
-    vae_arch: str = "AutoencoderKL"
-    text_encoder_archs: tuple[str, ...] = ("CLIPTextModel", "T5EncoderModel")
-    tokenizer_archs: tuple[str, ...] = ("CLIPTokenizer", "T5TokenizerFast")
-
-    dit_config: FluxDiTConfig = field(default_factory=FluxDiTConfig)
-    vae_config: AutoencoderKLVAEConfig = field(default_factory=AutoencoderKLVAEConfig)
-
-    embedded_cfg_scale: float = 3.5
-    flow_shift: float | None = None
-
-    text_encoder_configs: tuple[EncoderConfig, ...] = field(default_factory=lambda: (CLIPTextConfig(), T5LargeConfig()))
-    preprocess_text_funcs: tuple[Callable[[str], str],
-                                 ...] = field(default_factory=lambda: (preprocess_text, preprocess_text))
-    postprocess_text_funcs: tuple[Callable[[BaseEncoderOutput], torch.Tensor], ...] = field(
-        default_factory=lambda: (_flux_clip_pooled_postprocess, _flux_t5_sequence_postprocess))
-
-    dit_precision: str = "bf16"
-    vae_precision: str = "fp32"
-    text_encoder_precisions: tuple[str, ...] = field(default_factory=lambda: ("fp32", "bf16"))
-
-    def __post_init__(self) -> None:
-        te_cfgs = list(self.text_encoder_configs)
-        if len(te_cfgs) >= 1:
-            te_cfgs[0].tokenizer_kwargs.setdefault("padding", "max_length")
-            te_cfgs[0].tokenizer_kwargs.setdefault("max_length", 77)
-            te_cfgs[0].tokenizer_kwargs.setdefault("truncation", True)
-            te_cfgs[0].tokenizer_kwargs.setdefault("return_tensors", "pt")
-        if len(te_cfgs) >= 2:
-            cap = 512
-            te_cfgs[1].tokenizer_kwargs["max_length"] = min(int(te_cfgs[1].tokenizer_kwargs.get("max_length", cap)),
-                                                            cap)
-            te_cfgs[1].tokenizer_kwargs.setdefault("padding", "max_length")
-            te_cfgs[1].tokenizer_kwargs.setdefault("truncation", True)
-            te_cfgs[1].tokenizer_kwargs.setdefault("return_tensors", "pt")

fastvideo/configs/pipelines/glm_image.py

@@ -1,46 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-from collections.abc import Callable
-from dataclasses import dataclass, field
-
-import torch
-
-from fastvideo.configs.models import DiTConfig, EncoderConfig, VAEConfig
-from fastvideo.configs.models.dits.glm_image import GlmImageDiTConfig
-from fastvideo.configs.models.encoders import BaseEncoderOutput, T5Config
-from fastvideo.configs.models.vaes.glm_image import GlmImageVAEConfig
-from fastvideo.configs.pipelines.base import PipelineConfig
-
-
-def glm_image_t5_postprocess(outputs: BaseEncoderOutput) -> torch.Tensor:
-    mask: torch.Tensor = outputs.attention_mask
-    hidden_state: torch.Tensor = outputs.last_hidden_state
-    seq_lens = mask.gt(0).sum(dim=1).long()
-
-    assert torch.isnan(hidden_state).sum() == 0, "T5 hidden states contain NaN"
-
-    max_len = 512
-    prompt_embeds = [u[:min(v, max_len)] for u, v in zip(hidden_state, seq_lens, strict=True)]
-    prompt_embeds_tensor: torch.Tensor = torch.stack(
-        [torch.cat([u, u.new_zeros(max_len - u.size(0), u.size(1))]) for u in prompt_embeds], dim=0)
-
-    return prompt_embeds_tensor
-
-
-@dataclass
-class GlmImageConfig(PipelineConfig):
-
-    dit_config: DiTConfig = field(default_factory=GlmImageDiTConfig)
-    dit_precision: str = "bf16"
-
-    vae_config: VAEConfig = field(default_factory=GlmImageVAEConfig)
-    vae_precision: str = "fp32"
-    vae_tiling: bool = True
-    vae_sp: bool = False
-
-    text_encoder_configs: tuple[EncoderConfig, ...] = field(default_factory=lambda: (T5Config(), ))
-    text_encoder_precisions: tuple[str, ...] = field(default_factory=lambda: ("fp32", ))
-    postprocess_text_funcs: tuple[Callable[[BaseEncoderOutput], torch.Tensor],
-                                  ...] = field(default_factory=lambda: (glm_image_t5_postprocess, ))
-
-    flow_shift: float | None = 1.0
-    embedded_cfg_scale: float = 7.5

fastvideo/dataset/parquet_dataset_map_style.py

@@ -2,7 +2,6 @@
 import os
 import pickle
 import random
-from collections.abc import Sequence
 from typing import Any
 
 import pyarrow as pa
@@ -95,56 +94,8 @@ class DP_SP_BatchSampler(Sampler[list[int]]):
         return len(self.sp_group_local_indices) // self.batch_size
 
 
-def _parse_data_path_specs(path: str | Sequence[str] | dict[str, int]) -> list[tuple[str, int]]:
-    """Parse one or more dataset roots with old-framework repeat counts."""
-    if isinstance(path, dict):
-        return [
-            (str(root), int(repeat))
-            for root, repeat in path.items()
-            if int(repeat) > 0
-        ]
-    if isinstance(path, Sequence) and not isinstance(path, str):
-        return [(str(root), 1) for root in path]
-
-    specs: list[tuple[str, int]] = []
-    for part in str(path).split(","):
-        part = part.strip()
-        if not part:
-            continue
-        if ":" in part:
-            dir_path, count_str = part.rsplit(":", 1)
-            count = int(count_str)
-        else:
-            dir_path, count = part, 1
-        if count > 0:
-            specs.append((dir_path.strip(), count))
-    return specs
-
-
-def get_parquet_files_and_length(path: str | Sequence[str] | dict[str, int]):
-    specs = _parse_data_path_specs(path)
-    if len(specs) != 1 or specs[0][1] != 1:
-        all_file_names: list[str] = []
-        all_lengths: list[int] = []
-        for root, repeat in specs:
-            file_names, lengths = get_parquet_files_and_length(str(root))
-            for _ in range(repeat):
-                all_file_names.extend(file_names)
-                all_lengths.extend(lengths)
-        if not all_file_names:
-            raise FileNotFoundError(
-                "No parquet files found under dataset paths: "
-                f"{path}. "
-                "Please verify these paths point to preprocessed parquet data."
-            )
-        file_lengths = sorted(
-            zip(all_file_names, all_lengths, strict=True),
-            key=lambda x: x[0],
-        )
-        file_names_sorted, lengths_sorted = zip(*file_lengths, strict=True)
-        return file_names_sorted, lengths_sorted
-
-    dataset_root = os.path.realpath(os.path.expanduser(specs[0][0]))
+def get_parquet_files_and_length(path: str):
+    dataset_root = os.path.realpath(os.path.expanduser(path))
     # Check if cached info exists
     cache_dir = os.path.join(dataset_root, "map_style_cache")
     cache_file = os.path.join(cache_dir, "file_info.pkl")
@@ -317,7 +268,7 @@ class LatentsParquetMapStyleDataset(Dataset):
 
     def __init__(
         self,
-        path: str | Sequence[str] | dict[str, int],
+        path: str,
         batch_size: int,
         parquet_schema: pa.Schema,
         cfg_rate: float = 0.0,
@@ -407,6 +358,7 @@ class LatentsParquetMapStyleDataset(Dataset):
     def __len__(self):
         return sum(self.lengths)
 
+
 # ────────────────────────────────────────────────────────────────────────────
 # 3.  Loader helper – everything else stays just like your original trainer
 # ────────────────────────────────────────────────────────────────────────────

fastvideo/eval/README.md

@@ -122,44 +122,6 @@ to run a subset of the Evaluator's registered metrics on this batch
 (useful for scoring different corpora with different metric subsets in
 successive calls without burning model loads).
 
-### Training-time validation metrics
-
-The modular trainer can score validation videos during training through
-`callbacks.validation.metrics`. Each distributed rank that writes local
-validation videos also runs its local metrics on `cuda:<local_rank>`;
-rank 0 only merges scalar summaries and logs artifacts.
-
-```yaml
-callbacks:
-  validation:
-    pipeline_target: fastvideo.pipelines.basic.matrixgame2.matrixgame2_causal_dmd_pipeline.MatrixGame2CausalDMDPipeline
-    dataset_file: examples/distill/MatrixGame2.0/validation.json
-    sampling_steps: [4]
-    metrics:
-      enabled: true
-      names:
-        - vbench.imaging_quality
-        - vbench.aesthetic_quality
-        - optical_flow.synthetic_optical_flow
-        - common.fvd
-      skip_missing_deps: true
-      strict: false
-      unload_after_validation: true
-
-      # `synthetic_optical_flow` also needs actions in the validation
-      # manifest (`action_path`) plus a repo-local calibration file.
-      calibration_path: assets/eval/worldmodel_synthetic_flow_calibration.json
-```
-
-Metric summaries are written under
-`<output_dir>/eval/step_<step>/inference_steps_<n>_rank_<rank>.json` and
-scalar means are logged with the `metrics/validation/...` prefix.
-Install `fastvideo[eval]` for optical-flow dependencies such as
-`ptlflow`; VBench metrics use the pinned submodule under
-`fastvideo/third_party/eval/vbench`. FVD uses `ref_video` entries from
-the validation manifest when present, or the standard
-`FASTVIDEO_FVD_REF_FEATURES` / eval-cache reference feature path.
-
 ### CLI
 
 ```bash
@@ -388,3 +350,5 @@ sweep several baselines into a table, see
 
 - **MIND** metrics. Depend on a separate `vipe` upstream submodule.
 - **VBench-2.0**. Sibling vbench2 package; needs its own port.
+- **Training-time eval callback** (`EvalCallback`) and the
+  `RolloutEvaluator` helper.

fastvideo/eval/metrics/optical_flow/synthetic_optical_flow/_thirdperson.py

@@ -63,7 +63,7 @@ class ThirdPersonCalibration:
     beta_fwd: float
     beta_strafe: float
     focal_length: float
-    r_z: float = 0.0
+    r_z: float
     r_y: float = 0.0
     init_pitch: float = 0.0
     notes: str = ""

fastvideo/layers/quantization/nvfp4_config.py

@@ -261,22 +261,6 @@ def _mm_fp4(
     )
 
 
-def _coerce_fp4_input_dtype(x: torch.Tensor) -> torch.Tensor:
-    """Coerce an activation to a dtype the FP4 linear accepts.
-
-    The pre-attention norm can emit fp32 (e.g. in eager mode, without the
-    torch.compile fusion that keeps it bf16). The FP4 linear emits bf16
-    regardless (see _mm_fp4 out dtype), so cast fp32 -> bf16 rather than
-    failing, matching the sibling fastvideo/layers/fp4linear.py. Non-floating
-    inputs (e.g. int/bool) are a genuine error and are rejected fast.
-    """
-    if not x.is_floating_point():
-        raise TypeError(f"fp4 linear expects floating-point inputs, got {x.dtype}")
-    if x.dtype not in (torch.bfloat16, torch.float16):
-        x = x.to(torch.bfloat16)
-    return x
-
-
 class NVFP4QuantizeMethod(QuantizeMethodBase):
 
     def __init__(self, layer_prefix: str = ""):
@@ -301,7 +285,8 @@ class NVFP4QuantizeMethod(QuantizeMethodBase):
 
     def quantize_input(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
         SfLayout, _, _ = _require_flashinfer()
-        x = _coerce_fp4_input_dtype(x)
+        assert x.dtype == torch.bfloat16 or x.dtype == torch.float16, (
+            f"only allow bf16/fp16 inputs to fp4 linear, got {x.dtype}")
         x_2d = x.view(-1, x.shape[-1])
         x_fp4, x_scale = _nvfp4_quantize(
             x_2d,
@@ -347,7 +332,8 @@ class NVFP4QuantizeMethod(QuantizeMethodBase):
             if x_scale.dim() > 2:
                 x_scale = x_scale.view(-1, x_scale.shape[-1])
         else:
-            x = _coerce_fp4_input_dtype(x)
+            assert x.dtype == torch.bfloat16 or x.dtype == torch.float16, (
+                f"only allow bf16/fp16 inputs to fp4 linear, got {x.dtype}")
             x = x.view(-1, x.shape[-1])
             x_global_sf = self.x_global_sf
             x_fp4, x_scale = _nvfp4_quantize(

fastvideo/layers/rotary_embedding.py

@@ -295,7 +295,6 @@ def get_1d_rotary_pos_embed(
     interpolation_factor: float = 1.0,
     dtype: torch.dtype = torch.float32,
     use_real: bool = True,
-    freqs_dtype: torch.dtype | None = None,
 ) -> tuple[torch.Tensor, torch.Tensor]:
     """
     Precompute the frequency tensor for complex exponential (cis) with given dimensions.
@@ -320,16 +319,12 @@ def get_1d_rotary_pos_embed(
     if isinstance(pos, int):
         pos = torch.arange(pos).float()
 
-    # freqs_dtype is an alias for dtype (Diffusers-compatible calling convention).
-    if freqs_dtype is not None:
-        dtype = freqs_dtype
-
     # proposed by reddit user bloc97, to rescale rotary embeddings to longer sequence length without fine-tuning
     # has some connection to NTK literature
     if theta_rescale_factor != 1.0:
         theta *= theta_rescale_factor**(dim / (dim - 2))
 
-    freqs = 1.0 / (theta**(torch.arange(0, dim, 2, device=pos.device)[:(dim // 2)].to(dtype) / dim))  # [D/2]
+    freqs = 1.0 / (theta**(torch.arange(0, dim, 2)[:(dim // 2)].to(dtype) / dim))  # [D/2]
     freqs = torch.outer(pos * interpolation_factor, freqs)  # [S, D/2]
     freqs_cos = freqs.cos()  # [S, D/2]
     freqs_sin = freqs.sin()  # [S, D/2]
@@ -450,21 +445,6 @@ def get_nd_rotary_pos_embed(
     return cos, sin
 
 
-_ROTARY_POS_EMBED_CACHE: dict[tuple, tuple[torch.Tensor, torch.Tensor]] = {}
-# Bound the table cache so long-running servers / causal models (which vary
-# start_frame per frame) cannot grow it without limit; entries are large float64
-# tensors. Least-recently-used eviction keeps the active resolution(s) hot while
-# capping memory.
-_ROTARY_POS_EMBED_CACHE_MAXSIZE = 16
-
-
-def _hashable(value: Any) -> Any:
-    """Return a hashable view of a scalar or sequence for use in a cache key."""
-    if isinstance(value, list | tuple):
-        return tuple(value)
-    return value
-
-
 def get_rotary_pos_embed(
     rope_sizes,
     hidden_size,
@@ -515,31 +495,6 @@ def get_rotary_pos_embed(
         sp_rank = 0
         sp_world_size = 1
 
-    # Memoize on every output-affecting argument; the table is constant across
-    # denoising steps, so this avoids recomputing the float64 cos/sin tables.
-    cache_key = (
-        _hashable(rope_sizes),
-        tuple(rope_dim_list),
-        rope_theta,
-        _hashable(theta_rescale_factor),
-        _hashable(interpolation_factor),
-        shard_dim,
-        sp_rank,
-        sp_world_size,
-        dtype,
-        start_frame,
-        use_real,
-    )
-    cached = _ROTARY_POS_EMBED_CACHE.get(cache_key)
-    if cached is not None:
-        # Move to most-recently-used position so the active table is not evicted
-        # when several resolutions / buckets share the process (LRU recency).
-        # Pop with a default: a concurrent eviction between the get() above and
-        # here would otherwise raise KeyError on the hit path.
-        if _ROTARY_POS_EMBED_CACHE.pop(cache_key, None) is not None:
-            _ROTARY_POS_EMBED_CACHE[cache_key] = cached
-        return cached
-
     freqs_cos, freqs_sin = get_nd_rotary_pos_embed(
         rope_dim_list,
         rope_sizes,
@@ -553,14 +508,6 @@ def get_rotary_pos_embed(
         start_frame=start_frame,
         use_real=use_real,
     )
-    # The returned tensors are shared cache entries: callers must never mutate
-    # them in place. Note .to(device) is an identity alias when the tensor is
-    # already on the target device (e.g. CPU runs), so it does NOT guarantee a
-    # copy — treat the tables as read-only and copy before any in-place op.
-    # Reached only on a miss, so evict the least-recently-used entry at capacity.
-    if len(_ROTARY_POS_EMBED_CACHE) >= _ROTARY_POS_EMBED_CACHE_MAXSIZE:
-        _ROTARY_POS_EMBED_CACHE.pop(next(iter(_ROTARY_POS_EMBED_CACHE)))
-    _ROTARY_POS_EMBED_CACHE[cache_key] = (freqs_cos, freqs_sin)
     return freqs_cos, freqs_sin
 
 

fastvideo/models/dits/causal_wanvideo.py

@@ -665,8 +665,7 @@ class CausalWanTransformer3DModel(BaseDiT):
             else:
                 causal_kwargs = {
                     "kv_cache": kv_cache[block_index],
-                    "crossattn_cache": (crossattn_cache[block_index]
-                                        if crossattn_cache is not None else None),
+                    "crossattn_cache": crossattn_cache[block_index],
                     "current_start": current_start,
                     "cache_start": cache_start,
                     "block_mask": self.block_mask,

fastvideo/models/dits/flux.py

@@ -1,578 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-
-from __future__ import annotations
-
-from contextlib import nullcontext
-from dataclasses import dataclass
-import math
-from typing import Any
-
-import torch
-import torch.nn as nn
-
-from fastvideo.layers.rotary_embedding import apply_rotary_emb, get_1d_rotary_pos_embed
-
-from fastvideo.attention import DistributedAttention
-from fastvideo.configs.models import DiTConfig
-from fastvideo.forward_context import get_forward_context, set_forward_context
-from fastvideo.layers.linear import ReplicatedLinear
-from fastvideo.layers.visual_embedding import Timesteps
-from fastvideo.models.dits.base import BaseDiT
-from fastvideo.models.dits.sd3 import (
-    CombinedTimestepTextProjEmbeddings,
-    SD3AdaLayerNormContinuous,
-    SD3AdaLayerNormZero,
-    SD3FeedForward,
-    SD3TextProjection,
-    SD3TimestepEmbedding,
-)
-from fastvideo.platforms import AttentionBackendEnum
-
-
-@dataclass
-class FluxTransformer2DModelOutput:
-    sample: torch.Tensor
-
-
-class FluxPosEmbed(nn.Module):
-    """1D RoPE axes concatenated per Diffusers `FluxPosEmbed`."""
-
-    def __init__(self, theta: int, axes_dim: list[int]) -> None:
-        super().__init__()
-        self.theta = theta
-        self.axes_dim = axes_dim
-
-    def forward(self, ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
-        n_axes = ids.shape[-1]
-        cos_out: list[torch.Tensor] = []
-        sin_out: list[torch.Tensor] = []
-        pos = ids.float()
-        is_mps = ids.device.type == "mps"
-        is_npu = ids.device.type == "npu"
-        freqs_dtype = torch.float32 if (is_mps or is_npu) else torch.float64
-        for i in range(n_axes):
-            cos, sin = get_1d_rotary_pos_embed(
-                self.axes_dim[i],
-                pos[:, i],
-                theta=self.theta,
-                use_real=True,
-                freqs_dtype=freqs_dtype,
-            )
-            cos_out.append(cos)
-            sin_out.append(sin)
-        freqs_cos = torch.cat(cos_out, dim=-1).to(ids.device)
-        freqs_sin = torch.cat(sin_out, dim=-1).to(ids.device)
-        return freqs_cos, freqs_sin
-
-
-class FluxCombinedTimestepGuidanceTextProjEmbeddings(nn.Module):
-    def __init__(self, embedding_dim: int, pooled_projection_dim: int) -> None:
-        super().__init__()
-        self.time_proj = Timesteps(
-            num_channels=256,
-            flip_sin_to_cos=True,
-            downscale_freq_shift=0,
-        )
-        self.timestep_embedder = SD3TimestepEmbedding(
-            in_channels=256,
-            time_embed_dim=embedding_dim,
-            act_fn="silu",
-        )
-        self.guidance_embedder = SD3TimestepEmbedding(
-            in_channels=256,
-            time_embed_dim=embedding_dim,
-            act_fn="silu",
-        )
-        self.text_embedder = SD3TextProjection(
-            pooled_projection_dim,
-            embedding_dim,
-            act_fn="silu",
-        )
-
-    def forward(
-        self,
-        timestep: torch.Tensor,
-        guidance: torch.Tensor,
-        pooled_projection: torch.Tensor,
-    ) -> torch.Tensor:
-        timesteps_proj = self.time_proj(timestep)
-        timesteps_emb = self.timestep_embedder(timesteps_proj.to(dtype=pooled_projection.dtype))
-        guidance_proj = self.time_proj(guidance)
-        guidance_emb = self.guidance_embedder(guidance_proj.to(dtype=pooled_projection.dtype))
-        time_guidance_emb = timesteps_emb + guidance_emb
-        pooled_projections = self.text_embedder(pooled_projection)
-        return time_guidance_emb + pooled_projections
-
-
-class FluxAdaLayerNormZeroSingle(nn.Module):
-    def __init__(self, embedding_dim: int, bias: bool = True) -> None:
-        super().__init__()
-        self.silu = nn.SiLU()
-        self.linear = ReplicatedLinear(embedding_dim, 3 * embedding_dim, bias=bias)
-        self.norm = nn.LayerNorm(embedding_dim, elementwise_affine=False, eps=1e-6)
-
-    def forward(
-        self,
-        x: torch.Tensor,
-        emb: torch.Tensor,
-    ) -> tuple[torch.Tensor, torch.Tensor]:
-        emb, _ = self.linear(self.silu(emb))
-        shift_msa, scale_msa, gate_msa = emb.chunk(3, dim=1)
-        x = self.norm(x) * (1 + scale_msa[:, None]) + shift_msa[:, None]
-        return x, gate_msa
-
-
-class FluxJointAttention(nn.Module):
-    """Joint attention: text tokens precede image tokens (Diffusers order)."""
-
-    def __init__(
-        self,
-        dim: int,
-        num_attention_heads: int,
-        attention_head_dim: int,
-        supported_attention_backends: tuple[AttentionBackendEnum, ...] | None = None,
-    ) -> None:
-        super().__init__()
-        self.heads = num_attention_heads
-        self.head_dim = attention_head_dim
-        self.inner_dim = num_attention_heads * attention_head_dim
-
-        self.norm_q = nn.RMSNorm(attention_head_dim, eps=1e-6)
-        self.norm_k = nn.RMSNorm(attention_head_dim, eps=1e-6)
-        self.norm_added_q = nn.RMSNorm(attention_head_dim, eps=1e-6)
-        self.norm_added_k = nn.RMSNorm(attention_head_dim, eps=1e-6)
-
-        self.to_q = ReplicatedLinear(dim, self.inner_dim, bias=True)
-        self.to_k = ReplicatedLinear(dim, self.inner_dim, bias=True)
-        self.to_v = ReplicatedLinear(dim, self.inner_dim, bias=True)
-        self.add_q_proj = ReplicatedLinear(dim, self.inner_dim, bias=True)
-        self.add_k_proj = ReplicatedLinear(dim, self.inner_dim, bias=True)
-        self.add_v_proj = ReplicatedLinear(dim, self.inner_dim, bias=True)
-
-        self.to_out = nn.ModuleList(
-            [
-                ReplicatedLinear(self.inner_dim, dim, bias=True),
-                nn.Dropout(0.0),
-            ]
-        )
-        self.to_add_out = ReplicatedLinear(self.inner_dim, dim, bias=True)
-
-        self.attn = DistributedAttention(
-            num_heads=num_attention_heads,
-            head_size=attention_head_dim,
-            causal=False,
-            supported_attention_backends=supported_attention_backends,
-        )
-
-    def forward(
-        self,
-        hidden_states: torch.Tensor,
-        encoder_hidden_states: torch.Tensor,
-        image_rotary_emb: tuple[torch.Tensor, torch.Tensor],
-    ) -> tuple[torch.Tensor, torch.Tensor]:
-        batch_size = hidden_states.shape[0]
-        text_seq_len = encoder_hidden_states.shape[1]
-        img_seq_len = hidden_states.shape[1]
-
-        q, _ = self.to_q(hidden_states)
-        k, _ = self.to_k(hidden_states)
-        v, _ = self.to_v(hidden_states)
-        q = q.view(batch_size, img_seq_len, self.heads, self.head_dim)
-        k = k.view(batch_size, img_seq_len, self.heads, self.head_dim)
-        v = v.view(batch_size, img_seq_len, self.heads, self.head_dim)
-        q = self.norm_q(q)
-        k = self.norm_k(k)
-
-        enc_q, _ = self.add_q_proj(encoder_hidden_states)
-        enc_k, _ = self.add_k_proj(encoder_hidden_states)
-        enc_v, _ = self.add_v_proj(encoder_hidden_states)
-        enc_q = enc_q.view(batch_size, text_seq_len, self.heads, self.head_dim)
-        enc_k = enc_k.view(batch_size, text_seq_len, self.heads, self.head_dim)
-        enc_v = enc_v.view(batch_size, text_seq_len, self.heads, self.head_dim)
-        enc_q = self.norm_added_q(enc_q)
-        enc_k = self.norm_added_k(enc_k)
-
-        q = torch.cat([enc_q, q], dim=1)
-        k = torch.cat([enc_k, k], dim=1)
-        v = torch.cat([enc_v, v], dim=1)
-
-        q = apply_rotary_emb(q, image_rotary_emb, sequence_dim=1)
-        k = apply_rotary_emb(k, image_rotary_emb, sequence_dim=1)
-
-        joint_out, _ = self.attn(q, k, v)
-        joint_out = joint_out.reshape(batch_size, text_seq_len + img_seq_len, self.inner_dim)
-
-        enc_out = joint_out[:, :text_seq_len]
-        img_out = joint_out[:, text_seq_len:]
-
-        img_out, _ = self.to_out[0](img_out)
-        img_out = self.to_out[1](img_out)
-        enc_out, _ = self.to_add_out(enc_out)
-        return img_out, enc_out
-
-
-class FluxSingleStreamAttention(nn.Module):
-    """Self-attention on concatenated text+image sequence (single blocks)."""
-
-    def __init__(
-        self,
-        dim: int,
-        num_attention_heads: int,
-        attention_head_dim: int,
-        supported_attention_backends: tuple[AttentionBackendEnum, ...] | None = None,
-    ) -> None:
-        super().__init__()
-        self.heads = num_attention_heads
-        self.head_dim = attention_head_dim
-        self.inner_dim = num_attention_heads * attention_head_dim
-
-        self.norm_q = nn.RMSNorm(attention_head_dim, eps=1e-6)
-        self.norm_k = nn.RMSNorm(attention_head_dim, eps=1e-6)
-        self.to_q = ReplicatedLinear(dim, self.inner_dim, bias=True)
-        self.to_k = ReplicatedLinear(dim, self.inner_dim, bias=True)
-        self.to_v = ReplicatedLinear(dim, self.inner_dim, bias=True)
-        self.attn = DistributedAttention(
-            num_heads=num_attention_heads,
-            head_size=attention_head_dim,
-            causal=False,
-            supported_attention_backends=supported_attention_backends,
-        )
-
-    def forward(
-        self,
-        hidden_states: torch.Tensor,
-        image_rotary_emb: tuple[torch.Tensor, torch.Tensor],
-    ) -> torch.Tensor:
-        batch_size, seq_len, _ = hidden_states.shape
-        q, _ = self.to_q(hidden_states)
-        k, _ = self.to_k(hidden_states)
-        v, _ = self.to_v(hidden_states)
-        q = q.view(batch_size, seq_len, self.heads, self.head_dim)
-        k = k.view(batch_size, seq_len, self.heads, self.head_dim)
-        v = v.view(batch_size, seq_len, self.heads, self.head_dim)
-        q = self.norm_q(q)
-        k = self.norm_k(k)
-        q = apply_rotary_emb(q, image_rotary_emb, sequence_dim=1)
-        k = apply_rotary_emb(k, image_rotary_emb, sequence_dim=1)
-        out, _ = self.attn(q, k, v)
-        return out.reshape(batch_size, seq_len, self.inner_dim)
-
-
-class FluxTransformerBlock(nn.Module):
-    def __init__(
-        self,
-        dim: int,
-        num_attention_heads: int,
-        attention_head_dim: int,
-        supported_attention_backends: tuple[AttentionBackendEnum, ...] | None = None,
-    ) -> None:
-        super().__init__()
-        self.norm1 = SD3AdaLayerNormZero(dim)
-        self.norm1_context = SD3AdaLayerNormZero(dim)
-        self.attn = FluxJointAttention(
-            dim=dim,
-            num_attention_heads=num_attention_heads,
-            attention_head_dim=attention_head_dim,
-            supported_attention_backends=supported_attention_backends,
-        )
-        self.norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
-        self.ff = SD3FeedForward(
-            dim=dim,
-            dim_out=dim,
-            activation_fn="gelu-approximate",
-        )
-        self.norm2_context = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
-        self.ff_context = SD3FeedForward(
-            dim=dim,
-            dim_out=dim,
-            activation_fn="gelu-approximate",
-        )
-
-    def forward(
-        self,
-        hidden_states: torch.Tensor,
-        encoder_hidden_states: torch.Tensor,
-        temb: torch.Tensor,
-        image_rotary_emb: tuple[torch.Tensor, torch.Tensor],
-        joint_attention_kwargs: dict[str, Any] | None = None,
-    ) -> tuple[torch.Tensor, torch.Tensor]:
-        del joint_attention_kwargs
-        norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1(hidden_states, emb=temb)
-        (norm_encoder_hidden_states, c_gate_msa, c_shift_mlp, c_scale_mlp, c_gate_mlp) = self.norm1_context(
-            encoder_hidden_states, emb=temb
-        )
-
-        attn_output, context_attn_output = self.attn(
-            hidden_states=norm_hidden_states,
-            encoder_hidden_states=norm_encoder_hidden_states,
-            image_rotary_emb=image_rotary_emb,
-        )
-
-        attn_output = gate_msa.unsqueeze(1) * attn_output
-        hidden_states = hidden_states + attn_output
-
-        norm_hidden_states = self.norm2(hidden_states)
-        norm_hidden_states = norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]
-        ff_output = self.ff(norm_hidden_states)
-        hidden_states = hidden_states + gate_mlp.unsqueeze(1) * ff_output
-
-        context_attn_output = c_gate_msa.unsqueeze(1) * context_attn_output
-        encoder_hidden_states = encoder_hidden_states + context_attn_output
-
-        norm_encoder_hidden_states = self.norm2_context(encoder_hidden_states)
-        norm_encoder_hidden_states = norm_encoder_hidden_states * (1 + c_scale_mlp[:, None]) + c_shift_mlp[:, None]
-        context_ff_output = self.ff_context(norm_encoder_hidden_states)
-        encoder_hidden_states = encoder_hidden_states + (c_gate_mlp.unsqueeze(1) * context_ff_output)
-        if encoder_hidden_states.dtype == torch.float16:
-            encoder_hidden_states = encoder_hidden_states.clip(-65504, 65504)
-
-        return encoder_hidden_states, hidden_states
-
-
-class FluxSingleTransformerBlock(nn.Module):
-    def __init__(
-        self,
-        dim: int,
-        num_attention_heads: int,
-        attention_head_dim: int,
-        mlp_ratio: float = 4.0,
-        supported_attention_backends: tuple[AttentionBackendEnum, ...] | None = None,
-    ) -> None:
-        super().__init__()
-        mlp_hidden_dim = int(dim * mlp_ratio)
-        self.norm = FluxAdaLayerNormZeroSingle(dim)
-        self.proj_mlp = ReplicatedLinear(dim, mlp_hidden_dim, bias=True)
-        self.act_mlp = nn.GELU(approximate="tanh")
-        self.proj_out = ReplicatedLinear(dim + mlp_hidden_dim, dim, bias=True)
-        self.attn = FluxSingleStreamAttention(
-            dim=dim,
-            num_attention_heads=num_attention_heads,
-            attention_head_dim=attention_head_dim,
-            supported_attention_backends=supported_attention_backends,
-        )
-
-    def forward(
-        self,
-        hidden_states: torch.Tensor,
-        encoder_hidden_states: torch.Tensor,
-        temb: torch.Tensor,
-        image_rotary_emb: tuple[torch.Tensor, torch.Tensor],
-        joint_attention_kwargs: dict[str, Any] | None = None,
-    ) -> tuple[torch.Tensor, torch.Tensor]:
-        del joint_attention_kwargs
-        text_seq_len = encoder_hidden_states.shape[1]
-        hidden_states = torch.cat([encoder_hidden_states, hidden_states], dim=1)
-        residual = hidden_states
-        norm_hidden_states, gate = self.norm(hidden_states, emb=temb)
-        mlp_hidden_states = self.act_mlp(self.proj_mlp(norm_hidden_states)[0])
-        attn_output = self.attn(
-            hidden_states=norm_hidden_states,
-            image_rotary_emb=image_rotary_emb,
-        )
-        hidden_states = torch.cat([attn_output, mlp_hidden_states], dim=2)
-        gate = gate.unsqueeze(1)
-        hidden_states = gate * self.proj_out(hidden_states)[0]
-        hidden_states = residual + hidden_states
-        if hidden_states.dtype == torch.float16:
-            hidden_states = hidden_states.clip(-65504, 65504)
-        encoder_hidden_states = hidden_states[:, :text_seq_len]
-        hidden_states = hidden_states[:, text_seq_len:]
-        return encoder_hidden_states, hidden_states
-
-
-class FluxTransformer2DModel(BaseDiT):
-    """FastVideo FLUX transformer; load Diffusers FLUX safetensors 1:1."""
-
-    _fsdp_shard_conditions = [
-        lambda n, m: (n.startswith("transformer_blocks.") or n.startswith("single_transformer_blocks."))
-        and n.split(".")[-1].isdigit(),
-    ]
-    _compile_conditions = _fsdp_shard_conditions
-    # HF weight names already match this module layout (cf. SGLang regex maps).
-    param_names_mapping: dict[str, Any] = {}
-    reverse_param_names_mapping: dict[str, Any] = {}
-    lora_param_names_mapping: dict[str, Any] = {}
-    _supported_attention_backends = (
-        AttentionBackendEnum.FLASH_ATTN,
-        AttentionBackendEnum.TORCH_SDPA,
-    )
-
-    def __init__(self, config: DiTConfig, hf_config: dict[str, Any], **kwargs) -> None:
-        del kwargs
-        super().__init__(config=config, hf_config=hf_config)
-        self.fastvideo_config = config
-        self.hf_config = hf_config
-        arch = config.arch_config
-
-        out_ch = arch.out_channels
-        self.out_channels = out_ch if out_ch is not None else arch.in_channels
-        self.inner_dim = arch.num_attention_heads * arch.attention_head_dim
-        self.hidden_size = self.inner_dim
-        self.num_attention_heads = arch.num_attention_heads
-        self.num_channels_latents = arch.in_channels
-
-        axes_list = list(arch.axes_dims_rope)
-        self.pos_embed = FluxPosEmbed(theta=10000, axes_dim=axes_list)
-        if arch.guidance_embeds:
-            self.time_text_embed = FluxCombinedTimestepGuidanceTextProjEmbeddings(
-                embedding_dim=self.inner_dim,
-                pooled_projection_dim=arch.pooled_projection_dim,
-            )
-        else:
-            self.time_text_embed = CombinedTimestepTextProjEmbeddings(
-                embedding_dim=self.inner_dim,
-                pooled_projection_dim=arch.pooled_projection_dim,
-            )
-        self.context_embedder = ReplicatedLinear(arch.joint_attention_dim, self.inner_dim)
-        self.x_embedder = ReplicatedLinear(arch.in_channels, self.inner_dim)
-
-        self.transformer_blocks = nn.ModuleList(
-            [
-                FluxTransformerBlock(
-                    dim=self.inner_dim,
-                    num_attention_heads=arch.num_attention_heads,
-                    attention_head_dim=arch.attention_head_dim,
-                    supported_attention_backends=self._supported_attention_backends,
-                )
-                for _ in range(arch.num_layers)
-            ]
-        )
-        self.single_transformer_blocks = nn.ModuleList(
-            [
-                FluxSingleTransformerBlock(
-                    dim=self.inner_dim,
-                    num_attention_heads=arch.num_attention_heads,
-                    attention_head_dim=arch.attention_head_dim,
-                    supported_attention_backends=self._supported_attention_backends,
-                )
-                for _ in range(arch.num_single_layers)
-            ]
-        )
-
-        self.norm_out = SD3AdaLayerNormContinuous(
-            self.inner_dim,
-            self.inner_dim,
-            elementwise_affine=False,
-            eps=1e-6,
-            bias=True,
-            norm_type="layer_norm",
-        )
-        self.proj_out = ReplicatedLinear(
-            self.inner_dim,
-            arch.patch_size * arch.patch_size * self.out_channels,
-            bias=True,
-        )
-        self.gradient_checkpointing = False
-        self.__post_init__()
-
-    def forward(
-        self,
-        hidden_states: torch.Tensor,
-        encoder_hidden_states: torch.Tensor | None = None,
-        pooled_projections: torch.Tensor | None = None,
-        timestep: torch.LongTensor | torch.Tensor | None = None,
-        img_ids: torch.Tensor | None = None,
-        txt_ids: torch.Tensor | None = None,
-        guidance: torch.Tensor | None = None,
-        joint_attention_kwargs: dict[str, Any] | None = None,
-        return_dict: bool = True,
-        controlnet_block_samples: Any | None = None,
-        controlnet_single_block_samples: Any | None = None,
-        controlnet_blocks_repeat: bool = False,
-        **kwargs: Any,
-    ) -> FluxTransformer2DModelOutput | tuple[torch.Tensor, ...]:
-        del kwargs
-        if encoder_hidden_states is None:
-            raise ValueError("encoder_hidden_states must be provided")
-        if pooled_projections is None:
-            raise ValueError("pooled_projections must be provided")
-        if timestep is None:
-            raise ValueError("timestep must be provided")
-        if img_ids is None or txt_ids is None:
-            raise ValueError("img_ids and txt_ids must be provided")
-
-        arch = self.fastvideo_config.arch_config
-        if arch.guidance_embeds and guidance is None:
-            raise ValueError("guidance must be provided when guidance_embeds=True")
-
-        if timestep.dim() == 0:
-            timestep = timestep[None]
-        if timestep.dim() > 1:
-            timestep = timestep.reshape(-1)
-        if timestep.shape[0] == 1 and hidden_states.shape[0] > 1:
-            timestep = timestep.expand(hidden_states.shape[0])
-
-        try:
-            get_forward_context()
-            forward_context = nullcontext()
-        except AssertionError:
-            if timestep.numel() == 0:
-                ts0 = 0
-            elif torch.is_floating_point(timestep):
-                ts0 = int(round(timestep[0].item() * 1000))
-            else:
-                ts0 = int(timestep[0].item())
-            forward_context = set_forward_context(current_timestep=ts0, attn_metadata=None)
-
-        with forward_context:
-            hidden_states, _ = self.x_embedder(hidden_states)
-
-            ts = timestep.to(hidden_states.dtype) * 1000
-            g = None if guidance is None else guidance.to(hidden_states.dtype) * 1000
-
-            if arch.guidance_embeds:
-                assert g is not None
-                temb = self.time_text_embed(ts, g, pooled_projections)
-            else:
-                temb = self.time_text_embed(timestep=ts, pooled_projection=pooled_projections)
-
-            encoder_hidden_states, _ = self.context_embedder(encoder_hidden_states)
-
-            if txt_ids.ndim == 3:
-                txt_ids = txt_ids[0]
-            if img_ids.ndim == 3:
-                img_ids = img_ids[0]
-
-            ids = torch.cat((txt_ids, img_ids), dim=0)
-            image_rotary_emb = self.pos_embed(ids)
-
-            jkwargs = joint_attention_kwargs or {}
-
-            for idx, block in enumerate(self.transformer_blocks):
-                encoder_hidden_states, hidden_states = block(
-                    hidden_states=hidden_states,
-                    encoder_hidden_states=encoder_hidden_states,
-                    temb=temb,
-                    image_rotary_emb=image_rotary_emb,
-                    joint_attention_kwargs=jkwargs,
-                )
-                if controlnet_block_samples:
-                    interval = len(self.transformer_blocks) / len(controlnet_block_samples)
-                    interval = int(math.ceil(interval))
-                    if controlnet_blocks_repeat:
-                        hidden_states = hidden_states + controlnet_block_samples[idx % len(controlnet_block_samples)]
-                    else:
-                        hidden_states = hidden_states + controlnet_block_samples[idx // interval]
-
-            for idx, block in enumerate(self.single_transformer_blocks):
-                encoder_hidden_states, hidden_states = block(
-                    hidden_states=hidden_states,
-                    encoder_hidden_states=encoder_hidden_states,
-                    temb=temb,
-                    image_rotary_emb=image_rotary_emb,
-                    joint_attention_kwargs=jkwargs,
-                )
-                if controlnet_single_block_samples:
-                    interval = len(self.single_transformer_blocks) / len(controlnet_single_block_samples)
-                    interval = int(math.ceil(interval))
-                    hidden_states = hidden_states + controlnet_single_block_samples[idx // interval]
-
-            hidden_states = self.norm_out(hidden_states, temb)
-            output, _ = self.proj_out(hidden_states)
-
-            if not return_dict:
-                return (output,)
-            return FluxTransformer2DModelOutput(sample=output)
-
-
-EntryClass = FluxTransformer2DModel

fastvideo/models/dits/glm_image.py

@@ -1,776 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-from typing import Any, Dict, List, Optional, Tuple, Union
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from fastvideo.layers.mlp import MLP
-
-from fastvideo.attention import LocalAttention
-from fastvideo.configs.models.dits.glm_image import GlmImageDiTConfig
-from fastvideo.layers.layernorm import ScaleResidualLayerNormScaleShift
-from fastvideo.layers.linear import ReplicatedLinear
-from fastvideo.layers.rotary_embedding import _apply_rotary_emb
-from fastvideo.layers.visual_embedding import Timesteps
-from fastvideo.models.dits.base import BaseDiT
-from fastvideo.platforms import AttentionBackendEnum
-
-
-class GlmImageLayerKVCache:
-
-    def __init__(self):
-        self.k_cache = None
-        self.v_cache = None
-        self.mode: Optional[str] = None
-
-    def store(self, k: torch.Tensor, v: torch.Tensor):
-        # Append along seq (dim=1).
-        if self.k_cache is None:
-            self.k_cache = k
-            self.v_cache = v
-        else:
-            self.k_cache = torch.cat([self.k_cache, k], dim=1)
-            self.v_cache = torch.cat([self.v_cache, v], dim=1)
-
-    def get(self):
-        return self.k_cache, self.v_cache
-
-    def clear(self):
-        self.k_cache = None
-        self.v_cache = None
-        self.mode = None
-
-
-class GlmImageKVCache:
-
-    def __init__(self, num_layers: int):
-        self.num_layers = num_layers
-        self.caches = [GlmImageLayerKVCache() for _ in range(num_layers)]
-
-    def __getitem__(self, layer_idx: int) -> GlmImageLayerKVCache:
-        return self.caches[layer_idx]
-
-    def set_mode(self, mode: Optional[str]):
-        if mode is not None and mode not in ["write", "read", "skip"]:
-            raise ValueError(
-                f"Invalid mode: {mode}, must be one of 'write', 'read', 'skip'"
-            )
-        for cache in self.caches:
-            cache.mode = mode
-
-    def clear(self):
-        for cache in self.caches:
-            cache.clear()
-
-
-# =============================================================================
-# Timestep and Text Projection
-# =============================================================================
-class GlmImageTimestepEmbedding(nn.Module):
-
-    def __init__(
-        self,
-        in_channels: int,
-        time_embed_dim: int,
-        act_fn: str = "silu",
-        out_dim: int = None,
-    ):
-        super().__init__()
-        if out_dim is None:
-            out_dim = time_embed_dim
-        self.linear_1 = ReplicatedLinear(in_channels, time_embed_dim, bias=True)
-        if act_fn == "silu":
-            self.act = nn.SiLU()
-        elif act_fn == "gelu":
-            self.act = nn.GELU(approximate="tanh")
-        else:
-            self.act = nn.SiLU()
-        self.linear_2 = ReplicatedLinear(time_embed_dim, out_dim, bias=True)
-
-    def forward(self, sample: torch.Tensor) -> torch.Tensor:
-        sample, _ = self.linear_1(sample)
-        sample = self.act(sample)
-        sample, _ = self.linear_2(sample)
-        return sample
-
-
-class GlmImageTextProjection(nn.Module):
-
-    def __init__(
-        self,
-        in_features: int,
-        hidden_size: int,
-        out_features: int = None,
-        act_fn: str = "silu",
-    ):
-        super().__init__()
-        if out_features is None:
-            out_features = hidden_size
-        self.linear_1 = ReplicatedLinear(in_features, hidden_size, bias=True)
-        if act_fn == "silu":
-            self.act_1 = nn.SiLU()
-        elif act_fn == "gelu_tanh":
-            self.act_1 = nn.GELU(approximate="tanh")
-        else:
-            self.act_1 = nn.SiLU()
-        self.linear_2 = ReplicatedLinear(hidden_size, out_features, bias=True)
-
-    def forward(self, caption: torch.Tensor) -> torch.Tensor:
-        hidden_states, _ = self.linear_1(caption)
-        hidden_states = self.act_1(hidden_states)
-        hidden_states, _ = self.linear_2(hidden_states)
-        return hidden_states
-
-
-class GlmImageCombinedTimestepSizeEmbeddings(nn.Module):
-
-    def __init__(
-        self,
-        embedding_dim: int,
-        condition_dim: int,
-        pooled_projection_dim: int,
-        timesteps_dim: int = 256,
-    ):
-        super().__init__()
-
-        self.time_proj = Timesteps(
-            num_channels=timesteps_dim, flip_sin_to_cos=True, downscale_freq_shift=0
-        )
-        self.condition_proj = Timesteps(
-            num_channels=condition_dim, flip_sin_to_cos=True, downscale_freq_shift=0
-        )
-        self.timestep_embedder = GlmImageTimestepEmbedding(
-            in_channels=timesteps_dim, time_embed_dim=embedding_dim
-        )
-        self.condition_embedder = GlmImageTextProjection(
-            pooled_projection_dim, embedding_dim, act_fn="silu"
-        )
-
-    def forward(
-        self,
-        timestep: torch.Tensor,
-        target_size: torch.Tensor,
-        crop_coords: torch.Tensor,
-        hidden_dtype: torch.dtype,
-    ) -> torch.Tensor:
-        timesteps_proj = self.time_proj(timestep)
-
-        crop_coords_proj = self.condition_proj(crop_coords.flatten()).view(
-            crop_coords.size(0), -1
-        )
-        target_size_proj = self.condition_proj(target_size.flatten()).view(
-            target_size.size(0), -1
-        )
-
-        condition_proj = torch.cat([crop_coords_proj, target_size_proj], dim=1)
-
-        timesteps_emb = self.timestep_embedder(
-            timesteps_proj.to(dtype=hidden_dtype)
-        )
-        condition_emb = self.condition_embedder(
-            condition_proj.to(dtype=hidden_dtype)
-        )
-
-        conditioning = timesteps_emb + condition_emb
-        return conditioning
-
-
-# =============================================================================
-# Image Projector
-# =============================================================================
-class GlmImageImageProjector(nn.Module):
-
-    def __init__(
-        self,
-        in_channels: int = 16,
-        hidden_size: int = 2560,
-        patch_size: int = 2,
-    ):
-        super().__init__()
-        self.patch_size = patch_size
-        self.proj = nn.Linear(in_channels * patch_size**2, hidden_size)
-
-    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
-        batch_size, channel, height, width = hidden_states.shape
-        post_patch_height = height // self.patch_size
-        post_patch_width = width // self.patch_size
-
-        hidden_states = hidden_states.reshape(
-            batch_size,
-            channel,
-            post_patch_height,
-            self.patch_size,
-            post_patch_width,
-            self.patch_size,
-        )
-        hidden_states = (
-            hidden_states.permute(0, 2, 4, 1, 3, 5).flatten(3, 5).flatten(1, 2)
-        )
-        hidden_states = self.proj(hidden_states)
-
-        return hidden_states
-
-
-# =============================================================================
-# AdaLayerNorm
-# =============================================================================
-class GlmImageAdaLayerNormZero(nn.Module):
-
-    def __init__(self, embedding_dim: int, dim: int) -> None:
-        super().__init__()
-
-        self.norm = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-5)
-        self.norm_context = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-5)
-        self.linear = ReplicatedLinear(embedding_dim, 12 * dim, bias=True)
-
-    def forward(
-        self,
-        hidden_states: torch.Tensor,
-        encoder_hidden_states: torch.Tensor,
-        temb: torch.Tensor,
-    ) -> Tuple[torch.Tensor, ...]:
-        dtype = hidden_states.dtype
-        norm_hidden_states = self.norm(hidden_states).to(dtype=dtype)
-        norm_encoder_hidden_states = self.norm_context(encoder_hidden_states).to(
-            dtype=dtype
-        )
-
-        emb, _ = self.linear(temb)
-        (
-            shift_msa,
-            c_shift_msa,
-            scale_msa,
-            c_scale_msa,
-            gate_msa,
-            c_gate_msa,
-            shift_mlp,
-            c_shift_mlp,
-            scale_mlp,
-            c_scale_mlp,
-            gate_mlp,
-            c_gate_mlp,
-        ) = emb.chunk(12, dim=1)
-
-        hidden_states = norm_hidden_states * (
-            1 + scale_msa.unsqueeze(1)
-        ) + shift_msa.unsqueeze(1)
-        encoder_hidden_states = norm_encoder_hidden_states * (
-            1 + c_scale_msa.unsqueeze(1)
-        ) + c_shift_msa.unsqueeze(1)
-
-        return (
-            hidden_states,
-            gate_msa,
-            shift_mlp,
-            scale_mlp,
-            gate_mlp,
-            encoder_hidden_states,
-            c_gate_msa,
-            c_shift_mlp,
-            c_scale_mlp,
-            c_gate_mlp,
-        )
-
-
-# =============================================================================
-# Attention
-# =============================================================================
-class GlmImageAttention(nn.Module):
-
-    def __init__(
-        self,
-        query_dim: int,
-        heads: int,
-        dim_head: int,
-        out_dim: int,
-        bias: bool = True,
-        qk_norm: str = "layer_norm",
-        elementwise_affine: bool = False,
-        eps: float = 1e-5,
-        supported_attention_backends: set[AttentionBackendEnum] | None = None,
-        prefix: str = "",
-    ):
-        super().__init__()
-
-        self.heads = out_dim // dim_head if out_dim is not None else heads
-        self.num_kv_heads = self.heads
-        self.dim_head = dim_head
-        self.inner_dim = out_dim if out_dim is not None else dim_head * heads
-        self.inner_kv_dim = self.inner_dim
-        self.out_dim = out_dim if out_dim is not None else query_dim
-
-        self.to_q = ReplicatedLinear(query_dim, self.inner_dim, bias=bias)
-        self.to_k = ReplicatedLinear(query_dim, self.inner_kv_dim, bias=bias)
-        self.to_v = ReplicatedLinear(query_dim, self.inner_kv_dim, bias=bias)
-
-        self.to_out = nn.ModuleList(
-            [ReplicatedLinear(self.inner_dim, self.out_dim, bias=True)]
-        )
-
-        if qk_norm is None:
-            self.norm_q = None
-            self.norm_k = None
-        elif qk_norm == "layer_norm":
-            self.norm_q = nn.LayerNorm(
-                dim_head, eps=eps, elementwise_affine=elementwise_affine
-            )
-            self.norm_k = nn.LayerNorm(
-                dim_head, eps=eps, elementwise_affine=elementwise_affine
-            )
-        else:
-            raise ValueError(f"unknown qk_norm: {qk_norm}")
-
-        self.attn = LocalAttention(
-            num_heads=self.heads,
-            head_size=dim_head,
-            num_kv_heads=self.heads,
-            softmax_scale=None,
-            causal=False,
-            supported_attention_backends=supported_attention_backends,
-        )
-
-    def forward(
-        self,
-        hidden_states: torch.Tensor,
-        encoder_hidden_states: torch.Tensor,
-        image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
-        kv_cache: Optional[GlmImageLayerKVCache] = None,
-    ) -> Tuple[torch.Tensor, torch.Tensor]:
-        dtype = encoder_hidden_states.dtype
-
-        batch_size, text_seq_length, embed_dim = encoder_hidden_states.shape
-        batch_size, image_seq_length, embed_dim = hidden_states.shape
-        hidden_states = torch.cat([encoder_hidden_states, hidden_states], dim=1)
-
-        # 1. QKV projections
-        query, _ = self.to_q(hidden_states)
-        key, _ = self.to_k(hidden_states)
-        value, _ = self.to_v(hidden_states)
-
-        query = query.unflatten(2, (self.heads, -1))
-        key = key.unflatten(2, (self.heads, -1))
-        value = value.unflatten(2, (self.heads, -1))
-
-        # 2. QK normalization
-        if self.norm_q is not None:
-            query = self.norm_q(query).to(dtype=dtype)
-        if self.norm_k is not None:
-            key = self.norm_k(key).to(dtype=dtype)
-
-        # 3. Rotational positional embeddings applied to latent stream
-        if image_rotary_emb is not None:
-            cos, sin = image_rotary_emb
-
-            query[:, text_seq_length:, :, :] = _apply_rotary_emb(
-                query[:, text_seq_length:, :, :], cos, sin, is_neox_style=True
-            )
-            key[:, text_seq_length:, :, :] = _apply_rotary_emb(
-                key[:, text_seq_length:, :, :], cos, sin, is_neox_style=True
-            )
-
-        # 4. KV Cache handling
-        if kv_cache is not None:
-            if kv_cache.mode == "write":
-                kv_cache.store(key, value)
-            elif kv_cache.mode == "read":
-                # Prepend cached condition k/v along seq (dim=1).
-                k_cache, v_cache = kv_cache.get()
-                key = torch.cat([k_cache, key], dim=1) if k_cache is not None else key
-                value = (
-                    torch.cat([v_cache, value], dim=1) if v_cache is not None else value
-                )
-            elif kv_cache.mode == "skip":
-                pass
-
-        hidden_states = self.attn(query, key, value)
-        hidden_states = hidden_states.flatten(2, 3)
-        hidden_states = hidden_states.to(query.dtype)
-
-        # 6. Output projection
-        hidden_states, _ = self.to_out[0](hidden_states)
-
-        encoder_hidden_states, hidden_states = hidden_states.split(
-            [text_seq_length, hidden_states.size(1) - text_seq_length], dim=1
-        )
-        return hidden_states, encoder_hidden_states
-
-
-# =============================================================================
-# Transformer Block
-# =============================================================================
-class GlmImageTransformerBlock(nn.Module):
-
-    def __init__(
-        self,
-        dim: int = 2560,
-        num_attention_heads: int = 64,
-        attention_head_dim: int = 40,
-        time_embed_dim: int = 512,
-        supported_attention_backends: set[AttentionBackendEnum] | None = None,
-        prefix: str = "",
-    ) -> None:
-        super().__init__()
-
-        # 1. Attention
-        self.norm1 = GlmImageAdaLayerNormZero(time_embed_dim, dim)
-
-        self.attn1 = GlmImageAttention(
-            query_dim=dim,
-            heads=num_attention_heads,
-            dim_head=attention_head_dim,
-            out_dim=dim,
-            bias=True,
-            qk_norm="layer_norm",
-            elementwise_affine=False,
-            eps=1e-5,
-            supported_attention_backends=supported_attention_backends,
-            prefix=f"{prefix}.attn1",
-        )
-
-        # 2. Feedforward with fused ScaleResidualLayerNorm
-        self.norm2 = ScaleResidualLayerNormScaleShift(
-            dim, norm_type="layer", eps=1e-5, elementwise_affine=False
-        )
-        self.norm2_context = ScaleResidualLayerNormScaleShift(
-            dim, norm_type="layer", eps=1e-5, elementwise_affine=False
-        )
-        self.ff = MLP(input_dim=dim, mlp_hidden_dim=dim * 4, output_dim=dim, act_type="gelu_pytorch_tanh")
-
-    def forward(
-        self,
-        hidden_states: torch.Tensor,
-        encoder_hidden_states: torch.Tensor,
-        temb: Optional[torch.Tensor] = None,
-        image_rotary_emb: Optional[
-            Union[
-                Tuple[torch.Tensor, torch.Tensor],
-                List[Tuple[torch.Tensor, torch.Tensor]],
-            ]
-        ] = None,
-        attention_kwargs: Optional[Dict[str, Any]] = None,
-        kv_cache: Optional[GlmImageLayerKVCache] = None,
-    ) -> Tuple[torch.Tensor, torch.Tensor]:
-        # 1. Timestep conditioning
-        (
-            norm_hidden_states,
-            gate_msa,
-            shift_mlp,
-            scale_mlp,
-            gate_mlp,
-            norm_encoder_hidden_states,
-            c_gate_msa,
-            c_shift_mlp,
-            c_scale_mlp,
-            c_gate_mlp,
-        ) = self.norm1(hidden_states, encoder_hidden_states, temb)
-
-        # 2. Attention
-        if attention_kwargs is None:
-            attention_kwargs = {}
-
-        attn_hidden_states, attn_encoder_hidden_states = self.attn1(
-            hidden_states=norm_hidden_states,
-            encoder_hidden_states=norm_encoder_hidden_states,
-            image_rotary_emb=image_rotary_emb,
-            kv_cache=kv_cache,
-            **attention_kwargs,
-        )
-
-        # 3. Feedforward (fused residual + norm + scale/shift)
-        norm_hidden_states, hidden_states = self.norm2(
-            hidden_states,
-            attn_hidden_states,
-            gate_msa.unsqueeze(1),
-            shift_mlp.unsqueeze(1),
-            scale_mlp.unsqueeze(1),
-        )
-        norm_encoder_hidden_states, encoder_hidden_states = self.norm2_context(
-            encoder_hidden_states,
-            attn_encoder_hidden_states,
-            c_gate_msa.unsqueeze(1),
-            c_shift_mlp.unsqueeze(1),
-            c_scale_mlp.unsqueeze(1),
-        )
-
-        ff_output = self.ff(norm_hidden_states)
-        ff_output_context = self.ff(norm_encoder_hidden_states)
-        hidden_states = hidden_states + ff_output * gate_mlp.unsqueeze(1)
-        encoder_hidden_states = (
-            encoder_hidden_states + ff_output_context * c_gate_mlp.unsqueeze(1)
-        )
-
-        return hidden_states, encoder_hidden_states
-
-
-# =============================================================================
-# Rotary Positional Embedding
-# =============================================================================
-class GlmImageRotaryPosEmbed(nn.Module):
-
-    def __init__(self, dim: int, patch_size: int, theta: float = 10000.0) -> None:
-        super().__init__()
-        self.dim = dim
-        self.patch_size = patch_size
-        self.theta = theta
-        self._cache_key: tuple | None = None
-        self._cache_value: tuple[torch.Tensor, torch.Tensor] | None = None
-
-    def forward(self, hidden_states: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
-        _, _, raw_h, raw_w = hidden_states.shape
-        height = raw_h // self.patch_size
-        width = raw_w // self.patch_size
-        device = hidden_states.device
-
-        cache_key = (height, width, device.type,
-                     device.index if device.index is not None else -1)
-        if self._cache_key == cache_key and self._cache_value is not None:
-            return self._cache_value
-
-        dim_h, dim_w = self.dim // 2, self.dim // 2
-        h_inv_freq = 1.0 / (
-            self.theta
-            ** (
-                torch.arange(0, dim_h, 2, dtype=torch.float32, device=device)[
-                    : (dim_h // 2)
-                ]
-                / dim_h
-            )
-        )
-        w_inv_freq = 1.0 / (
-            self.theta
-            ** (
-                torch.arange(0, dim_w, 2, dtype=torch.float32, device=device)[
-                    : (dim_w // 2)
-                ]
-                / dim_w
-            )
-        )
-        h_seq = torch.arange(height, device=device)
-        w_seq = torch.arange(width, device=device)
-        freqs_h = torch.outer(h_seq, h_inv_freq).unsqueeze(1).expand(height, width, -1)
-        freqs_w = torch.outer(w_seq, w_inv_freq).unsqueeze(0).expand(height, width, -1)
-
-        freqs = torch.cat([freqs_h, freqs_w], dim=-1).reshape(height * width, -1)
-        result = (freqs.cos(), freqs.sin())
-        self._cache_key = cache_key
-        self._cache_value = result
-        return result
-
-
-# =============================================================================
-# Final AdaLayerNorm
-# =============================================================================
-class GlmImageAdaLayerNormContinuous(nn.Module):
-
-    def __init__(
-        self,
-        embedding_dim: int,
-        conditioning_embedding_dim: int,
-        elementwise_affine: bool = True,
-        eps: float = 1e-5,
-        bias: bool = True,
-        norm_type: str = "layer_norm",
-    ):
-        super().__init__()
-        self.linear = nn.Linear(
-            conditioning_embedding_dim, embedding_dim * 2, bias=bias
-        )
-        if norm_type == "layer_norm":
-            self.norm = nn.LayerNorm(embedding_dim, eps, elementwise_affine, bias)
-        elif norm_type == "rms_norm":
-            self.norm = nn.RMSNorm(embedding_dim, eps, elementwise_affine)
-        else:
-            raise ValueError(f"unknown norm_type {norm_type}")
-
-    def forward(
-        self, x: torch.Tensor, conditioning_embedding: torch.Tensor
-    ) -> torch.Tensor:
-        emb = self.linear(conditioning_embedding.to(x.dtype))
-        scale, shift = torch.chunk(emb, 2, dim=1)
-        x = self.norm(x) * (1 + scale)[:, None, :] + shift[:, None, :]
-        return x
-
-
-# =============================================================================
-# Main Model
-# =============================================================================
-class GlmImageTransformer2DModel(BaseDiT):
-
-    _fsdp_shard_conditions = GlmImageDiTConfig().arch_config._fsdp_shard_conditions
-    _compile_conditions = GlmImageDiTConfig().arch_config._compile_conditions
-    _supported_attention_backends = (
-        AttentionBackendEnum.FLASH_ATTN,
-        AttentionBackendEnum.TORCH_SDPA,
-    )
-
-    param_names_mapping = GlmImageDiTConfig().arch_config.param_names_mapping
-    reverse_param_names_mapping = {}
-    lora_param_names_mapping = {}
-
-    def __init__(
-        self,
-        config: GlmImageDiTConfig,
-        hf_config: dict[str, Any],
-    ):
-        super().__init__(config=config, hf_config=hf_config)
-
-        arch_config = config.arch_config
-
-        self.in_channels = arch_config.in_channels
-        self.out_channels = arch_config.out_channels
-        self.patch_size = arch_config.patch_size
-        self.num_layers = arch_config.num_layers
-        self.attention_head_dim = arch_config.attention_head_dim
-        self.num_attention_heads = arch_config.num_attention_heads
-        self.text_embed_dim = arch_config.text_embed_dim
-        self.time_embed_dim = arch_config.time_embed_dim
-
-        # GlmImage uses 2 additional SDXL-like conditions - target_size, crop_coords
-        # Each of these are sincos embeddings of shape 2 * condition_dim
-        pooled_projection_dim = 2 * 2 * arch_config.condition_dim
-        inner_dim = arch_config.num_attention_heads * arch_config.attention_head_dim
-
-        self.hidden_size = inner_dim
-        self.num_channels_latents = arch_config.out_channels
-
-        # 1. RoPE
-        self.rotary_emb = GlmImageRotaryPosEmbed(
-            arch_config.attention_head_dim, arch_config.patch_size, theta=10000.0
-        )
-
-        # 2. Patch & Text-timestep embedding
-        self.image_projector = GlmImageImageProjector(
-            arch_config.in_channels, inner_dim, arch_config.patch_size
-        )
-        self.glyph_projector = MLP(
-            input_dim=arch_config.text_embed_dim,
-            mlp_hidden_dim=inner_dim,
-            output_dim=inner_dim,
-            act_type="gelu",
-        )
-        self.prior_token_embedding = nn.Embedding(
-            arch_config.prior_vq_quantizer_codebook_size, inner_dim
-        )
-        self.prior_projector = MLP(
-            input_dim=inner_dim,
-            mlp_hidden_dim=inner_dim,
-            output_dim=inner_dim,
-            act_type="silu",
-        )
-
-        self.time_condition_embed = GlmImageCombinedTimestepSizeEmbeddings(
-            embedding_dim=arch_config.time_embed_dim,
-            condition_dim=arch_config.condition_dim,
-            pooled_projection_dim=pooled_projection_dim,
-            timesteps_dim=arch_config.time_embed_dim,
-        )
-
-        # 3. Transformer blocks
-        self.transformer_blocks = nn.ModuleList(
-            [
-                GlmImageTransformerBlock(
-                    inner_dim,
-                    arch_config.num_attention_heads,
-                    arch_config.attention_head_dim,
-                    arch_config.time_embed_dim,
-                    supported_attention_backends=self._supported_attention_backends,
-                    prefix=f"transformer_blocks.{i}",
-                )
-                for i in range(arch_config.num_layers)
-            ]
-        )
-
-        # 4. Output projection
-        self.norm_out = GlmImageAdaLayerNormContinuous(
-            inner_dim, arch_config.time_embed_dim, elementwise_affine=False
-        )
-        self.proj_out = nn.Linear(
-            inner_dim,
-            arch_config.patch_size * arch_config.patch_size * arch_config.out_channels,
-            bias=True,
-        )
-
-        self.gradient_checkpointing = False
-
-        self.__post_init__()
-
-    def forward(
-        self,
-        hidden_states: torch.Tensor,
-        encoder_hidden_states: torch.Tensor,
-        prior_token_id: torch.Tensor,
-        prior_token_drop: torch.Tensor,
-        timestep: torch.LongTensor,
-        target_size: torch.Tensor,
-        crop_coords: torch.Tensor,
-        attention_kwargs: Optional[Dict[str, Any]] = None,
-        kv_caches: Optional[GlmImageKVCache] = None,
-        kv_caches_mode: Optional[str] = None,
-        freqs_cis: Optional[
-            Union[
-                Tuple[torch.Tensor, torch.Tensor],
-                List[Tuple[torch.Tensor, torch.Tensor]],
-            ]
-        ] = None,
-        guidance: torch.Tensor = None,
-        **kwargs,
-    ) -> torch.Tensor:
-        if kv_caches is not None:
-            kv_caches.set_mode(kv_caches_mode)
-
-        batch_size, num_channels, height, width = hidden_states.shape
-
-        if isinstance(encoder_hidden_states, list):
-            encoder_hidden_states = encoder_hidden_states[0]
-
-        # 1. RoPE
-        image_rotary_emb = freqs_cis
-        if image_rotary_emb is None:
-            image_rotary_emb = self.rotary_emb(hidden_states)
-
-        # 2. Patch & Timestep embeddings
-        p = self.patch_size
-        post_patch_height = height // p
-        post_patch_width = width // p
-
-        hidden_states = self.image_projector(hidden_states)
-        encoder_hidden_states = self.glyph_projector(encoder_hidden_states)
-        prior_embedding = self.prior_token_embedding(prior_token_id)
-        # Zero dropped priors by multiply: boolean indexing + .any() syncs each step.
-        keep = (~prior_token_drop).to(device=prior_embedding.device, dtype=prior_embedding.dtype)
-        while keep.dim() < prior_embedding.dim():
-            keep = keep.unsqueeze(-1)
-        prior_embedding = prior_embedding * keep
-        prior_hidden_states = self.prior_projector(prior_embedding)
-        hidden_states = hidden_states + prior_hidden_states
-
-        temb = self.time_condition_embed(
-            timestep, target_size, crop_coords, hidden_states.dtype
-        )
-        temb = F.silu(temb)
-
-        # 3. Transformer blocks
-        for idx, block in enumerate(self.transformer_blocks):
-            hidden_states, encoder_hidden_states = block(
-                hidden_states,
-                encoder_hidden_states,
-                temb,
-                image_rotary_emb,
-                attention_kwargs,
-                kv_cache=kv_caches[idx] if kv_caches is not None else None,
-            )
-
-        # 4. Output norm & projection
-        hidden_states = self.norm_out(hidden_states, temb)
-        hidden_states = self.proj_out(hidden_states)
-
-        # 5. Unpatchify
-        hidden_states = hidden_states.reshape(
-            batch_size, post_patch_height, post_patch_width, -1, p, p
-        )
-        output = hidden_states.permute(0, 3, 1, 4, 2, 5).flatten(4, 5).flatten(2, 3)
-
-        return output.float()
-
-
-EntryClass = GlmImageTransformer2DModel

fastvideo/models/dits/ltx2.py

@@ -72,13 +72,8 @@ def _rms_norm_dispatch(
     """Use QuACK RMSNorm only for LTX-2 refine stage, else Torch RMSNorm."""
     if _is_ltx2_refine_stage():
         return _quack_rmsnorm(x, weight=weight, eps=eps)
-    # Official LTX-2 runs the DiT natively in bf16 with no autocast, so its
-    # rms_norm outputs stay bf16 on every torch version. torch 2.12 added
-    # rms_norm to autocast's float32 cast policy, which under our
-    # autocast(bfloat16) forward returns float32 instead. Restore the input
-    # dtype to match the official numerics (and torch <= 2.11 behavior).
     return torch.nn.functional.rms_norm(
-        x, (x.shape[-1], ), weight=weight, eps=eps).to(x.dtype)
+        x, (x.shape[-1], ), weight=weight, eps=eps)
 
 
 class StageAwareRMSNorm(nn.RMSNorm):

fastvideo/models/dits/matrixgame2/action_module.py

@@ -16,8 +16,6 @@ from fastvideo.layers.rotary_embedding import (
 )
 from fastvideo.platforms import AttentionBackendEnum
 
-from .utils import retain_kv_with_sink
-
 
 DISABLE_COMPILE = False
 flex_attention = torch.compile(
@@ -102,7 +100,6 @@ def _update_kv_cache_and_attend(
     start_frame: int,
     num_frame_per_block: int,
     local_attn_size: int,
-    sink_size: int,
     *,
     use_k_for_num_tokens: bool = False,
     store_first_only: bool = False,
@@ -136,6 +133,7 @@ def _update_kv_cache_and_attend(
         k.shape[1] if use_k_for_num_tokens else q.shape[1]
     ) == num_frame_per_block
 
+    sink_size = 0
     max_attention_size = local_attn_size
     sink_tokens = sink_size * 1
     kv_cache_size = kv_cache["k"].shape[1]
@@ -208,22 +206,16 @@ def _update_kv_cache_and_attend(
 
     # Store new k, v in cache
     if store_first_only:
-        kv_cache["k"][:, local_start_index:local_end_index] = k[
-            : kv_cache["k"].shape[0]
-        ]
-        kv_cache["v"][:, local_start_index:local_end_index] = v[
-            : kv_cache["v"].shape[0]
-        ]
+        kv_cache["k"][:, local_start_index:local_end_index] = k[:1]
+        kv_cache["v"][:, local_start_index:local_end_index] = v[:1]
     else:
         kv_cache["k"][:, local_start_index:local_end_index] = k
         kv_cache["v"][:, local_start_index:local_end_index] = v
 
-    cached_k, cached_v = retain_kv_with_sink(
-        kv_cache["k"][:, :local_end_index],
-        kv_cache["v"][:, :local_end_index],
-        min(local_end_index, max_attention_size),
-        sink_tokens,
-    )
+    # Retrieve from cache and perform attention
+    cache_start = max(0, local_end_index - max_attention_size)
+    cached_k = kv_cache["k"][:, cache_start:local_end_index]
+    cached_v = kv_cache["v"][:, cache_start:local_end_index]
 
     if repeat_factor is not None:
         cached_k = cached_k.repeat(repeat_factor, 1, 1, 1)
@@ -269,7 +261,6 @@ class ActionModule(nn.Module):
         enable_mouse=True,
         enable_keyboard=True,
         local_attn_size=6,
-        sink_size=0,
         blocks: list | None = None,
     ):
         super().__init__()
@@ -283,7 +274,6 @@ class ActionModule(nn.Module):
         )
         blocks = blocks if blocks is not None else []
         self.local_attn_size = local_attn_size
-        self.sink_size = sink_size
         self.enable_mouse = enable_mouse
         self.enable_keyboard = enable_keyboard
 
@@ -575,7 +565,6 @@ class ActionModule(nn.Module):
                     start_frame,
                     num_frame_per_block,
                     self.local_attn_size,
-                    self.sink_size,
                     use_k_for_num_tokens=False,
                 )
         else:
@@ -700,7 +689,6 @@ class ActionModule(nn.Module):
                         start_frame,
                         num_frame_per_block,
                         self.local_attn_size,
-                        self.sink_size,
                         use_k_for_num_tokens=True,
                         store_first_only=True,
                         repeat_factor=S,
@@ -731,7 +719,6 @@ class ActionModule(nn.Module):
                         start_frame,
                         num_frame_per_block,
                         self.local_attn_size,
-                        self.sink_size,
                         use_k_for_num_tokens=True,
                     )
             else:

fastvideo/models/dits/matrixgame2/causal_model.py

@@ -46,7 +46,6 @@ from fastvideo.platforms import AttentionBackendEnum, current_platform
 
 from .action_module import ActionModule
 from .model import MatrixGame2CrossAttention
-from .utils import retain_kv_with_sink
 
 logger = init_logger(__name__)
 
@@ -346,12 +345,9 @@ class CausalMatrixGame2SelfAttention(nn.Module):
                 kv_cache["k"][:, local_start_index:local_end_index] = stored_key
                 kv_cache["v"][:, local_start_index:local_end_index] = v
 
-            k_for_attn, v_for_attn = retain_kv_with_sink(
-                kv_cache["k"][:, :local_end_index],
-                kv_cache["v"][:, :local_end_index],
-                min(local_end_index, max_attention_size),
-                sink_tokens,
-            )
+            kv_start = max(0, local_end_index - max_attention_size)
+            k_for_attn = kv_cache["k"][:, kv_start:local_end_index]
+            v_for_attn = kv_cache["v"][:, kv_start:local_end_index]
 
             if relativistic:
                 window_len, query_lo, _ = relativistic_window_offsets(
@@ -422,7 +418,6 @@ class CausalMatrixGame2TransformerBlock(nn.Module):
         self.hidden_dim = dim
         self.num_attention_heads = num_heads
         self.local_attn_size = local_attn_size
-        self.sink_size = sink_size
         dim_head = dim // num_heads
         if qk_norm == "rms_norm":
             self.norm_q = RMSNorm(dim_head, eps=eps)
@@ -479,7 +474,6 @@ class CausalMatrixGame2TransformerBlock(nn.Module):
                 ),
                 patch_size=action_config["patch_size"],
                 local_attn_size=local_attn_size,
-                sink_size=sink_size,
                 qk_norm=action_config["qk_norm"],
                 qkv_bias=action_config["qkv_bias"],
                 vae_time_compression_ratio=action_config[
@@ -653,8 +647,6 @@ class CausalMatrixGame2WanModel(BaseDiT):
         self.num_channels_latents = config.num_channels_latents
         self.patch_size = config.patch_size
 
-        # Long tuning controls the causal window from YAML; consume it here
-        # like Wan so train wrappers do not need runtime patching.
         arch_cfg = getattr(config, "arch_config", None)
         self.local_attn_size = (
             getattr(
@@ -670,12 +662,6 @@ class CausalMatrixGame2WanModel(BaseDiT):
             if arch_cfg
             else getattr(config, "sink_size", 0)
         )
-        if self.sink_size < 0:
-            raise ValueError("sink_size must be non-negative")
-        if self.local_attn_size != -1 and self.sink_size >= self.local_attn_size:
-            raise ValueError(
-                "sink_size must be smaller than local_attn_size for "
-                "MatrixGame2 causal attention")
         self.rope_cache_policy = (
             getattr(arch_cfg, "rope_cache_policy",
                     getattr(config, "rope_cache_policy", "absolute"))
@@ -763,52 +749,6 @@ class CausalMatrixGame2WanModel(BaseDiT):
 
         self.__post_init__()
 
-    def set_causal_attention_window(
-        self,
-        *,
-        local_attn_size: int | None = None,
-        sink_size: int | None = None,
-    ) -> None:
-        local_attn_size = (
-            self.local_attn_size if local_attn_size is None else int(local_attn_size)
-        )
-        sink_size = self.sink_size if sink_size is None else int(sink_size)
-        if sink_size < 0:
-            raise ValueError("sink_size must be non-negative")
-        if local_attn_size != -1 and sink_size >= local_attn_size:
-            raise ValueError(
-                "sink_size must be smaller than local_attn_size for "
-                "MatrixGame2 causal attention")
-
-        self.local_attn_size = local_attn_size
-        self.sink_size = sink_size
-        self.block_mask = None
-        self.block_mask_keyboard = None
-        self.block_mask_mouse = None
-
-        for block in self.blocks:
-            if hasattr(block, "local_attn_size"):
-                block.local_attn_size = local_attn_size
-            if hasattr(block, "sink_size"):
-                block.sink_size = sink_size
-
-            attn1 = getattr(block, "attn1", None)
-            if attn1 is not None:
-                if hasattr(attn1, "local_attn_size"):
-                    attn1.local_attn_size = local_attn_size
-                if hasattr(attn1, "sink_size"):
-                    attn1.sink_size = sink_size
-
-            action_model = getattr(block, "action_model", None)
-            if action_model is not None:
-                if hasattr(action_model, "local_attn_size"):
-                    action_model.local_attn_size = local_attn_size
-                if hasattr(action_model, "sink_size"):
-                    action_model.sink_size = sink_size
-
-    def set_sink_size(self, sink_size: int) -> None:
-        self.set_causal_attention_window(sink_size=sink_size)
-
     @staticmethod
     def _prepare_blockwise_causal_attn_mask(
         device: torch.device | str,
@@ -816,7 +756,6 @@ class CausalMatrixGame2WanModel(BaseDiT):
         frame_seqlen: int = 880,
         num_frame_per_block: int = 1,
         local_attn_size: int = -1,
-        sink_size: int = 0,
     ) -> BlockMask:
         total_length = num_frames * frame_seqlen
         padded_length = math.ceil(total_length / 128) * 128 - total_length
@@ -843,10 +782,7 @@ class CausalMatrixGame2WanModel(BaseDiT):
             else:
                 return (
                     (kv_idx < ends[q_idx])
-                    & (
-                        (kv_idx < sink_size * frame_seqlen)
-                        | (kv_idx >= (ends[q_idx] - local_attn_size * frame_seqlen))
-                    )
+                    & (kv_idx >= (ends[q_idx] - local_attn_size * frame_seqlen))
                 ) | (q_idx == kv_idx)
 
         block_mask = create_block_mask(
@@ -860,9 +796,8 @@ class CausalMatrixGame2WanModel(BaseDiT):
         )
 
         if not dist.is_initialized() or dist.get_rank() == 0:
-            logger.info(
-                "cache a block wise causal mask with block size of %s frames",
-                num_frame_per_block,
+            print(
+                f" cache a block wise causal mask with block size of {num_frame_per_block} frames"
             )
 
         return block_mask
@@ -874,7 +809,6 @@ class CausalMatrixGame2WanModel(BaseDiT):
         frame_seqlen: int = 880,
         num_frame_per_block: int = 1,
         local_attn_size: int = -1,
-        sink_size: int = 0,
     ) -> BlockMask:
         total_length2 = num_frames * frame_seqlen
         padded_length2 = math.ceil(total_length2 / 32) * 32 - total_length2
@@ -900,7 +834,7 @@ class CausalMatrixGame2WanModel(BaseDiT):
             else:
                 return (
                     (kv_idx < ends2[q_idx])
-                    & ((kv_idx < sink_size) | (kv_idx >= (ends2[q_idx] - local_attn_size)))
+                    & (kv_idx >= (ends2[q_idx] - local_attn_size))
                 ) | (q_idx == kv_idx)
 
         block_mask2 = create_block_mask(
@@ -914,9 +848,8 @@ class CausalMatrixGame2WanModel(BaseDiT):
         )
 
         if not dist.is_initialized() or dist.get_rank() == 0:
-            logger.info(
-                "cache a block wise causal mask for keyboard with block size of %s frames",
-                num_frame_per_block,
+            print(
+                f" cache a block wise causal mask for keyboard with block size of {num_frame_per_block} frames"
             )
 
         return block_mask2
@@ -928,7 +861,6 @@ class CausalMatrixGame2WanModel(BaseDiT):
         frame_seqlen: int = 1,
         num_frame_per_block: int = 1,
         local_attn_size: int = -1,
-        sink_size: int = 0,
     ) -> BlockMask:
         total_length2 = num_frames * frame_seqlen
         padded_length2 = math.ceil(total_length2 / 32) * 32 - total_length2
@@ -954,7 +886,7 @@ class CausalMatrixGame2WanModel(BaseDiT):
             else:
                 return (
                     (kv_idx < ends2[q_idx])
-                    & ((kv_idx < sink_size) | (kv_idx >= (ends2[q_idx] - local_attn_size)))
+                    & (kv_idx >= (ends2[q_idx] - local_attn_size))
                 ) | (q_idx == kv_idx)
 
         block_mask2 = create_block_mask(
@@ -968,9 +900,8 @@ class CausalMatrixGame2WanModel(BaseDiT):
         )
 
         if not dist.is_initialized() or dist.get_rank() == 0:
-            logger.info(
-                "cache a block wise causal mask for action with block size of %s frames",
-                num_frame_per_block,
+            print(
+                f" cache a block wise causal mask for action with block size of {num_frame_per_block} frames"
             )
 
         return block_mask2
@@ -1132,7 +1063,6 @@ class CausalMatrixGame2WanModel(BaseDiT):
             frame_seqlen=post_patch_height * post_patch_width,
             num_frame_per_block=self.num_frame_per_block,
             local_attn_size=self.local_attn_size,
-            sink_size=self.sink_size,
         )
         if self.use_rope_keyboard:
             block_mask_keyboard = self._prepare_blockwise_causal_attn_mask_action(
@@ -1141,7 +1071,6 @@ class CausalMatrixGame2WanModel(BaseDiT):
                 frame_seqlen=1,
                 num_frame_per_block=self.num_frame_per_block,
                 local_attn_size=self.local_attn_size,
-                sink_size=self.sink_size,
             )
         else:
             block_mask_keyboard = self._prepare_blockwise_causal_attn_mask_keyboard(
@@ -1150,7 +1079,6 @@ class CausalMatrixGame2WanModel(BaseDiT):
                 frame_seqlen=post_patch_height * post_patch_width,
                 num_frame_per_block=self.num_frame_per_block,
                 local_attn_size=self.local_attn_size,
-                sink_size=self.sink_size,
             )
         block_mask_mouse = self._prepare_blockwise_causal_attn_mask_action(
             device=hidden_states.device,
@@ -1158,7 +1086,6 @@ class CausalMatrixGame2WanModel(BaseDiT):
             frame_seqlen=1,
             num_frame_per_block=self.num_frame_per_block,
             local_attn_size=self.local_attn_size,
-            sink_size=self.sink_size,
         )
         if kv_cache is None:
             kv_cache = [None] * len(self.blocks)
@@ -1312,7 +1239,6 @@ class CausalMatrixGame2WanModel(BaseDiT):
                 frame_seqlen=post_patch_height * post_patch_width,
                 num_frame_per_block=self.num_frame_per_block,
                 local_attn_size=self.local_attn_size,
-                sink_size=self.sink_size,
             )
         if self.block_mask_keyboard is None:
             if self.use_rope_keyboard:
@@ -1322,7 +1248,6 @@ class CausalMatrixGame2WanModel(BaseDiT):
                     frame_seqlen=1,
                     num_frame_per_block=self.num_frame_per_block,
                     local_attn_size=self.local_attn_size,
-                    sink_size=self.sink_size,
                 )
             else:
                 self.block_mask_keyboard = self._prepare_blockwise_causal_attn_mask_keyboard(
@@ -1331,7 +1256,6 @@ class CausalMatrixGame2WanModel(BaseDiT):
                     frame_seqlen=post_patch_height * post_patch_width,
                     num_frame_per_block=self.num_frame_per_block,
                     local_attn_size=self.local_attn_size,
-                    sink_size=self.sink_size,
                 )
         if self.block_mask_mouse is None:
             self.block_mask_mouse = self._prepare_blockwise_causal_attn_mask_action(
@@ -1340,7 +1264,6 @@ class CausalMatrixGame2WanModel(BaseDiT):
                 frame_seqlen=1,
                 num_frame_per_block=self.num_frame_per_block,
                 local_attn_size=self.local_attn_size,
-                sink_size=self.sink_size,
             )
 
         hidden_states = self.patch_embedding(hidden_states)

fastvideo/models/dits/matrixgame2/model.py

@@ -452,7 +452,7 @@ class MatrixGame2WanModel(BaseDiT):
             and len(encoder_hidden_states_image) > 0
         ):
             encoder_hidden_states_image = encoder_hidden_states_image[0]
-        elif not isinstance(encoder_hidden_states_image, torch.Tensor):
+        else:
             encoder_hidden_states_image = None
 
         batch_size, num_channels, num_frames, height, width = (

fastvideo/models/dits/matrixgame2/utils.py

@@ -4,18 +4,12 @@ import asyncio
 import os
 import random
 
-import numpy as np
 import torch
 from PIL import Image
 
 from fastvideo.distributed.parallel_state import get_local_torch_device
 from fastvideo.utils import logger
 
-try:
-    import cv2
-except ImportError:
-    cv2 = None
-
 
 CAM_VALUE = 0.1
 CAMERA_MAP = {
@@ -40,256 +34,6 @@ KEYBOARD_MAP_7 = {  # templerun_distilled_model: still/w/s/left/right/a/d
     "d": [0, 0, 0, 0, 0, 0, 1],  # d
 }
 KEYBOARD_MAP = KEYBOARD_MAP_4  # Default for backward compatibility
-SOLARIS_MOVEMENT_KEY_INDICES = {
-    "W": 11,
-    "S": 12,
-    "A": 13,
-    "D": 14,
-}
-
-
-def retain_kv_with_sink(
-    k: torch.Tensor,
-    v: torch.Tensor,
-    target_len: int,
-    sink_size: int,
-) -> tuple[torch.Tensor, torch.Tensor]:
-    if k.shape[1] <= target_len or sink_size <= 0:
-        return k[:, -target_len:], v[:, -target_len:]
-
-    sink_len = min(int(sink_size), target_len, k.shape[1])
-    tail_len = target_len - sink_len
-    sink_k = k[:, :sink_len]
-    sink_v = v[:, :sink_len]
-    if tail_len <= 0:
-        return sink_k, sink_v
-
-    tail_k = k[:, sink_len:][:, -tail_len:]
-    tail_v = v[:, sink_len:][:, -tail_len:]
-    return torch.cat([sink_k, tail_k], dim=1), torch.cat([sink_v, tail_v], dim=1)
-
-
-def _require_cv2() -> bool:
-    if cv2 is None:
-        logger.warning(
-            "OpenCV is not available; skipping MatrixGame2 validation overlay."
-        )
-        return False
-    return True
-
-
-def _matrixgame2_overlay_keys_from_keyboard(
-    keyboard_frame: np.ndarray | torch.Tensor,
-) -> dict[str, bool]:
-    vec = np.asarray(keyboard_frame, dtype=np.float32).reshape(-1)
-    dim = vec.shape[0]
-
-    if dim >= 23:
-        return {
-            key: bool(vec[idx] > 0.5)
-            for key, idx in SOLARIS_MOVEMENT_KEY_INDICES.items()
-        }
-    if dim == 7:
-        return {
-            "W": bool(vec[1] > 0.5),
-            "S": bool(vec[2] > 0.5),
-            "A": bool(vec[5] > 0.5),
-            "D": bool(vec[6] > 0.5),
-        }
-    if dim >= 6:
-        return {
-            "W": bool(vec[0] > 0.5),
-            "S": bool(vec[1] > 0.5),
-            "A": bool(vec[2] > 0.5),
-            "D": bool(vec[3] > 0.5),
-        }
-    if dim == 4:
-        return {
-            "W": bool(vec[0] > 0.5),
-            "S": bool(vec[1] > 0.5),
-            "A": bool(vec[2] > 0.5),
-            "D": bool(vec[3] > 0.5),
-        }
-    if dim == 2:
-        return {
-            "W": bool(vec[0] > 0.5),
-            "S": bool(vec[1] > 0.5),
-            "A": False,
-            "D": False,
-        }
-    return {"W": False, "S": False, "A": False, "D": False}
-
-
-def draw_rounded_rectangle(
-    image: np.ndarray,
-    top_left: tuple[int, int],
-    bottom_right: tuple[int, int],
-    color: tuple[int, int, int],
-    radius: int = 10,
-    alpha: float = 0.5,
-) -> None:
-    if not _require_cv2():
-        return
-
-    overlay = image.copy()
-    x1, y1 = top_left
-    x2, y2 = bottom_right
-
-    cv2.rectangle(overlay, (x1 + radius, y1), (x2 - radius, y2), color, -1)
-    cv2.rectangle(overlay, (x1, y1 + radius), (x2, y2 - radius), color, -1)
-    cv2.ellipse(overlay, (x1 + radius, y1 + radius), (radius, radius), 180, 0, 90, color, -1)
-    cv2.ellipse(overlay, (x2 - radius, y1 + radius), (radius, radius), 270, 0, 90, color, -1)
-    cv2.ellipse(overlay, (x1 + radius, y2 - radius), (radius, radius), 90, 0, 90, color, -1)
-    cv2.ellipse(overlay, (x2 - radius, y2 - radius), (radius, radius), 0, 0, 90, color, -1)
-    cv2.addWeighted(overlay, alpha, image, 1 - alpha, 0, image)
-
-
-def draw_keys_on_frame(
-    frame: np.ndarray,
-    keys: dict[str, bool],
-    key_size: tuple[int, int] = (30, 30),
-    top_margin: int = 15,
-) -> None:
-    if not _require_cv2():
-        return
-
-    left_margin = 15
-    gap = 3
-    key_positions = {
-        "W": (left_margin + key_size[0] + gap, top_margin),
-        "A": (left_margin, top_margin + key_size[1] + gap),
-        "S": (left_margin + key_size[0] + gap, top_margin + key_size[1] + gap),
-        "D": (
-            left_margin + (key_size[0] + gap) * 2,
-            top_margin + key_size[1] + gap,
-        ),
-    }
-
-    for key, (x, y) in key_positions.items():
-        pressed = keys.get(key, False)
-        color = (0, 255, 0) if pressed else (200, 200, 200)
-        alpha = 0.8 if pressed else 0.5
-        draw_rounded_rectangle(
-            frame,
-            (x, y),
-            (x + key_size[0], y + key_size[1]),
-            color,
-            radius=5,
-            alpha=alpha,
-        )
-        text_size = cv2.getTextSize(
-            key, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 1
-        )[0]
-        text_x = x + (key_size[0] - text_size[0]) // 2
-        text_y = y + (key_size[1] + text_size[1]) // 2
-        cv2.putText(
-            frame,
-            key,
-            (text_x, text_y),
-            cv2.FONT_HERSHEY_SIMPLEX,
-            0.5,
-            (0, 0, 0),
-            1,
-        )
-
-
-def draw_mouse_on_frame(
-    frame: np.ndarray,
-    yaw: float,
-    pitch: float,
-    top_margin: int = 15,
-) -> None:
-    if not _require_cv2():
-        return
-
-    _, width, _ = frame.shape
-    right_margin = 15
-    crosshair_radius = 25
-    crosshair_x = width - right_margin - crosshair_radius
-    crosshair_y = top_margin + crosshair_radius
-
-    dx = int(yaw * crosshair_radius * 8)
-    dy = int(-pitch * crosshair_radius * 8)
-    max_arrow = crosshair_radius - 5
-    dx = max(-max_arrow, min(max_arrow, dx))
-    dy = max(-max_arrow, min(max_arrow, dy))
-
-    cv2.circle(frame, (crosshair_x, crosshair_y), crosshair_radius, (50, 50, 50), -1)
-    cv2.circle(frame, (crosshair_x, crosshair_y), crosshair_radius, (200, 200, 200), 1)
-    cv2.line(
-        frame,
-        (crosshair_x - crosshair_radius + 5, crosshair_y),
-        (crosshair_x + crosshair_radius - 5, crosshair_y),
-        (100, 100, 100),
-        1,
-    )
-    cv2.line(
-        frame,
-        (crosshair_x, crosshair_y - crosshair_radius + 5),
-        (crosshair_x, crosshair_y + crosshair_radius - 5),
-        (100, 100, 100),
-        1,
-    )
-
-    if abs(dx) > 1 or abs(dy) > 1:
-        cv2.arrowedLine(
-            frame,
-            (crosshair_x, crosshair_y),
-            (crosshair_x + dx, crosshair_y + dy),
-            (0, 255, 0),
-            2,
-            tipLength=0.3,
-        )
-
-
-def overlay_validation_actions_on_frames(
-    frames: list[np.ndarray],
-    keyboard_cond: np.ndarray | torch.Tensor | None = None,
-    mouse_cond: np.ndarray | torch.Tensor | None = None,
-) -> list[np.ndarray]:
-    if (keyboard_cond is None and mouse_cond is None) or not _require_cv2():
-        return frames
-
-    if keyboard_cond is not None and torch.is_tensor(keyboard_cond):
-        keyboard_cond = keyboard_cond.detach().cpu().float().numpy()
-    if mouse_cond is not None and torch.is_tensor(mouse_cond):
-        mouse_cond = mouse_cond.detach().cpu().float().numpy()
-
-    if keyboard_cond is not None:
-        keyboard_cond = np.asarray(keyboard_cond, dtype=np.float32)
-        if keyboard_cond.ndim == 3 and keyboard_cond.shape[0] == 1:
-            keyboard_cond = keyboard_cond[0]
-    if mouse_cond is not None:
-        mouse_cond = np.asarray(mouse_cond, dtype=np.float32)
-        if mouse_cond.ndim == 3 and mouse_cond.shape[0] == 1:
-            mouse_cond = mouse_cond[0]
-
-    processed_frames: list[np.ndarray] = []
-    for frame_idx, frame in enumerate(frames):
-        frame = np.ascontiguousarray(frame.copy())
-        if keyboard_cond is not None and frame_idx < len(keyboard_cond):
-            draw_keys_on_frame(
-                frame,
-                _matrixgame2_overlay_keys_from_keyboard(keyboard_cond[frame_idx]),
-            )
-        if mouse_cond is not None and frame_idx < len(mouse_cond):
-            # MG action convention: mouse[0]=pitch, mouse[1]=yaw.
-            pitch = float(mouse_cond[frame_idx, 0])
-            yaw = float(mouse_cond[frame_idx, 1])
-            draw_mouse_on_frame(frame, yaw=yaw, pitch=pitch)
-        processed_frames.append(frame)
-
-    return processed_frames
-
-
-def scale_keyboard_condition(
-    keyboard_condition: torch.Tensor,
-    scale: float,
-) -> torch.Tensor:
-    if float(scale) == 1.0:
-        return keyboard_condition
-    return keyboard_condition * float(scale)
-
 
 def expand_action_to_frames(action: dict, num_frames: int) -> tuple[torch.Tensor, torch.Tensor]:
     result = {}

fastvideo/models/dits/wanvideo.py

@@ -1,6 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
 
-import copy
 import math
 from typing import Any
 
@@ -60,11 +59,6 @@ class WanTimeTextImageEmbedding(nn.Module):
         time_freq_dim: int,
         text_embed_dim: int,
         image_embed_dim: int | None = None,
-        *,
-        r_embedder: bool = False,
-        r_embedder_fusion: str = "additive",
-        r_embedder_gate_value: float = 0.25,
-        r_embedder_deltatime_type: str = "r",
     ):
         super().__init__()
 
@@ -83,57 +77,14 @@ class WanTimeTextImageEmbedding(nn.Module):
         if image_embed_dim is not None:
             self.image_embedder = WanImageEmbedding(image_embed_dim, dim)
 
-        # AnyFlow dual-timestep support. When r_embedder is False the forward
-        # path bypasses delta_embedder entirely and the output is byte-identical
-        # to the legacy single-timestep implementation.
-        self._r_embedder_enabled = bool(r_embedder)
-        self._r_embedder_fusion = r_embedder_fusion
-        self._r_embedder_deltatime_type = r_embedder_deltatime_type
-        if self._r_embedder_enabled:
-            if r_embedder_fusion not in ("additive", "gated"):
-                raise ValueError(
-                    "r_embedder_fusion must be one of {additive, gated}, "
-                    f"got {r_embedder_fusion!r}")
-            if r_embedder_deltatime_type not in ("r", "t-r"):
-                raise ValueError(
-                    "r_embedder_deltatime_type must be one of {r, t-r}, "
-                    f"got {r_embedder_deltatime_type!r}")
-            # Deep-copy preserves identical initialization with time_embedder,
-            # matching AnyFlow reference setup_flowmap_model() behavior.
-            self.delta_embedder = copy.deepcopy(self.time_embedder)
-            # Non-persistent buffer — gate is a hyperparameter, not learned.
-            self.register_buffer(
-                "_r_embedder_gate",
-                torch.tensor(float(r_embedder_gate_value)),
-                persistent=False,
-            )
-        else:
-            self.delta_embedder = None
-
     def forward(
         self,
         timestep: torch.Tensor,
         encoder_hidden_states: torch.Tensor,
         encoder_hidden_states_image: torch.Tensor | None = None,
         timestep_seq_len: int | None = None,
-        r_timestep: torch.Tensor | None = None,
     ):
         temb = self.time_embedder(timestep, timestep_seq_len)
-
-        if self._r_embedder_enabled and r_timestep is not None:
-            assert self.delta_embedder is not None
-            if self._r_embedder_deltatime_type == "r":
-                delta_input = r_timestep
-            else:
-                delta_input = timestep - r_timestep
-            delta_emb = self.delta_embedder(delta_input, timestep_seq_len)
-            gate = self._r_embedder_gate
-            if self._r_embedder_fusion == "gated":
-                temb = (1.0 - gate) * temb + gate * delta_emb
-            else:
-                # Additive (additional channel, no convex blend).
-                temb = temb + gate * delta_emb
-
         timestep_proj = self.time_modulation(temb)
 
         if self.text_embedder is not None:
@@ -644,10 +595,6 @@ class WanTransformer3DModel(BaseDiT):
             time_freq_dim=config.freq_dim,
             text_embed_dim=config.text_dim,
             image_embed_dim=config.image_dim,
-            r_embedder=config.r_embedder,
-            r_embedder_fusion=config.r_embedder_fusion,
-            r_embedder_gate_value=config.r_embedder_gate_value,
-            r_embedder_deltatime_type=config.r_embedder_deltatime_type,
         )
 
         # 3. Transformer blocks
@@ -689,7 +636,6 @@ class WanTransformer3DModel(BaseDiT):
                 encoder_hidden_states_image: torch.Tensor | list[torch.Tensor]
                 | None = None,
                 guidance=None,
-                r_timestep: torch.Tensor | None = None,
                 **kwargs) -> torch.Tensor:
         orig_dtype = hidden_states.dtype
         if encoder_hidden_states is not None and not isinstance(encoder_hidden_states, torch.Tensor):
@@ -738,17 +684,8 @@ class WanTransformer3DModel(BaseDiT):
         else:
             ts_seq_len = None
 
-        # AnyFlow dual-timestep — match timestep's flattening so embedder
-        # sees aligned shapes.
-        if r_timestep is not None and r_timestep.dim() == 2:
-            r_timestep = r_timestep.flatten()
-
         temb, timestep_proj, encoder_hidden_states, encoder_hidden_states_image = self.condition_embedder(
-            timestep,
-            encoder_hidden_states,
-            encoder_hidden_states_image,
-            timestep_seq_len=ts_seq_len,
-            r_timestep=r_timestep)
+            timestep, encoder_hidden_states, encoder_hidden_states_image, timestep_seq_len=ts_seq_len)
         if ts_seq_len is not None:
             # batch_size, seq_len, 6, inner_dim
             timestep_proj = timestep_proj.unflatten(2, (6, -1))

fastvideo/models/encoders/glm_image_ar_loader.py

@@ -1,64 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-"""Sole HF-import boundary for GLM-Image's AR encoder (lazy-wrapper exception E001)."""
-from __future__ import annotations
-
-from typing import Any
-
-import torch
-import torch.nn as nn
-
-from fastvideo.logger import init_logger
-
-logger = init_logger(__name__)
-
-
-class GlmImageARLoader(nn.Module):
-
-    def __init__(self, model_path: str, processor_path: str | None = None,
-                 *, torch_dtype: torch.dtype = torch.bfloat16,
-                 trust_remote_code: bool = True) -> None:
-        super().__init__()
-        from transformers import (AutoProcessor,
-                                  GlmImageForConditionalGeneration)
-        logger.info("Loading GLM-Image AR encoder from %s", model_path)
-        self._model = GlmImageForConditionalGeneration.from_pretrained(
-            model_path,
-            torch_dtype=torch_dtype,
-            trust_remote_code=trust_remote_code,
-        )
-        if processor_path is not None:
-            logger.info("Loading GLM-Image processor from %s", processor_path)
-            self.processor = AutoProcessor.from_pretrained(
-                processor_path, trust_remote_code=trust_remote_code)
-        else:
-            self.processor = None
-
-    @torch.no_grad()
-    def generate(self, *args: Any, **kwargs: Any) -> torch.Tensor:
-        return self._model.generate(*args, **kwargs)
-
-    @torch.no_grad()
-    def get_image_features(self, pixel_values: torch.Tensor,
-                           image_grid_thw: torch.Tensor) -> Any:
-        return self._model.get_image_features(pixel_values, image_grid_thw)
-
-    @torch.no_grad()
-    def get_image_tokens(self, image_embeds: torch.Tensor,
-                         image_grid_thw: torch.Tensor) -> torch.Tensor:
-        return self._model.get_image_tokens(image_embeds, image_grid_thw)
-
-    @property
-    def config(self):  # type: ignore[no-untyped-def]
-        return self._model.config
-
-    @property
-    def generation_config(self):  # type: ignore[no-untyped-def]
-        return self._model.generation_config
-
-    def to(self, *args, **kwargs):  # type: ignore[override]
-        self._model = self._model.to(*args, **kwargs)
-        return super().to(*args, **kwargs)
-
-    def eval(self):  # type: ignore[override]
-        self._model = self._model.eval()
-        return super().eval()

fastvideo/models/loader/benchmarks/benchmark_weight_loading_comparison.py

@@ -42,7 +42,7 @@ def load_independent(files: list[str], device: str):
     """Before-PR behavior: every rank reads every tensor from disk to GPU."""
     for st_file in files:
         with safe_open(st_file, framework="pt", device=device) as f:
-            for name in f.keys():  # noqa: SIM118
+            for name in f:
                 param = f.get_tensor(name)
                 yield name, param
 
@@ -54,7 +54,7 @@ def load_broadcast(files: list[str], device: str, node_group,
     handles = []
     for st_file in files:
         with safe_open(st_file, framework="pt", device=device) as f:
-            for name in f.keys():  # noqa: SIM118
+            for name in f:
                 if local_rank == 0:
                     param = f.get_tensor(name)
                 else:

fastvideo/models/loader/component_loader.py

@@ -95,8 +95,6 @@ class ComponentLoader(ABC):
             "image_processor": (ImageProcessorLoader, "transformers"),
             "feature_extractor": (ImageProcessorLoader, "transformers"),
             "image_encoder": (ImageEncoderLoader, "transformers"),
-            "vision_language_encoder": (VisionLanguageEncoderLoader, "transformers"),
-            "processor": (ProcessorLoader, "transformers"),
             "upsampler": (UpsamplerLoader, "diffusers"),
             "upsampler_2": (UpsamplerLoader, "diffusers"),
             # Stable Audio's `StableAudioMultiConditioner` bundles T5 +
@@ -525,39 +523,6 @@ class ImageEncoderLoader(TextEncoderLoader):
         )
 
 
-class VisionLanguageEncoderLoader(ComponentLoader):
-    """Loader for vision-language autoregressive encoders."""
-
-    def load(self, model_path: str, fastvideo_args: FastVideoArgs):
-        from fastvideo.distributed.parallel_state import get_local_torch_device
-        from fastvideo.models.encoders.glm_image_ar_loader import (
-            GlmImageARLoader)
-
-        logger.info("Loading vision-language encoder from %s", model_path)
-        target_device = get_local_torch_device()
-        loader = GlmImageARLoader(
-            model_path,
-            torch_dtype=torch.bfloat16,
-            trust_remote_code=fastvideo_args.trust_remote_code,
-        ).to(target_device).eval()
-        return loader
-
-
-class ProcessorLoader(ComponentLoader):
-    """Loader for HF processors that pair with vision-language encoders."""
-
-    def load(self, model_path: str, fastvideo_args: FastVideoArgs):
-        from transformers import AutoProcessor
-
-        logger.info("Loading processor from %s", model_path)
-        processor = AutoProcessor.from_pretrained(
-            model_path,
-            trust_remote_code=fastvideo_args.trust_remote_code,
-        )
-        logger.info("Loaded processor: %s", processor.__class__.__name__)
-        return processor
-
-
 class ImageProcessorLoader(ComponentLoader):
     """Loader for image processor."""
 

fastvideo/models/registry.py

@@ -61,11 +61,6 @@ _IMAGE_TO_VIDEO_DIT_MODELS = {
     "MatrixGame3WanModel": ("dits", "matrixgame3", "MatrixGame3WanModel"),
 }
 
-# Text-to-image DiT models (2D image generation)
-_TEXT_TO_IMAGE_DIT_MODELS = {
-    "GlmImageTransformer2DModel": ("dits", "glm_image", "GlmImageTransformer2DModel"),
-}
-
 _TEXT_ENCODER_MODELS = {
     "CLIPTextModel": ("encoders", "clip", "CLIPTextModel"),
     "CLIPTextModelWithProjection":
@@ -139,7 +134,6 @@ _UPSAMPLERS = {
 _LEGACY_FAST_VIDEO_MODELS = {
     **_TEXT_TO_VIDEO_DIT_MODELS,
     **_IMAGE_TO_VIDEO_DIT_MODELS,
-    **_TEXT_TO_IMAGE_DIT_MODELS,
     **_TEXT_ENCODER_MODELS,
     **_IMAGE_ENCODER_MODELS,
     **_VAE_MODELS,

fastvideo/models/schedulers/scheduling_flow_map_euler_discrete.py

@@ -1,202 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-"""Flow-map any-step Euler scheduler for AnyFlow.
-
-The model predicts the *average* velocity ``u_θ(x_t, t, r)`` from time
-``t`` back to time ``r``, so one Euler step is
-
-    x_r = x_t - ((t - r) / num_train_timesteps) * u_θ(x_t, t, r)
-
-regardless of how far apart ``t`` and ``r`` are. The scheduler also
-provides the AnyFlow training-time helpers ``apply_shift`` (flow-matching
-shift transform) and ``get_train_weight`` (per-timestep loss weight,
-including ``beta08``).
-
-Standalone — does not depend on diffusers' ConfigMixin/SchedulerMixin.
-"""
-
-from __future__ import annotations
-
-from typing import Literal
-
-import torch
-
-from fastvideo.models.schedulers.base import BaseScheduler
-
-
-WeightType = Literal["uniform", "gaussian", "beta08"]
-
-
-class FlowMapEulerDiscreteScheduler(BaseScheduler):
-    """Minimal flow-map scheduler.
-
-    Parameters
-    ----------
-    num_train_timesteps:
-        Discretization granularity for training. ``t`` is expressed in
-        absolute units in ``[0, num_train_timesteps]``.
-    shift:
-        Flow-matching shift parameter (Wan video default: ``5.0``). Set
-        to ``1.0`` for an identity shift.
-    """
-
-    order: int = 1
-
-    def __init__(
-        self,
-        *,
-        num_train_timesteps: int = 1000,
-        shift: float = 1.0,
-    ) -> None:
-        self.num_train_timesteps = int(num_train_timesteps)
-        self.shift = float(shift)
-        self.timesteps: torch.Tensor = torch.empty(0)
-        self.sigmas: torch.Tensor = torch.empty(0)
-        super().__init__()
-
-    # ------------------------------------------------------------------
-    # BaseScheduler abstract surface
-
-    def set_shift(self, shift: float) -> None:
-        self.shift = float(shift)
-
-    def scale_model_input(
-        self,
-        sample: torch.Tensor,
-        timestep: int | None = None,
-    ) -> torch.Tensor:
-        # Flow-matching has no per-step input scaling; pass through.
-        del timestep
-        return sample
-
-    # ------------------------------------------------------------------
-    # Public helpers used by the AnyFlow pretrain method.
-
-    def apply_shift(
-        self,
-        t: torch.Tensor,
-        *,
-        shift: float | None = None,
-    ) -> torch.Tensor:
-        """Apply the flow-matching shift: ``t' = s * t / (1 + (s - 1) * t)``.
-
-        Operates in the normalized ``[0, 1]`` domain — callers should pass
-        ``t / num_train_timesteps`` (or sample ``t`` directly from
-        ``[0, 1]``).
-        """
-        s = self.shift if shift is None else float(shift)
-        if s == 1.0:
-            return t
-        return s * t / (1.0 + (s - 1.0) * t)
-
-    def get_train_weight(
-        self,
-        t: torch.Tensor,
-        *,
-        weight_type: WeightType = "beta08",
-    ) -> torch.Tensor:
-        """Per-timestep training weight, renormalized so the total weight
-        mass equals ``num_train_timesteps`` (matching AnyFlow reference's
-        ``scheduling_flowmap_euler_discrete.py``).
-
-        ``beta08``: ``w(t) = t * sqrt(1 - t)`` (in normalized t-space).
-        """
-        # Auto-detect domain: if t was given in absolute units, normalize.
-        t_f = t.float()
-        max_val = t_f.max() if t_f.numel() > 0 else torch.tensor(0.0)
-        if max_val > 1.0 + 1e-6:
-            t_norm = t_f / self.num_train_timesteps
-        else:
-            t_norm = t_f
-        t_norm = t_norm.clamp(min=0.0, max=1.0)
-
-        if weight_type == "uniform":
-            w = torch.ones_like(t_norm)
-        elif weight_type == "gaussian":
-            w = torch.exp(-0.5 * ((t_norm - 0.5) / 0.2) ** 2)
-        elif weight_type == "beta08":
-            w = t_norm.pow(1.0) * (1.0 - t_norm).clamp_min(0.0).pow(0.5)
-        else:
-            raise ValueError(f"Unknown weight_type: {weight_type!r}")
-
-        denom = w.sum().clamp_min(1e-8)
-        return w * (float(self.num_train_timesteps) / denom)
-
-    # ------------------------------------------------------------------
-
-    def set_timesteps(
-        self,
-        *,
-        num_inference_steps: int,
-        device: torch.device | str = "cpu",
-        custom_timesteps: list[float] | torch.Tensor | None = None,
-    ) -> None:
-        """Build a descending timestep schedule ending at 0.
-
-        With ``num_inference_steps=N`` the schedule has ``N + 1`` entries
-        ``[T_max, ..., 0]`` so a rollout consumes ``N`` Euler steps.
-
-        ``custom_timesteps`` overrides the linspace+shift schedule with
-        a pinned list (in absolute train-timestep units), useful for the
-        AnyFlow paper's hand-tuned ``[999, 937, 833, 624, 0]`` schedule.
-        """
-        if num_inference_steps <= 0:
-            raise ValueError(
-                "num_inference_steps must be positive, "
-                f"got {num_inference_steps}")
-        device = torch.device(device)
-
-        if custom_timesteps is not None:
-            ts = torch.as_tensor(
-                custom_timesteps, dtype=torch.float32, device=device)
-            if ts.ndim != 1:
-                raise ValueError(
-                    "custom_timesteps must be 1-D, got shape "
-                    f"{tuple(ts.shape)}")
-            if not torch.all(ts[:-1] >= ts[1:]):
-                raise ValueError(
-                    "custom_timesteps must be descending (largest first)")
-        else:
-            ts_norm = torch.linspace(
-                1.0, 0.0, num_inference_steps + 1, device=device)
-            ts_norm = self.apply_shift(ts_norm)
-            ts = ts_norm * self.num_train_timesteps
-
-        self.timesteps = ts
-        self.sigmas = ts / self.num_train_timesteps
-
-    def step(
-        self,
-        model_output: torch.Tensor,
-        *,
-        sample: torch.Tensor,
-        timestep: torch.Tensor,
-        r_timestep: torch.Tensor,
-    ) -> torch.Tensor:
-        """One Euler step from ``t`` to ``r``.
-
-        ``model_output`` is the average-velocity prediction
-        ``u_θ(x_t, t, r)``. Both ``timestep`` and ``r_timestep`` are in
-        absolute train-timestep units (``[0, num_train_timesteps]``).
-        """
-        t = timestep.to(sample.device, dtype=sample.dtype)
-        r = r_timestep.to(sample.device, dtype=sample.dtype)
-        dt_norm = (t - r) / float(self.num_train_timesteps)
-        # Broadcast dt over channel/spatial dims.
-        view: list[int] = [-1] + [1] * (sample.ndim - 1)
-        return sample - dt_norm.view(*view) * model_output
-
-    def add_noise(
-        self,
-        original_samples: torch.Tensor,
-        noise: torch.Tensor,
-        timestep: torch.Tensor,
-    ) -> torch.Tensor:
-        """Linear flow-matching interpolation: ``x_t = (1 - σ) * x_0 + σ * ε``,
-        where ``σ = t / num_train_timesteps``.
-        """
-        sigma = (timestep.to(original_samples.device,
-                             dtype=original_samples.dtype)
-                 / float(self.num_train_timesteps))
-        view: list[int] = [-1] + [1] * (original_samples.ndim - 1)
-        sigma = sigma.view(*view)
-        return (1.0 - sigma) * original_samples + sigma * noise

fastvideo/models/vision_utils.py

@@ -1,9 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
-import io
 import os
 import tempfile
-import time
 from collections.abc import Callable
 from typing import Any
 from urllib.parse import unquote, urlparse
@@ -18,10 +16,6 @@ import torch.nn.functional as F
 import torchvision.transforms.functional as TF
 from packaging import version
 
-from fastvideo.logger import init_logger
-
-logger = init_logger(__name__)
-
 if version.parse(version.parse(
         PIL.__version__).base_version) >= version.parse("9.1.0"):
     PIL_INTERPOLATION = {
@@ -95,39 +89,6 @@ def normalize(images: np.ndarray | torch.Tensor) -> np.ndarray | torch.Tensor:
     return 2.0 * images - 1.0
 
 
-def _fetch_image_bytes(url: str,
-                       attempts: int = 3,
-                       timeout: float = 30.0) -> bytes:
-    """Download ``url`` fully, retrying transient network errors.
-
-    Reads the whole body via ``response.content`` instead of handing PIL a
-    ``.raw`` stream: a connection dropped mid-body (e.g. an HF-CDN
-    ``IncompleteRead``) then fails here, where it can be retried, instead of
-    surfacing as a truncated image inside PIL.
-    """
-    for attempt in range(attempts):
-        try:
-            response = requests.get(url, timeout=timeout)
-            response.raise_for_status()
-            return response.content
-        except requests.RequestException as e:
-            status = getattr(getattr(e, "response", None), "status_code",
-                             None)
-            # Permanent client errors (4xx except 408/429) won't heal on
-            # retry — fail fast.
-            if (status is not None and 400 <= status < 500
-                    and status not in (408, 429)):
-                raise
-            if attempt == attempts - 1:
-                raise
-            backoff = 2**attempt
-            logger.warning(
-                "Failed to download image %s (attempt %d/%d): %s. "
-                "Retrying in %ds.", url, attempt + 1, attempts, e, backoff)
-            time.sleep(backoff)
-    raise AssertionError("unreachable")
-
-
 # adapted from diffusers.utils import load_image
 def load_image(
     image: str | PIL.Image.Image,
@@ -149,7 +110,7 @@ def load_image(
     """
     if isinstance(image, str):
         if image.startswith("http://") or image.startswith("https://"):
-            image = PIL.Image.open(io.BytesIO(_fetch_image_bytes(image)))
+            image = PIL.Image.open(requests.get(image, stream=True).raw)
         elif os.path.isfile(image):
             image = PIL.Image.open(image)
         else:

fastvideo/performance_dashboard/service.py

@@ -86,32 +86,20 @@ def record_metadata(record: Record) -> Record:
     }
 
 
-def _cohort_metadata_value(value: Any) -> Any:
-    if value is None:
-        return ""
-    if isinstance(value, str) and not value.strip():
-        return ""
-    return value
-
-
-def _cohort_key_value(value: Any) -> str:
-    return str(_cohort_metadata_value(value))
-
-
 def record_comparison_metadata(record: Record) -> Record:
-    return {key: _cohort_metadata_value(record.get(key)) for key in COMPARISON_COHORT_KEYS}
+    return {key: record.get(key) or "" for key in COMPARISON_COHORT_KEYS}
 
 
 def comparison_cohort_key(record: Record) -> CohortKey:
     return (
         str(record.get("model_id") or "unknown"),
         str(record.get("gpu_type") or "unknown"),
-        _cohort_key_value(record.get("workload_id")),
-        _cohort_key_value(record.get("variant_id")),
-        _cohort_key_value(record.get("benchmark_version")),
-        _cohort_key_value(record.get("recipe_fingerprint")),
-        _cohort_key_value(record.get("hardware_profile_id")),
-        _cohort_key_value(record.get("software_profile_id")),
+        str(record.get("workload_id") or ""),
+        str(record.get("variant_id") or ""),
+        str(record.get("benchmark_version") or ""),
+        str(record.get("recipe_fingerprint") or ""),
+        str(record.get("hardware_profile_id") or ""),
+        str(record.get("software_profile_id") or ""),
     )
 
 
@@ -158,10 +146,9 @@ def build_latest_summary(records: list[Record],
             continue
 
         latest = latest_candidates[-1]
-        latest_index = next(index for index, record in enumerate(group) if record is latest)
         baseline_pool = [
-            record for record in group[:latest_index]
-            if record.get("success", True) and is_baseline_eligible_record(record)
+            record for record in group
+            if record is not latest and record.get("success", True) and is_baseline_eligible_record(record)
         ]
         baseline_records = baseline_pool[-baseline_window:]
         metric_policies = resolve_metric_policies(latest.get("regression_thresholds"))

fastvideo/pipelines/basic/flux/__init__.py

@@ -1 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0

fastvideo/pipelines/basic/flux/flux_pipeline.py

@@ -1,74 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-
-from __future__ import annotations
-
-from fastvideo.fastvideo_args import FastVideoArgs
-from fastvideo.pipelines.composed_pipeline_base import ComposedPipelineBase
-from fastvideo.pipelines.stages.flux_stages import (
-    FluxConditioningStage,
-    FluxDecodingStage,
-    FluxDenoisingStage,
-    FluxInputValidationStage,
-    FluxLatentPreparationStage,
-    FluxTimestepPreparationStage,
-)
-from fastvideo.pipelines.stages.text_encoding import TextEncodingStage
-
-
-class FluxPipeline(ComposedPipelineBase):
-    """FLUX.1-dev T2I (Diffusers module layout, packed latents, embedded guidance)."""
-
-    _required_config_modules = [
-        "scheduler",
-        "transformer",
-        "vae",
-        "text_encoder",
-        "text_encoder_2",
-        "tokenizer",
-        "tokenizer_2",
-    ]
-
-    def create_pipeline_stages(self, fastvideo_args: FastVideoArgs) -> None:
-        self.add_stage(stage_name="input_validation_stage", stage=FluxInputValidationStage())
-
-        self.add_stage(
-            stage_name="text_encoding_stage",
-            stage=TextEncodingStage(
-                text_encoders=[
-                    self.get_module("text_encoder"),
-                    self.get_module("text_encoder_2"),
-                ],
-                tokenizers=[
-                    self.get_module("tokenizer"),
-                    self.get_module("tokenizer_2"),
-                ],
-            ),
-        )
-
-        self.add_stage(stage_name="flux_conditioning_stage", stage=FluxConditioningStage())
-
-        self.add_stage(
-            stage_name="timestep_preparation_stage",
-            stage=FluxTimestepPreparationStage(scheduler=self.get_module("scheduler")),
-        )
-
-        self.add_stage(
-            stage_name="latent_preparation_stage",
-            stage=FluxLatentPreparationStage(scheduler=self.get_module("scheduler")),
-        )
-
-        self.add_stage(
-            stage_name="denoising_stage",
-            stage=FluxDenoisingStage(
-                transformer=self.get_module("transformer"),
-                scheduler=self.get_module("scheduler"),
-            ),
-        )
-
-        self.add_stage(
-            stage_name="decoding_stage",
-            stage=FluxDecodingStage(vae=self.get_module("vae")),
-        )
-
-
-EntryClass = FluxPipeline

fastvideo/pipelines/basic/glm_image/__init__.py

@@ -1,7 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-"""GLM-Image pipeline package."""
-
-from fastvideo.pipelines.basic.glm_image.glm_image_pipeline import (
-    GlmImagePipeline, )
-
-__all__ = ["GlmImagePipeline"]

fastvideo/pipelines/basic/glm_image/glm_image_pipeline.py

@@ -1,82 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-from fastvideo.fastvideo_args import FastVideoArgs
-from fastvideo.logger import init_logger
-from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import (
-    FlowMatchEulerDiscreteScheduler, )
-from fastvideo.pipelines import ComposedPipelineBase, LoRAPipeline
-from fastvideo.pipelines.basic.glm_image.stages import (
-    GlmImageBeforeDenoisingStage,
-    GlmImageConditionEncodingStage,
-    GlmImageDecodingStage,
-    GlmImageDenoisingStage,
-)
-from fastvideo.pipelines.stages import InputValidationStage
-
-logger = init_logger(__name__)
-
-
-class GlmImagePipeline(LoRAPipeline, ComposedPipelineBase):
-    pipeline_name = "GlmImagePipeline"
-
-    _required_config_modules = [
-        "text_encoder",
-        "tokenizer",
-        "vae",
-        "transformer",
-        "scheduler",
-        "vision_language_encoder",
-        "processor",
-    ]
-
-    _optional_config_modules: list[str] = []
-
-    def initialize_pipeline(self, fastvideo_args: FastVideoArgs) -> None:
-        self.modules["scheduler"] = FlowMatchEulerDiscreteScheduler(shift=1.0)
-
-    def create_pipeline_stages(self, fastvideo_args: FastVideoArgs) -> None:
-        self.add_stage(
-            stage_name="input_validation_stage",
-            stage=InputValidationStage(),
-        )
-
-        self.add_stage(
-            stage_name="glm_image_before_denoising_stage",
-            stage=GlmImageBeforeDenoisingStage(
-                vae=self.get_module("vae"),
-                text_encoder=self.get_module("text_encoder"),
-                tokenizer=self.get_module("tokenizer"),
-                processor=self.get_module("processor"),
-                transformer=self.get_module("transformer"),
-                scheduler=self.get_module("scheduler"),
-                vision_language_encoder=self.get_module("vision_language_encoder"),
-            ),
-        )
-
-        self.add_stage(
-            stage_name="glm_image_condition_encoding_stage",
-            stage=GlmImageConditionEncodingStage(
-                vae=self.get_module("vae"),
-                transformer=self.get_module("transformer"),
-            ),
-        )
-
-        self.add_stage(
-            stage_name="denoising_stage",
-            stage=GlmImageDenoisingStage(
-                transformer=self.get_module("transformer"),
-                scheduler=self.get_module("scheduler"),
-                vae=self.get_module("vae"),
-                pipeline=self,
-            ),
-        )
-
-        self.add_stage(
-            stage_name="decoding_stage",
-            stage=GlmImageDecodingStage(
-                vae=self.get_module("vae"),
-                pipeline=self,
-            ),
-        )
-
-
-EntryClass = GlmImagePipeline

fastvideo/pipelines/basic/glm_image/stages/__init__.py

@@ -1,14 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-"""GLM-Image pipeline stages."""
-
-from fastvideo.pipelines.basic.glm_image.stages.before_denoising import (GlmImageBeforeDenoisingStage)
-from fastvideo.pipelines.basic.glm_image.stages.condition_encoding import (GlmImageConditionEncodingStage)
-from fastvideo.pipelines.basic.glm_image.stages.decoding import (GlmImageDecodingStage)
-from fastvideo.pipelines.basic.glm_image.stages.denoising import (GlmImageDenoisingStage)
-
-__all__ = [
-    "GlmImageBeforeDenoisingStage",
-    "GlmImageConditionEncodingStage",
-    "GlmImageDecodingStage",
-    "GlmImageDenoisingStage",
-]

fastvideo/pipelines/basic/glm_image/stages/before_denoising.py

@@ -1,269 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-from __future__ import annotations
-
-import re
-from math import sqrt
-
-import numpy as np
-import torch
-
-from fastvideo.distributed import get_local_torch_device
-from fastvideo.fastvideo_args import FastVideoArgs
-from fastvideo.logger import init_logger
-from fastvideo.pipelines.pipeline_batch_info import ForwardBatch
-from fastvideo.pipelines.stages.base import PipelineStage
-from fastvideo.pipelines.stages.validators import StageValidators as V
-from fastvideo.pipelines.stages.validators import VerificationResult
-
-logger = init_logger(__name__)
-
-
-def calculate_shift(
-    image_seq_len: int,
-    base_seq_len: int = 256,
-    base_shift: float = 0.25,
-    max_shift: float = 0.75,
-) -> float:
-    return (image_seq_len / base_seq_len)**0.5 * max_shift + base_shift
-
-
-def get_glyph_texts(prompt: str | list[str]) -> list[str] | list[list[str]]:
-    if isinstance(prompt, str):
-        prompts: list[str] = [prompt]
-        is_batch = False
-    else:
-        prompts = prompt
-        is_batch = True
-    out: list[list[str]] = []
-    for p in prompts:
-        out.append(
-            re.findall(r"'([^']*)'", p) + re.findall(r"“([^“”]*)”", p) + re.findall(r'"([^"]*)"', p) +
-            re.findall(r"「([^「」]*)」", p))
-    return out if is_batch else out[0]
-
-
-def compute_glyph_embeds(
-    prompts: list[str],
-    tokenizer,
-    text_encoder,
-    device: torch.device,
-    dtype: torch.dtype,
-    max_sequence_length: int = 2048,
-) -> torch.Tensor:
-    all_glyph_texts = get_glyph_texts(prompts)
-    all_glyph_embeds = []
-    for glyph_texts in all_glyph_texts:
-        if len(glyph_texts) == 0:
-            glyph_texts = [""]
-        input_ids = tokenizer(
-            glyph_texts,
-            max_length=max_sequence_length,
-            truncation=True,
-        ).input_ids
-        input_ids = [[tokenizer.pad_token_id] * ((len(input_ids) + 1) % 2) + ids for ids in input_ids]
-        max_length = max(len(ids) for ids in input_ids)
-        attention_mask = torch.tensor(
-            [[1] * len(ids) + [0] * (max_length - len(ids)) for ids in input_ids],
-            device=device,
-        )
-        input_ids_t = torch.tensor(
-            [ids + [tokenizer.pad_token_id] * (max_length - len(ids)) for ids in input_ids],
-            device=device,
-        )
-        outputs = text_encoder(input_ids_t, attention_mask=attention_mask)
-        glyph_embeds = outputs.last_hidden_state[attention_mask.bool()].unsqueeze(0)
-        all_glyph_embeds.append(glyph_embeds)
-
-    max_seq_len = max(emb.size(1) for emb in all_glyph_embeds)
-    padded = []
-    for emb in all_glyph_embeds:
-        if emb.size(1) < max_seq_len:
-            pad = torch.zeros(emb.size(0), max_seq_len - emb.size(1), emb.size(2), device=device, dtype=emb.dtype)
-            emb = torch.cat([pad, emb], dim=1)
-        padded.append(emb)
-    return torch.cat(padded, dim=0).to(device=device, dtype=dtype)
-
-
-def _grid_dims(height: int, width: int) -> tuple[int, int, int, int]:
-    th, tw = height // 32, width // 32
-    ratio = th / tw
-    pth = int(sqrt(ratio) * 16)
-    ptw = int(sqrt(1 / ratio) * 16)
-    return th, tw, pth, ptw
-
-
-def _upsample_d32_to_d16(tokens: torch.Tensor, th: int, tw: int) -> torch.Tensor:
-    tokens = tokens.view(1, 1, th, tw).float()
-    tokens = torch.nn.functional.interpolate(tokens, scale_factor=2, mode="nearest").long()
-    return tokens.view(1, -1)
-
-
-class GlmImageBeforeDenoisingStage(PipelineStage):
-
-    def __init__(self,
-                 vae,
-                 text_encoder,
-                 tokenizer,
-                 processor,
-                 transformer,
-                 scheduler,
-                 vision_language_encoder=None) -> None:
-        super().__init__()
-        self.vae = vae
-        self.text_encoder = text_encoder
-        self.tokenizer = tokenizer
-        self.processor = processor
-        self.transformer = transformer
-        self.scheduler = scheduler
-        if isinstance(vision_language_encoder, tuple):
-            self.vision_language_encoder, self.vl_processor = (vision_language_encoder[0], vision_language_encoder[1]
-                                                               or processor)
-        else:
-            self.vision_language_encoder = vision_language_encoder
-            self.vl_processor = processor
-
-    @torch.no_grad()
-    def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch:
-        device = get_local_torch_device()
-        dtype = torch.bfloat16
-        th, tw, pth, ptw = _grid_dims(batch.height, batch.width)
-
-        if batch.seed is not None:
-            torch.manual_seed(batch.seed)
-            if torch.cuda.is_available():
-                torch.cuda.manual_seed(batch.seed)
-
-        # 1-3. AR token generation. I2I prepends the condition image and uses a
-        # single-scale target grid; T2I is multi-scale.
-        is_t2i = batch.pil_image is None
-        if self.vision_language_encoder is not None:
-            content = [{"type": "text", "text": batch.prompt}]
-            if not is_t2i:
-                content.insert(0, {"type": "image", "image": batch.pil_image})
-            messages = [{"role": "user", "content": content}]
-            inputs = self.vl_processor.apply_chat_template(messages,
-                                                           tokenize=True,
-                                                           target_h=batch.height,
-                                                           target_w=batch.width,
-                                                           return_dict=True,
-                                                           return_tensors="pt").to(device)
-
-            if is_t2i:
-                up_h, up_w = th, tw
-                large_start, large_count = pth * ptw, th * tw
-                max_new = large_count + (pth * ptw) + 1
-            else:
-                # Condition grid(s) first, target grid last.
-                _, t_h, t_w = inputs["image_grid_thw"][-1].tolist()
-                up_h, up_w = int(t_h), int(t_w)
-                large_start, large_count = 0, up_h * up_w
-                max_new = large_count + 1
-
-            outputs = self.vision_language_encoder.generate(**inputs, max_new_tokens=max_new, do_sample=True)
-            gen_tokens = outputs[0][inputs.input_ids.shape[-1]:]
-            if gen_tokens.shape[0] >= large_start + large_count:
-                large_tokens = gen_tokens[large_start:large_start + large_count]
-            else:
-                available = gen_tokens[large_start:]
-                large_tokens = torch.zeros(large_count, dtype=gen_tokens.dtype, device=gen_tokens.device)
-                if available.shape[0] > 0:
-                    large_tokens[:min(available.shape[0], large_count)] = available[:large_count]
-                logger.warning("AR generated %d tokens, expected %d. Padding with zeros.", gen_tokens.shape[0],
-                               large_start + large_count)
-            batch.prior_token_id = _upsample_d32_to_d16(large_tokens, up_h, up_w)
-            batch.prior_token_drop = torch.zeros(batch.prior_token_id.shape, dtype=torch.bool, device=device)
-
-            if not is_t2i:
-                self._compute_source_prior_tokens(batch, inputs)
-        else:
-            num_prior_tokens = 4 * th * tw
-            logger.warning("No vision_language_encoder provided; using random dropped priors.")
-            batch.prior_token_id = torch.randint(0, 16384, (1, num_prior_tokens), device=device)
-            batch.prior_token_drop = torch.ones(batch.prior_token_id.shape, dtype=torch.bool, device=device)
-
-        # 4. Glyph T5 encoding.
-        prompts = [batch.prompt] if isinstance(batch.prompt, str) else list(batch.prompt)
-        prompt_embeds = compute_glyph_embeds(prompts, self.tokenizer, self.text_encoder, device, dtype)
-
-        # 5. CFG-side negative encoding.
-        if batch.do_classifier_free_guidance:
-            neg_prompts = [batch.negative_prompt or ""] * len(prompts)
-            neg_embeds = compute_glyph_embeds(neg_prompts, self.tokenizer, self.text_encoder, device, dtype)
-            L_pos, L_neg = prompt_embeds.shape[1], neg_embeds.shape[1]
-            max_L = max(L_pos, L_neg)
-            if L_pos < max_L:
-                pad = torch.zeros(prompt_embeds.shape[0],
-                                  max_L - L_pos,
-                                  prompt_embeds.shape[2],
-                                  device=device,
-                                  dtype=dtype)
-                prompt_embeds = torch.cat([pad, prompt_embeds], dim=1)
-            if L_neg < max_L:
-                pad = torch.zeros(neg_embeds.shape[0], max_L - L_neg, neg_embeds.shape[2], device=device, dtype=dtype)
-                neg_embeds = torch.cat([pad, neg_embeds], dim=1)
-            # Row 0 conditional (positive), row 1 unconditional (negative).
-            prompt_embeds = torch.cat([prompt_embeds, neg_embeds], dim=0)
-            att_pos = torch.ones((1, max_L), device=device)
-            att_neg = torch.ones((1, max_L), device=device)
-            if L_pos < max_L:
-                att_pos[:, :max_L - L_pos] = 0
-            if L_neg < max_L:
-                att_neg[:, :max_L - L_neg] = 0
-            attention_mask = torch.cat([att_pos, att_neg], dim=0)
-        else:
-            attention_mask = torch.ones((1, prompt_embeds.shape[1]), device=device)
-
-        batch.prompt_embeds = [prompt_embeds]
-        batch.attention_mask = attention_mask
-
-        # 6. Latents + dynamic flow shift.
-        if batch.seed is not None:
-            torch.manual_seed(batch.seed)
-            if torch.cuda.is_available():
-                torch.cuda.manual_seed(batch.seed)
-        batch.latents = torch.randn((1, 16, 1, batch.height // 8, batch.width // 8), device=device, dtype=dtype)
-
-        # Integer-cast linspace timesteps with resolution-dependent shift applied to
-        # sigmas only; the DiT is conditioned on the unshifted integer timesteps.
-        ntt = self.scheduler.config.num_train_timesteps
-        patch_size = self.transformer.patch_size
-        image_seq_len = ((batch.height // 8) * (batch.width // 8)) // (patch_size**2)
-        sched_timesteps = np.linspace(ntt, 1.0, batch.num_inference_steps + 1)[:-1].astype(np.int64).astype(np.float32)
-        sched_sigmas = sched_timesteps / ntt
-        self.scheduler.set_shift(calculate_shift(image_seq_len))
-        self.scheduler.set_timesteps(batch.num_inference_steps,
-                                     device=device,
-                                     sigmas=sched_sigmas.tolist(),
-                                     timesteps=sched_timesteps.tolist())
-        batch.timesteps = self.scheduler.timesteps
-        return batch
-
-    @torch.no_grad()
-    def _compute_source_prior_tokens(self, batch: ForwardBatch, inputs) -> None:
-        image_grid_thw = inputs["image_grid_thw"]
-        num_condition_images = image_grid_thw.shape[0] - 1
-        source_grids = image_grid_thw[:num_condition_images]
-        image_features = self.vision_language_encoder.get_image_features(inputs["pixel_values"], source_grids)
-        image_feature_parts = getattr(image_features, "pooler_output", image_features)
-        embed = torch.cat(image_feature_parts, dim=0)
-        src_ids_d32 = self.vision_language_encoder.get_image_tokens(embed, source_grids)
-        split_sizes = source_grids.prod(dim=-1).tolist()
-        upsampled = [
-            _upsample_d32_to_d16(ids, int(grid[1]), int(grid[2])).squeeze(0)
-            for ids, grid in zip(torch.split(src_ids_d32, split_sizes), source_grids, strict=False)
-        ]
-        src_grids_up = source_grids.clone()
-        src_grids_up[:, 1] *= 2
-        src_grids_up[:, 2] *= 2
-        batch.extra["glm_prior_token_image_ids"] = torch.cat(upsampled, dim=0)
-        batch.extra["glm_source_image_grid_thw"] = src_grids_up
-
-    def verify_input(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> VerificationResult:
-        result = VerificationResult()
-        result.add_check("prompt", batch.prompt, V.string_not_empty)
-        return result
-
-    def verify_output(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> VerificationResult:
-        result = VerificationResult()
-        result.add_check("prior_token_id", batch.prior_token_id, V.is_tensor)
-        return result

fastvideo/pipelines/basic/glm_image/stages/condition_encoding.py

@@ -1,67 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-from __future__ import annotations
-
-import torch
-
-from fastvideo.distributed import get_local_torch_device
-from fastvideo.fastvideo_args import FastVideoArgs
-from fastvideo.forward_context import set_forward_context
-from fastvideo.image_processor import ImageProcessor
-from fastvideo.models.dits.glm_image import GlmImageKVCache
-from fastvideo.pipelines.pipeline_batch_info import ForwardBatch
-from fastvideo.pipelines.stages.base import PipelineStage
-
-_CONDITION_MULTIPLE_OF = 16  # vae_scale_factor (8) * DiT patch_size (2)
-
-
-class GlmImageConditionEncodingStage(PipelineStage):
-
-    def __init__(self, vae, transformer) -> None:
-        super().__init__()
-        self.vae = vae
-        self.transformer = transformer
-        self.image_processor = ImageProcessor(vae_scale_factor=_CONDITION_MULTIPLE_OF)
-
-    @torch.no_grad()
-    def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch:
-        if batch.pil_image is None:
-            return batch
-
-        device = get_local_torch_device()
-        dtype = torch.bfloat16
-        self.vae.to(device)
-
-        prior_ids = batch.extra["glm_prior_token_image_ids"].to(device)
-        if prior_ids.dim() == 1:
-            prior_ids = prior_ids.unsqueeze(0)
-
-        # Latent patch count must match the source prior tokens; mismatch is fatal.
-        src_grid = batch.extra["glm_source_image_grid_thw"][0]
-        cond_h = int(src_grid[1]) * _CONDITION_MULTIPLE_OF
-        cond_w = int(src_grid[2]) * _CONDITION_MULTIPLE_OF
-        cond_img = self.image_processor.preprocess(batch.pil_image, cond_h, cond_w).to(device=device,
-                                                                                       dtype=torch.float32)
-        latent = self.vae.encode(cond_img).latent_dist.mode()
-        # NOTE: at runtime self.vae.config is a diffusers FrozenDict with flat
-        # latents_mean/latents_std fields. Access the flat fields directly.
-        cfg = self.vae.config
-        mean = torch.tensor(cfg.latents_mean, device=device, dtype=torch.float32).view(1, -1, 1, 1)
-        std = torch.tensor(cfg.latents_std, device=device, dtype=torch.float32).view(1, -1, 1, 1)
-        latent = ((latent - mean) / std).to(dtype)
-
-        kv_caches = GlmImageKVCache(num_layers=self.transformer.num_layers)
-        empty_text = batch.prompt_embeds[0][:1, :0, :].to(device=device, dtype=dtype)
-        with set_forward_context(current_timestep=0, attn_metadata=None, forward_batch=batch):
-            self.transformer(
-                hidden_states=latent,
-                encoder_hidden_states=empty_text,
-                prior_token_id=prior_ids,
-                prior_token_drop=torch.zeros((prior_ids.shape[0], ), dtype=torch.bool, device=device),
-                timestep=torch.zeros((1, ), device=device),
-                target_size=torch.tensor([tuple(cond_img.shape[-2:])], device=device, dtype=torch.long),
-                crop_coords=torch.zeros((1, 2), device=device, dtype=torch.long),
-                kv_caches=kv_caches,
-                kv_caches_mode="write",
-            )
-        batch.extra["glm_kv_caches"] = kv_caches
-        return batch

fastvideo/pipelines/basic/glm_image/stages/decoding.py

@@ -1,35 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-from __future__ import annotations
-
-import torch
-
-from fastvideo.distributed import get_local_torch_device
-from fastvideo.fastvideo_args import FastVideoArgs
-from fastvideo.pipelines.stages.decoding import DecodingStage
-from fastvideo.utils import PRECISION_TO_TYPE
-
-
-class GlmImageDecodingStage(DecodingStage):
-
-    @torch.no_grad()
-    def decode(self, latents: torch.Tensor, fastvideo_args: FastVideoArgs) -> torch.Tensor:
-        self.vae.to(get_local_torch_device())
-        latents = latents.to(get_local_torch_device())
-
-        vae_dtype = PRECISION_TO_TYPE[fastvideo_args.pipeline_config.vae_precision]
-        vae_autocast = (vae_dtype != torch.float32 and not fastvideo_args.disable_autocast)
-
-        latents = self._denormalize_latents(latents)
-        if latents.dim() == 5:
-            latents = latents.squeeze(2)
-
-        with torch.autocast(device_type="cuda", dtype=vae_dtype, enabled=vae_autocast):
-            if fastvideo_args.pipeline_config.vae_tiling:
-                self.vae.enable_tiling()
-            if not vae_autocast:
-                latents = latents.to(vae_dtype)
-            decoded = self.vae.decode(latents)
-
-        image = decoded.sample if hasattr(decoded, "sample") else decoded
-        image = (image / 2 + 0.5).clamp(0, 1)
-        return image.unsqueeze(2)

fastvideo/pipelines/basic/glm_image/stages/denoising.py

@@ -1,192 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-"""CFG convention (both denoise paths): row 0 conditional (positive), row 1 unconditional."""
-from __future__ import annotations
-
-import torch
-
-from fastvideo.attention.backends.sdpa import SDPAMetadata
-from fastvideo.distributed import get_local_torch_device
-from fastvideo.fastvideo_args import FastVideoArgs
-from fastvideo.forward_context import set_forward_context
-from fastvideo.pipelines.pipeline_batch_info import ForwardBatch
-from fastvideo.pipelines.stages.denoising import DenoisingStage
-from fastvideo.pipelines.stages.validators import StageValidators as V
-from fastvideo.pipelines.stages.validators import VerificationResult
-from fastvideo.platforms import AttentionBackendEnum
-
-
-class GlmImageDenoisingStage(DenoisingStage):
-
-    def verify_input(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> VerificationResult:
-        result = VerificationResult()
-        result.add_check("timesteps", batch.timesteps, [V.is_tensor, V.min_dims(1)])
-        latents = getattr(batch, "latent", getattr(batch, "latents", None))
-        result.add_check("latents", latents, [V.is_tensor, V.with_dims(5)])
-        result.add_check("num_inference_steps", batch.num_inference_steps, V.positive_int)
-        result.add_check("prompt_embeds", batch.prompt_embeds, V.list_not_empty)
-        return result
-
-    def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch:
-        device = get_local_torch_device()
-        dtype = torch.bfloat16
-        guidance_scale = batch.guidance_scale
-        do_cfg = guidance_scale > 1.0
-
-        latents = getattr(batch, "latent", getattr(batch, "latents", None))
-        if latents is None:
-            raise ValueError("No latents found in batch.")
-        if latents.dim() == 5:
-            latents = latents.squeeze(2)
-
-        prompt_embeds = batch.prompt_embeds[0]
-        text_attention_mask = getattr(batch, "attention_mask", None)
-        timesteps = batch.timesteps
-
-        patch_size = self.transformer.patch_size
-        _, _, h, w = latents.shape
-        image_seq_length = (h // patch_size) * (w // patch_size)
-        text_seq_length = prompt_embeds.shape[1] if prompt_embeds.dim() >= 2 else 0
-
-        first_block = self.transformer.transformer_blocks[0]
-        backend = getattr(first_block.attn1.attn, "backend", None)
-        sdpa = backend == AttentionBackendEnum.TORCH_SDPA and text_attention_mask is not None
-
-        kv_caches = batch.extra.get("glm_kv_caches")
-        if kv_caches is None:
-            self._denoise_t2i(batch, latents, prompt_embeds, text_attention_mask, timesteps, do_cfg, guidance_scale,
-                              text_seq_length, image_seq_length, sdpa, device, dtype)
-        else:
-            self._denoise_i2i(batch, latents, prompt_embeds, text_attention_mask, timesteps, do_cfg, guidance_scale,
-                              text_seq_length, image_seq_length, sdpa, kv_caches, device, dtype)
-        return batch
-
-    def _denoise_t2i(self, batch, latents, prompt_embeds, text_attention_mask, timesteps, do_cfg, guidance_scale,
-                     text_seq_length, image_seq_length, sdpa, device, dtype) -> None:
-        num_inference_steps = batch.num_inference_steps
-        bs = 2 if do_cfg else 1
-        target_size = torch.tensor([[batch.height, batch.width]], device=device, dtype=torch.long).repeat(bs, 1)
-        crop_coords = torch.zeros((bs, 2), device=device, dtype=torch.long)
-
-        prior_token_id = batch.prior_token_id
-        if do_cfg and prior_token_id.shape[0] == 1:
-            prior_token_id = prior_token_id.repeat(2, 1)
-        if do_cfg:
-            prior_token_drop = torch.tensor([False, True], device=device)
-        else:
-            prior_token_drop = getattr(batch, "prior_token_drop", torch.tensor([False], device=device))
-
-        attention_mask_kv = None
-        if sdpa:
-            if (text_attention_mask.shape[0] == 1 and bs > 1):
-                text_attention_mask = text_attention_mask.repeat(bs, 1)
-            mix_attn_mask = torch.ones((bs, text_seq_length + image_seq_length), device=device, dtype=torch.float32)
-            mix_attn_mask[:, :text_seq_length] = (text_attention_mask.float().to(device))
-            attention_mask_kv = (mix_attn_mask > 0).unsqueeze(1).unsqueeze(2)
-
-        with self.progress_bar(total=num_inference_steps) as progress_bar:
-            for i, t in enumerate(timesteps):
-                latent_model_input = torch.cat([latents] * 2) if do_cfg else latents
-                latent_model_input = self.scheduler.scale_model_input(latent_model_input, t).to(dtype)
-                t_expand = t.expand(latent_model_input.shape[0]) - 1
-
-                attn_metadata = (SDPAMetadata(current_timestep=i, attn_mask=attention_mask_kv)
-                                 if attention_mask_kv is not None else None)
-
-                with torch.no_grad(), set_forward_context(current_timestep=i,
-                                                          attn_metadata=attn_metadata,
-                                                          forward_batch=batch):
-                    noise_pred = self.transformer(
-                        latent_model_input,
-                        prompt_embeds,
-                        prior_token_id,
-                        prior_token_drop,
-                        t_expand,
-                        target_size,
-                        crop_coords,
-                    )
-
-                    if do_cfg:
-                        noise_pred_cond, noise_pred_uncond = noise_pred.float().chunk(2)
-                        noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_cond - noise_pred_uncond)
-
-                        guidance_rescale = getattr(batch, "guidance_rescale", 0.0)
-                        if guidance_rescale > 0.0:
-                            dims = list(range(1, noise_pred_cond.ndim))
-                            std_text = noise_pred_cond.std(dim=dims, keepdim=True)
-                            std_cfg = noise_pred.std(dim=dims, keepdim=True)
-                            rescaled = noise_pred * (std_text / std_cfg)
-                            noise_pred = (guidance_rescale * rescaled + (1 - guidance_rescale) * noise_pred)
-
-                    latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
-                progress_bar.update()
-
-        batch.latents = latents.unsqueeze(2)
-
-    def _denoise_i2i(self, batch, latents, prompt_embeds, text_attention_mask, timesteps, do_cfg, guidance_scale,
-                     text_seq_length, image_seq_length, sdpa, kv_caches, device, dtype) -> None:
-        """Two separate transformer calls (cond reads the cache, uncond skips it):
-        the cache mode is one global flag with batch-1 k/v, so a 2-row CFG call
-        cannot express both."""
-        num_inference_steps = batch.num_inference_steps
-        target_size = torch.tensor([[batch.height, batch.width]], device=device, dtype=torch.long)
-        crop_coords = torch.zeros((1, 2), device=device, dtype=torch.long)
-        prior_token_id = batch.prior_token_id[:1]
-        drop_keep = torch.zeros((1, ), dtype=torch.bool, device=device)
-        drop_all = torch.ones((1, ), dtype=torch.bool, device=device)
-        cache_len = kv_caches[0].k_cache.shape[1] if kv_caches[0].k_cache is not None else 0
-
-        def _mask(row: int, with_cache: bool):
-            if not sdpa:
-                return None
-            prefix = cache_len if with_cache else 0
-            m = torch.ones((1, prefix + text_seq_length + image_seq_length), device=device, dtype=torch.float32)
-            m[:, prefix:prefix + text_seq_length] = text_attention_mask[row:row + 1].float().to(device)
-            return (m > 0).unsqueeze(1).unsqueeze(2)
-
-        cond_mask = _mask(0, with_cache=True)
-        uncond_mask = _mask(1, with_cache=False) if do_cfg else None
-        pos_embeds = prompt_embeds[:1]
-        neg_embeds = prompt_embeds[1:2] if do_cfg else None
-
-        with self.progress_bar(total=num_inference_steps) as progress_bar:
-            for i, t in enumerate(timesteps):
-                latent_model_input = self.scheduler.scale_model_input(latents, t).to(dtype)
-                t_expand = t.expand(1) - 1
-
-                cond_meta = (SDPAMetadata(current_timestep=i, attn_mask=cond_mask) if cond_mask is not None else None)
-                with torch.no_grad(), set_forward_context(current_timestep=i,
-                                                          attn_metadata=cond_meta,
-                                                          forward_batch=batch):
-                    noise_pred = self.transformer(latent_model_input,
-                                                  pos_embeds,
-                                                  prior_token_id,
-                                                  drop_keep,
-                                                  t_expand,
-                                                  target_size,
-                                                  crop_coords,
-                                                  kv_caches=kv_caches,
-                                                  kv_caches_mode="read")
-
-                if do_cfg:
-                    uncond_meta = (SDPAMetadata(current_timestep=i, attn_mask=uncond_mask)
-                                   if uncond_mask is not None else None)
-                    with torch.no_grad(), set_forward_context(current_timestep=i,
-                                                              attn_metadata=uncond_meta,
-                                                              forward_batch=batch):
-                        noise_pred_uncond = self.transformer(latent_model_input,
-                                                             neg_embeds,
-                                                             prior_token_id,
-                                                             drop_all,
-                                                             t_expand,
-                                                             target_size,
-                                                             crop_coords,
-                                                             kv_caches=kv_caches,
-                                                             kv_caches_mode="skip")
-                    noise_pred = noise_pred_uncond.float() + guidance_scale * (noise_pred.float() -
-                                                                               noise_pred_uncond.float())
-
-                latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
-                progress_bar.update()
-
-        kv_caches.clear()
-        batch.latents = latents.unsqueeze(2)

fastvideo/pipelines/basic/ltx2/ltx2_pipeline.py

@@ -19,39 +19,6 @@ from fastvideo.pipelines.stages import (DecodingStage, InputValidationStage, LTX
 logger = init_logger(__name__)
 
 
-def _resolve_refine_path(model_path: str, value: str | None) -> str | None:
-    """Resolve a refine component path, preferring paths inside the local
-    model snapshot when the value is relative."""
-    if value is None:
-        return None
-    if os.path.isabs(value):
-        return value
-    candidate = os.path.join(model_path, value)
-    if os.path.exists(candidate):
-        return candidate
-    return value
-
-
-def _resolve_refine_upsampler_path(model_path: str, model_index: dict[str, Any]) -> str | None:
-    """Resolve the refine upsampler directory for a local model snapshot.
-
-    model_index.json keys ("fastvideo_refine_upsampler_path", then
-    "spatial_upsampler") are the documented override and win when present.
-    Otherwise probe the snapshot root: published LTX-2 distilled checkpoints
-    bundle the upsampler weights on disk (spelled "spatial_upscaler")
-    without declaring them in model_index.json.
-    """
-    path = _resolve_refine_path(model_path, model_index.get("fastvideo_refine_upsampler_path"))
-    if path is None and "spatial_upsampler" in model_index:
-        path = _resolve_refine_path(model_path, "spatial_upsampler")
-    if path is None:
-        for dirname in ("spatial_upscaler", "spatial_upsampler"):
-            candidate = os.path.join(model_path, dirname)
-            if os.path.isdir(candidate):
-                return candidate
-    return path
-
-
 class LTX2Pipeline(LoRAPipeline):
 
     _required_config_modules = [
@@ -222,16 +189,28 @@ class LTX2Pipeline(LoRAPipeline):
         # Apply optional FastVideo-specific refine defaults embedded in
         # model_index.json. These are bundled with distilled checkpoints
         # so the pipeline can self-configure without explicit user kwargs.
+        def _resolve_refine_path(value: str | None) -> str | None:
+            if value is None:
+                return None
+            if os.path.isabs(value):
+                return value
+            candidate = os.path.join(self.model_path, value)
+            if os.path.exists(candidate):
+                return candidate
+            return value
+
         if (model_index.get("fastvideo_refine_enabled") is True and fastvideo_args.refine_enabled is None):
             fastvideo_args.ltx2_refine_enabled = True
         if (fastvideo_args.refine_upsampler_path is None and fastvideo_args.ltx2_refine_upsampler_path is None):
-            fastvideo_args.ltx2_refine_upsampler_path = (_resolve_refine_upsampler_path(self.model_path, model_index))
+            fastvideo_args.ltx2_refine_upsampler_path = _resolve_refine_path(
+                model_index.get("fastvideo_refine_upsampler_path"))
+            if (fastvideo_args.ltx2_refine_upsampler_path is None and "spatial_upsampler" in model_index):
+                fastvideo_args.ltx2_refine_upsampler_path = (_resolve_refine_path("spatial_upsampler"))
         if (fastvideo_args.refine_transformer_path is None and fastvideo_args.ltx2_refine_transformer_path is None):
             fastvideo_args.ltx2_refine_transformer_path = (_resolve_refine_path(
-                self.model_path, model_index.get("fastvideo_refine_transformer_path")))
+                model_index.get("fastvideo_refine_transformer_path")))
         if (fastvideo_args.refine_lora_path is None and fastvideo_args.ltx2_refine_lora_path is None):
-            fastvideo_args.ltx2_refine_lora_path = _resolve_refine_path(self.model_path,
-                                                                        model_index.get("fastvideo_refine_lora_path"))
+            fastvideo_args.ltx2_refine_lora_path = _resolve_refine_path(model_index.get("fastvideo_refine_lora_path"))
         if (fastvideo_args.refine_num_inference_steps is None
                 and fastvideo_args.ltx2_refine_num_inference_steps == FastVideoArgs.ltx2_refine_num_inference_steps
                 and model_index.get("fastvideo_refine_num_inference_steps") is not None):
@@ -247,11 +226,10 @@ class LTX2Pipeline(LoRAPipeline):
         if (fastvideo_args.refine_add_noise is None and model_index.get("fastvideo_refine_add_noise") is not None):
             fastvideo_args.ltx2_refine_add_noise = bool(model_index["fastvideo_refine_add_noise"])
         if (fastvideo_args.refine_noise_path is None and fastvideo_args.ltx2_refine_noise_path is None):
-            fastvideo_args.ltx2_refine_noise_path = _resolve_refine_path(self.model_path,
-                                                                         model_index.get("fastvideo_refine_noise_path"))
+            fastvideo_args.ltx2_refine_noise_path = _resolve_refine_path(model_index.get("fastvideo_refine_noise_path"))
         if (fastvideo_args.refine_audio_noise_path is None and fastvideo_args.ltx2_refine_audio_noise_path is None):
             fastvideo_args.ltx2_refine_audio_noise_path = (_resolve_refine_path(
-                self.model_path, model_index.get("fastvideo_refine_audio_noise_path")))
+                model_index.get("fastvideo_refine_audio_noise_path")))
 
         model_index.pop("_class_name")
         model_index.pop("_diffusers_version")

fastvideo/pipelines/pipeline_batch_info.py

@@ -109,10 +109,6 @@ class ForwardBatch:
     max_sequence_length: int | None = None
     prompt_template: dict[str, Any] | None = None
     do_classifier_free_guidance: bool = False
-    # When True, ``guidance_scale`` is passed into models that use embedded guidance (e.g. FLUX)
-    # and must not imply classic dual-forward CFG. Use ``true_cfg_scale > 1`` for true CFG.
-    use_embedded_guidance: bool = False
-    true_cfg_scale: float = 1.0
 
     # Batch info
     batch_size: int | None = None
@@ -256,12 +252,9 @@ class ForwardBatch:
     def __post_init__(self):
         """Initialize dependent fields after dataclass initialization."""
 
-        # LTX-2 text CFG scales; FLUX uses ``use_embedded_guidance`` so ``guidance_scale > 1`` alone
-        # does not enable classifier-free guidance.
+        # Enable CFG for standard guidance_scale and LTX-2 text CFG scales.
         ltx2_text_cfg_enabled = (self.ltx2_cfg_scale_video != 1.0 or self.ltx2_cfg_scale_audio != 1.0)
-        if self.use_embedded_guidance:
-            self.do_classifier_free_guidance = (self.true_cfg_scale > 1.0) or ltx2_text_cfg_enabled
-        elif self.guidance_scale > 1.0 or ltx2_text_cfg_enabled:
+        if self.guidance_scale > 1.0 or ltx2_text_cfg_enabled:
             self.do_classifier_free_guidance = True
         if self.negative_prompt_embeds is None:
             self.negative_prompt_embeds = []

fastvideo/pipelines/stages/flux_stages.py

@@ -1,424 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-
-from __future__ import annotations
-
-import inspect
-from typing import Any
-
-import torch
-from diffusers.utils.torch_utils import randn_tensor
-
-from fastvideo.distributed import get_local_torch_device
-from fastvideo.fastvideo_args import FastVideoArgs
-from fastvideo.forward_context import set_forward_context
-from fastvideo.logger import init_logger
-from fastvideo.pipelines.pipeline_batch_info import ForwardBatch
-from fastvideo.pipelines.stages.base import PipelineStage
-from fastvideo.pipelines.stages.input_validation import InputValidationStage
-from fastvideo.pipelines.stages.timestep_preparation import TimestepPreparationStage
-from fastvideo.utils import PRECISION_TO_TYPE
-
-logger = init_logger(__name__)
-
-
-def _pack_latents(
-    latents: torch.Tensor,
-    batch_size: int,
-    num_channels_latents: int,
-    height: int,
-    width: int,
-) -> torch.Tensor:
-    """Diffusers ``_pack_latents`` for FLUX (2×2 spatial pack in latent space)."""
-    latents = latents.view(batch_size, num_channels_latents, height // 2, 2, width // 2, 2)
-    latents = latents.permute(0, 2, 4, 1, 3, 5)
-    return latents.reshape(batch_size, (height // 2) * (width // 2), num_channels_latents * 4)
-
-
-def _unpack_latents(
-    latents: torch.Tensor,
-    batch_size: int,
-    num_channels_latents: int,
-    height: int,
-    width: int,
-) -> torch.Tensor:
-    """Inverse of ``_pack_latents``."""
-    latents = latents.reshape(batch_size, height // 2, width // 2, num_channels_latents, 2, 2)
-    latents = latents.permute(0, 3, 1, 4, 2, 5)
-    return latents.reshape(batch_size, num_channels_latents, height, width)
-
-
-def _prepare_latent_image_ids(
-    patch_height: int,
-    patch_width: int,
-    device: torch.device,
-    dtype: torch.dtype = torch.long,
-) -> torch.Tensor:
-    """Match Diffusers ``FluxPipeline._prepare_latent_image_ids`` (no batch dim)."""
-    latent_image_ids = torch.zeros(patch_height, patch_width, 3, device=device, dtype=torch.float32)
-    latent_image_ids[..., 1] = latent_image_ids[..., 1] + torch.arange(patch_height, device=device)[:, None]
-    latent_image_ids[..., 2] = latent_image_ids[..., 2] + torch.arange(patch_width, device=device)[None, :]
-    h, w, c = latent_image_ids.shape
-    return latent_image_ids.reshape(h * w, c).to(dtype=dtype)
-
-
-class FluxInputValidationStage(InputValidationStage):
-    """Require height/width divisible by 16 (VAE scale × 2 for FLUX packing)."""
-
-    def forward(
-        self,
-        batch: ForwardBatch,
-        fastvideo_args: FastVideoArgs,
-    ) -> ForwardBatch:
-        if (batch.height is not None and batch.width is not None and (batch.height % 16 != 0 or batch.width % 16 != 0)):
-            raise ValueError("FLUX expects height and width divisible by 16 "
-                             f"(VAE latent grid × 2× packing); got {batch.height}×{batch.width}.")
-        return super().forward(batch, fastvideo_args)
-
-
-class FluxConditioningStage(PipelineStage):
-    """Build CLIP pooled + T5 sequence + ``text_ids`` (and optional negative for true CFG)."""
-
-    @torch.no_grad()
-    def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch:
-        if len(batch.prompt_embeds) < 2:
-            raise ValueError("FluxConditioningStage expects 2 prompt_embeds (CLIP pooled, T5 sequence), "
-                             f"got {len(batch.prompt_embeds)}")
-
-        device = get_local_torch_device()
-        target_dtype = PRECISION_TO_TYPE[fastvideo_args.pipeline_config.dit_precision]
-
-        pooled = batch.prompt_embeds[0].to(device=device, dtype=target_dtype)
-        enc = batch.prompt_embeds[1].to(device=device, dtype=target_dtype)
-        seq_len = enc.shape[1]
-        text_ids = torch.zeros(seq_len, 3, device=device, dtype=torch.long)
-
-        batch.extra["flux_pooled_projections"] = pooled
-        batch.extra["flux_encoder_hidden_states"] = enc
-        batch.extra["flux_text_ids"] = text_ids
-
-        if batch.do_classifier_free_guidance:
-            if not batch.negative_prompt_embeds or len(batch.negative_prompt_embeds) < 2:
-                raise ValueError("True CFG requires two negative_prompt_embeds (CLIP, T5).")
-            neg_pooled = batch.negative_prompt_embeds[0].to(device=device, dtype=target_dtype)
-            neg_enc = batch.negative_prompt_embeds[1].to(device=device, dtype=target_dtype)
-            batch.extra["flux_negative_pooled_projections"] = neg_pooled
-            batch.extra["flux_negative_encoder_hidden_states"] = neg_enc
-
-        return batch
-
-
-class FluxTimestepPreparationStage(TimestepPreparationStage):
-    """Flow Match with resolution-dependent ``mu`` from packed image sequence length."""
-
-    @staticmethod
-    def _calculate_mu(
-        image_seq_len: int,
-        base_seq_len: int = 256,
-        max_seq_len: int = 4096,
-        base_shift: float = 0.5,
-        max_shift: float = 1.15,
-    ) -> float:
-        m = (max_shift - base_shift) / (max_seq_len - base_seq_len)
-        b = base_shift - m * base_seq_len
-        return float(image_seq_len) * m + b
-
-    def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch:
-        sig = inspect.signature(self.scheduler.set_timesteps)
-        if "mu" not in sig.parameters:
-            logger.warning(
-                "FLUX timestep prep: scheduler %s.set_timesteps does not accept 'mu'; falling back to the base "
-                "timestep schedule. FLUX expects a FlowMatchEulerDiscreteScheduler with resolution-dependent "
-                "dynamic shifting — output quality may degrade.",
-                type(self.scheduler).__name__)
-            return super().forward(batch, fastvideo_args)
-
-        cfg = getattr(self.scheduler, "config", None)
-        use_dynamic = bool(getattr(cfg, "use_dynamic_shifting", False)) if cfg is not None else False
-        if not use_dynamic:
-            logger.warning(
-                "FLUX timestep prep: scheduler has use_dynamic_shifting=False; falling back to the base timestep "
-                "schedule and skipping the resolution-dependent 'mu' shift. FLUX requires dynamic shifting for "
-                "correct timesteps — output quality may degrade.")
-            return super().forward(batch, fastvideo_args)
-
-        if batch.height is None or batch.width is None:
-            raise ValueError("height/width must be set before FluxTimestepPreparationStage")
-
-        vae_arch = fastvideo_args.pipeline_config.vae_config.arch_config
-        spatial_ratio = int(getattr(vae_arch, "spatial_compression_ratio", 8))
-        h_lat = batch.height // spatial_ratio
-        w_lat = batch.width // spatial_ratio
-        if h_lat % 2 != 0 or w_lat % 2 != 0:
-            raise ValueError(
-                f"Latent spatial dims must be even for FLUX packing; got {h_lat}×{w_lat} from {batch.height}×{batch.width}."
-            )
-        image_seq_len = (h_lat // 2) * (w_lat // 2)
-
-        base_seq_len = int(getattr(cfg, "base_image_seq_len", 256))
-        max_seq_len = int(getattr(cfg, "max_image_seq_len", 4096))
-        base_shift = float(getattr(cfg, "base_shift", 0.5))
-        max_shift = float(getattr(cfg, "max_shift", 1.15))
-
-        device = get_local_torch_device()
-        mu = self._calculate_mu(
-            image_seq_len=image_seq_len,
-            base_seq_len=base_seq_len,
-            max_seq_len=max_seq_len,
-            base_shift=base_shift,
-            max_shift=max_shift,
-        )
-        self.scheduler.set_timesteps(batch.num_inference_steps, device=device, mu=mu)
-        batch.timesteps = self.scheduler.timesteps
-        return batch
-
-
-class FluxLatentPreparationStage(PipelineStage):
-
-    def __init__(self, scheduler) -> None:
-        super().__init__()
-        self.scheduler = scheduler
-
-    @torch.no_grad()
-    def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch:
-        if batch.height is None or batch.width is None:
-            raise ValueError("height/width required for FluxLatentPreparationStage")
-
-        if isinstance(batch.prompt, list):
-            batch_size = len(batch.prompt)
-        elif batch.prompt is not None:
-            batch_size = 1
-        else:
-            if not batch.prompt_embeds:
-                raise ValueError("prompt or prompt_embeds must be provided")
-            batch_size = batch.prompt_embeds[0].shape[0]
-
-        batch_size *= batch.num_videos_per_prompt
-
-        if isinstance(batch.generator, list) and len(batch.generator) != batch_size:
-            raise ValueError(f"generator list length {len(batch.generator)} does not match batch_size {batch_size}")
-
-        arch = fastvideo_args.pipeline_config.dit_config.arch_config
-        in_channels = int(getattr(arch, "in_channels", 64))
-        num_channels_latents = in_channels // 4
-
-        vae_arch = fastvideo_args.pipeline_config.vae_config.arch_config
-        spatial_ratio = int(getattr(vae_arch, "spatial_compression_ratio", 8))
-
-        h_lat = batch.height // spatial_ratio
-        w_lat = batch.width // spatial_ratio
-
-        dtype = PRECISION_TO_TYPE[fastvideo_args.pipeline_config.dit_precision]
-        device = get_local_torch_device()
-
-        shape = (batch_size, num_channels_latents, h_lat, w_lat)
-        latents = batch.latents
-        if latents is None:
-            latents = randn_tensor(shape, generator=batch.generator, device=device, dtype=dtype)
-            if hasattr(self.scheduler, "init_noise_sigma"):
-                latents = latents * self.scheduler.init_noise_sigma
-        else:
-            latents = latents.to(device=device, dtype=dtype)
-            if latents.shape != shape:
-                raise ValueError(f"Expected latents shape {shape}, got {tuple(latents.shape)}")
-            if hasattr(self.scheduler, "init_noise_sigma"):
-                latents = latents * self.scheduler.init_noise_sigma
-
-        packed = _pack_latents(latents, batch_size, num_channels_latents, h_lat, w_lat)
-
-        patch_h, patch_w = h_lat // 2, w_lat // 2
-        img_ids = _prepare_latent_image_ids(patch_h, patch_w, device, dtype=torch.long)
-
-        batch.latents = packed
-        batch.raw_latent_shape = shape
-        batch.extra["flux_h_lat"] = h_lat
-        batch.extra["flux_w_lat"] = w_lat
-        batch.extra["flux_num_channels_latents"] = num_channels_latents
-        batch.extra["flux_latent_image_ids"] = img_ids
-
-        return batch
-
-
-class FluxDenoisingStage(PipelineStage):
-
-    def __init__(self, transformer, scheduler) -> None:
-        super().__init__()
-        self.transformer = transformer
-        self.scheduler = scheduler
-
-    @staticmethod
-    def _step_kwargs(scheduler_step, batch: ForwardBatch) -> dict[str, Any]:
-        kwargs: dict[str, Any] = {}
-        sig = inspect.signature(scheduler_step)
-        if "generator" in sig.parameters:
-            gen = batch.generator[0] if isinstance(batch.generator, list) else batch.generator
-            kwargs["generator"] = gen
-        return kwargs
-
-    @torch.no_grad()
-    def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch:
-        if batch.timesteps is None:
-            raise ValueError("timesteps must be set before FluxDenoisingStage")
-        if batch.latents is None:
-            raise ValueError("latents must be set before FluxDenoisingStage")
-
-        packed = batch.latents
-        timesteps = batch.timesteps
-
-        pooled = batch.extra["flux_pooled_projections"]
-        enc = batch.extra["flux_encoder_hidden_states"]
-        txt_ids = batch.extra["flux_text_ids"]
-        img_ids = batch.extra["flux_latent_image_ids"]
-
-        neg_pooled = batch.extra.get("flux_negative_pooled_projections")
-        neg_enc = batch.extra.get("flux_negative_encoder_hidden_states")
-
-        true_cfg_scale = float(batch.true_cfg_scale)
-        use_true_cfg = batch.do_classifier_free_guidance and true_cfg_scale > 1.0
-
-        # Prefer the loaded transformer's arch (HF ``guidance_embeds``), not static pipeline defaults.
-        tr_arch = self.transformer.fastvideo_config.arch_config
-        guidance_embeds = bool(getattr(tr_arch, "guidance_embeds", False))
-
-        device = get_local_torch_device()
-        target_dtype = PRECISION_TO_TYPE[fastvideo_args.pipeline_config.dit_precision]
-        autocast_enabled = (target_dtype != torch.float32) and not fastvideo_args.disable_autocast
-
-        bs = packed.shape[0]
-        if guidance_embeds:
-            guidance = torch.full((bs, ), float(batch.guidance_scale), device=device, dtype=torch.float32)
-        else:
-            guidance = None
-
-        step_extras = self._step_kwargs(self.scheduler.step, batch)
-
-        for t in timesteps:
-            t_scalar = t
-            if not isinstance(t_scalar, torch.Tensor):
-                t_scalar = torch.tensor([t_scalar], device=device, dtype=torch.float32)
-            t_scalar = t_scalar.to(device=device, dtype=torch.float32)
-
-            timestep_model = t_scalar.expand(bs).float() / 1000.0
-            timestep_model = timestep_model.to(dtype=target_dtype)
-
-            ts_ctx = int(t_scalar.reshape(-1)[0].item())
-            with (
-                    torch.autocast(
-                        device_type="cuda",
-                        dtype=target_dtype,
-                        enabled=autocast_enabled and device.type == "cuda",
-                    ),
-                    set_forward_context(
-                        current_timestep=ts_ctx,
-                        attn_metadata=None,
-                        forward_batch=batch,
-                    ),
-            ):
-                if use_true_cfg:
-                    assert neg_enc is not None and neg_pooled is not None
-                    n_neg = self.transformer(
-                        hidden_states=packed,
-                        encoder_hidden_states=neg_enc,
-                        pooled_projections=neg_pooled,
-                        timestep=timestep_model,
-                        guidance=guidance,
-                        txt_ids=txt_ids,
-                        img_ids=img_ids,
-                        return_dict=False,
-                    )[0]
-                    n_pos = self.transformer(
-                        hidden_states=packed,
-                        encoder_hidden_states=enc,
-                        pooled_projections=pooled,
-                        timestep=timestep_model,
-                        guidance=guidance,
-                        txt_ids=txt_ids,
-                        img_ids=img_ids,
-                        return_dict=False,
-                    )[0]
-                    noise_pred = n_neg + true_cfg_scale * (n_pos - n_neg)
-                else:
-                    noise_pred = self.transformer(
-                        hidden_states=packed,
-                        encoder_hidden_states=enc,
-                        pooled_projections=pooled,
-                        timestep=timestep_model,
-                        guidance=guidance,
-                        txt_ids=txt_ids,
-                        img_ids=img_ids,
-                        return_dict=False,
-                    )[0]
-
-            packed = self.scheduler.step(
-                noise_pred,
-                t_scalar,
-                packed,
-                return_dict=False,
-                **step_extras,
-            )[0]
-
-        batch.latents = packed
-        return batch
-
-
-class FluxDecodingStage(PipelineStage):
-    """Unpack latents, apply VAE scaling/shift, decode to pixels (5D output ``B×3×1×H×W``)."""
-
-    def __init__(self, vae) -> None:
-        super().__init__()
-        self.vae = vae
-
-    @staticmethod
-    def _denormalize_latents(latents: torch.Tensor, vae: Any) -> torch.Tensor:
-        cfg = getattr(vae, "config", None)
-        sf = getattr(cfg, "scaling_factor", None) if cfg is not None else None
-        sh = getattr(cfg, "shift_factor", None) if cfg is not None else None
-        if sf is None and hasattr(vae, "scaling_factor"):
-            sf = vae.scaling_factor
-        if sh is None and hasattr(vae, "shift_factor"):
-            sh = vae.shift_factor
-        if sf is not None:
-            latents = latents / (sf.to(latents.device, latents.dtype) if isinstance(sf, torch.Tensor) else sf)
-            if sh is not None:
-                latents = latents + (sh.to(latents.device, latents.dtype) if isinstance(sh, torch.Tensor) else sh)
-        return latents
-
-    @torch.no_grad()
-    def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch:
-        packed = batch.latents
-        if packed is None:
-            raise ValueError("latents must be set before FluxDecodingStage")
-
-        h_lat = int(batch.extra["flux_h_lat"])
-        w_lat = int(batch.extra["flux_w_lat"])
-        num_ch = int(batch.extra["flux_num_channels_latents"])
-        raw_shape = batch.raw_latent_shape
-        if raw_shape is None:
-            raise ValueError("raw_latent_shape missing; FluxLatentPreparationStage must run first.")
-        batch_size = int(raw_shape[0])
-
-        infer_device = get_local_torch_device()
-        packed = packed.to(infer_device)
-
-        latents_4d = _unpack_latents(packed, batch_size, num_ch, h_lat, w_lat)
-        latents_4d = self._denormalize_latents(latents_4d, self.vae)
-
-        vae_device = next(self.vae.parameters()).device
-        latents_4d = latents_4d.to(device=vae_device)
-
-        vae_dtype = PRECISION_TO_TYPE[fastvideo_args.pipeline_config.vae_precision]
-        autocast_enabled = (vae_dtype != torch.float32) and not fastvideo_args.disable_autocast
-        use_cuda_autocast = autocast_enabled and vae_device.type == "cuda"
-
-        with torch.autocast(
-                device_type="cuda",
-                dtype=vae_dtype,
-                enabled=use_cuda_autocast,
-        ):
-            if not autocast_enabled:
-                latents_4d = latents_4d.to(dtype=vae_dtype)
-            dec = self.vae.decode(latents_4d)
-            image = dec.sample if hasattr(dec, "sample") else dec[0]
-
-        image = (image / 2 + 0.5).clamp(0, 1)
-        batch.output = image.unsqueeze(2).detach().float().cpu()
-        return batch

fastvideo/registry.py

@@ -58,14 +58,11 @@ from fastvideo.configs.pipelines.wan import (
     WanT2V480PConfig,
     WanT2V720PConfig,
 )
-from fastvideo.configs.pipelines.glm_image import GlmImageConfig
-from fastvideo.configs.pipelines.flux import FluxPipelineConfig
 from fastvideo.configs.pipelines.sd35 import SD35Config
 from fastvideo.configs.pipelines.stable_audio import (StableAudioOpenSmallConfig, StableAudioT2AConfig)
 from fastvideo.api.sampling_param import SamplingParam
 from fastvideo.api.matrixgame2 import MatrixGame2SamplingParam
 from fastvideo.api.matrixgame3 import MatrixGame3SamplingParam
-from fastvideo.api.flux import FluxSamplingParam
 
 from fastvideo.fastvideo_args import WorkloadType
 from fastvideo.logger import init_logger
@@ -721,12 +718,6 @@ def _register_configs() -> None:
             "FastVideo/Matrix-Game-2.0-Base-Diffusers",
             "FastVideo/Matrix-Game-2.0-GTA-Diffusers",
             "FastVideo/Matrix-Game-2.0-TempleRun-Diffusers",
-            # Zelda World Model
-            "mignonjia/mg_longtuning_distilled_zelda",  # distilled using streaming long tuning which init from mg_sf_distilled_zelda_1k_steps
-            "mignonjia/mg_sf_distilled_zelda_1k_steps",  # distilled using self forcing for 1k steps
-            "mignonjia/mg_sf_distilled_zelda",  # distilled using self forcing for 3k steps
-            "mignonjia/mg_causal_zelda",
-            "mignonjia/mg_bidirectional_zelda",
         ],
         model_detectors=[
             lambda path: any(token in path.lower() for token in (
@@ -1077,33 +1068,6 @@ def _register_configs() -> None:
         default_preset="sd35_medium",
     )
 
-    # GLM-Image
-    register_configs(
-        sampling_param_cls=None,
-        pipeline_config_cls=GlmImageConfig,
-        hf_model_paths=[
-            "zai-org/GLM-Image",
-        ],
-        model_detectors=[lambda path: "glmimage" in path.lower() or "glm-image" in path.lower()],
-        workload_types=(WorkloadType.T2I, ),
-        model_family="glm_image",
-    )
-
-    # FLUX.1-dev (Diffusers)
-    register_configs(
-        sampling_param_cls=FluxSamplingParam,
-        pipeline_config_cls=FluxPipelineConfig,
-        workload_types=(WorkloadType.T2I, ),
-        hf_model_paths=[
-            "black-forest-labs/FLUX.1-dev",
-        ],
-        model_detectors=[
-            lambda path: "fluxpipeline" in path,
-            lambda path: "flux.1-dev" in path or "flux_1_dev" in path,
-            lambda path: "/flux/" in path or path.endswith("/flux"),
-        ],
-    )
-
 
 # --- Part 3: Main Resolver ---
 

fastvideo/tests/api/test_parser.py

@@ -183,7 +183,6 @@ def test_load_run_config_supports_yaml_roundtrip(tmp_path) -> None:
                 "guidance_scale_2": None,
                 "guidance_rescale": 0.0,
                 "true_cfg_scale": None,
-                "use_embedded_guidance": None,
                 "boundary_ratio": None,
                 "sigmas": None,
             },

fastvideo/tests/attention/test_flash_attn_default_custom_op.py

@@ -31,29 +31,23 @@ def fa_default_impls():
     if not torch.cuda.is_available():
         pytest.skip("CUDA is required for the FA2/FA3 default custom-op tests")
 
-    # The registration + dispatcher live in `attention/utils/`, alongside the
-    # FP4 cute template and the masked/varlen wrappers. The backend
-    # (`attention/backends/flash_attn.py`) just imports `flash_attn_func_compilable`
-    # from there, so test references the utils module directly.
     try:
-        from fastvideo.attention.utils import flash_attn_default as fa_module
+        from fastvideo.attention.backends import flash_attn as fa_backend
     except ImportError as exc:
-        pytest.skip(f"flash_attn_default not importable: {exc}")
+        pytest.skip(f"flash_attn backend not importable: {exc}")
 
-    if fa_module.fa_version not in ("2", "3"):
+    if fa_backend.fa_version not in ("2", "3"):
         pytest.skip(
             f"FA2/FA3 default custom op only exists for fa_version in (2, 3); "
-            f"got {fa_module.fa_version!r}"
+            f"got {fa_backend.fa_version!r}"
         )
 
-    # compilable dispatcher, the original FA wrapper it falls back to, the
-    # raw custom op for opcheck, and the fa_version (FA2 has full register_
-    # autograd; FA3 keeps the carve-out so some tests gate on this).
+    # compilable dispatcher, the original FA wrapper it falls back to, and
+    # the raw custom op for opcheck.
     return (
-        fa_module.flash_attn_func_compilable,
-        fa_module._fa_default,
+        fa_backend.flash_attn_func_compilable,
+        fa_backend._fa_default,
         torch.ops.fastvideo._flash_attn_default_forward,
-        fa_module.fa_version,
     )
 
 
@@ -77,7 +71,7 @@ def test_default_compilable_inference_matches_original(fa_default_impls, dtype,
     """No-grad path routes through the custom op and is numerically identical."""
     if dtype == torch.bfloat16 and not torch.cuda.is_bf16_supported():
         pytest.skip("bfloat16 is not supported on this GPU")
-    compilable, original, _, _ = fa_default_impls
+    compilable, original, _ = fa_default_impls
 
     torch.manual_seed(0)
     q, k, v = _qkv(dtype, requires_grad=False)
@@ -97,7 +91,7 @@ def test_default_compilable_training_backward_flows(fa_default_impls, dtype, cau
     """
     if dtype == torch.bfloat16 and not torch.cuda.is_bf16_supported():
         pytest.skip("bfloat16 is not supported on this GPU")
-    compilable, original, _, _ = fa_default_impls
+    compilable, original, _ = fa_default_impls
 
     torch.manual_seed(0)
     q_ref, k_ref, v_ref = _qkv(dtype, requires_grad=True)
@@ -121,65 +115,7 @@ def test_default_compilable_training_backward_flows(fa_default_impls, dtype, cau
 @pytest.mark.parametrize("causal", [False, True])
 def test_default_forward_opcheck(fa_default_impls, causal):
     """Schema / fake-kernel consistency for the custom op (forward only)."""
-    _, _, op, _ = fa_default_impls
+    _, _, op = fa_default_impls
     torch.manual_seed(0)
     q, k, v = _qkv(torch.float16, requires_grad=False)
     torch.library.opcheck(op, (q, k, v, None, causal))
-
-
-# --------------------------------------------------------------------------- #
-# FA2-only: backward is registered on the custom op itself. Exercises the     #
-# `register_autograd` wiring directly (not via the dispatcher's carve-out).   #
-# Skipped on FA3 until Kuan-Hao's Modal FA3 setup PR lands and we mirror the  #
-# pattern there.                                                              #
-# --------------------------------------------------------------------------- #
-
-
-@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
-@pytest.mark.parametrize("causal", [False, True])
-def test_default_op_backward_through_registered_autograd(fa_default_impls, dtype, causal):
-    """FA2: gradients flow through ``torch.ops.fastvideo._flash_attn_default_forward``
-    itself (no dispatcher carve-out involved) and match the original
-    ``flash_attn_func``'s gradients."""
-    if dtype == torch.bfloat16 and not torch.cuda.is_bf16_supported():
-        pytest.skip("bfloat16 is not supported on this GPU")
-    _, original, op, fa_version = fa_default_impls
-    if fa_version != "2":
-        pytest.skip(f"register_autograd is only wired for FA2 right now; got {fa_version!r}")
-
-    torch.manual_seed(0)
-    q_ref, k_ref, v_ref = _qkv(dtype, requires_grad=True)
-    q_test, k_test, v_test = _clone(q_ref, k_ref, v_ref)
-
-    # Reference grads via the original autograd.Function.
-    out_ref = original(q_ref, k_ref, v_ref, softmax_scale=None, causal=causal)
-    # Custom-op grads via the registered backward — unpack (out, lse), discard lse.
-    out_test, _ = op(q_test, k_test, v_test, None, causal)
-
-    torch.testing.assert_close(out_test, out_ref,
-                               atol=0 if dtype == torch.float16 else 1e-3,
-                               rtol=0 if dtype == torch.float16 else 1e-3)
-
-    dout = torch.randn_like(out_ref)
-    dq_ref, dk_ref, dv_ref = torch.autograd.grad(
-        (out_ref * dout).sum(), (q_ref, k_ref, v_ref))
-    dq_test, dk_test, dv_test = torch.autograd.grad(
-        (out_test * dout).sum(), (q_test, k_test, v_test))
-
-    atol = rtol = 6e-3 if dtype == torch.float16 else 2e-2
-    torch.testing.assert_close(dq_test, dq_ref, atol=atol, rtol=rtol)
-    torch.testing.assert_close(dk_test, dk_ref, atol=atol, rtol=rtol)
-    torch.testing.assert_close(dv_test, dv_ref, atol=atol, rtol=rtol)
-
-
-@pytest.mark.parametrize("causal", [False, True])
-def test_default_op_opcheck_with_grad_inputs(fa_default_impls, causal):
-    """FA2: full ``opcheck`` including ``test_autograd_registration`` —
-    catches a missing/inconsistent backward at unit-test time, which was
-    exactly the gap that #1373's first revision shipped."""
-    _, _, op, fa_version = fa_default_impls
-    if fa_version != "2":
-        pytest.skip(f"autograd registration only wired for FA2; got {fa_version!r}")
-    torch.manual_seed(0)
-    q, k, v = _qkv(torch.float16, requires_grad=True)
-    torch.library.opcheck(op, (q, k, v, None, causal))

fastvideo/tests/attention/test_flash_attn_no_pad_custom_op.py

@@ -1,239 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-"""Regression guard for the masked/varlen custom ops.
-
-`flash_attn_no_pad_compilable` / `flash_attn_varlen_qk_no_pad_compilable` wrap
-the whole masked-attention functions in `torch.library.custom_op`s so dynamo
-sees one traceable node (the internal unpad/pad bookkeeping runs eager inside).
-
-On FA2 these ops register a real backward — `softmax_lse` is padded back to a
-statically-shaped `[batch, nheads, seqlen]` form on the way out and re-unpadded
-in backward, which calls FA2's `_flash_attn_varlen_backward`. So training also
-backprops through the op (no graph break on the training path either).
-
-On FA3/FA4 these ops are forward+fake only and the `*_compilable` dispatchers
-carve out to the autograd.Function for grad-enabled calls (PR #1373 pattern).
-Tests gating on FA2 are skipped on FA3/FA4.
-
-These tests pin:
-  - inference (no grad): output through the custom op is bit-identical to the
-    original function;
-  - training (requires_grad): gradients through the registered op match the
-    original autograd.Function;
-  - schema/fake-kernel consistency via torch.library.opcheck, both with and
-    without grad-requiring inputs (the latter exercises
-    `test_autograd_registration` — would have caught a missing/broken backward
-    at unit-test time).
-
-GPU assumptions: requires CUDA and the FA2 `flash_attn` varlen package.
-Skips on CPU and when flash_attn is unavailable.
-"""
-
-from __future__ import annotations
-
-import pytest
-import torch
-
-
-@pytest.fixture(scope="module")
-def no_pad_impls():
-    if not torch.cuda.is_available():
-        pytest.skip("CUDA is required for the masked/varlen custom-op tests")
-    try:
-        from fastvideo.attention.utils import flash_attn_no_pad as mod
-    except ImportError as exc:
-        pytest.skip(f"flash_attn_no_pad not importable (flash_attn missing?): {exc}")
-    return mod
-
-
-def _dtype_skip(dtype):
-    if dtype == torch.bfloat16 and not torch.cuda.is_bf16_supported():
-        pytest.skip("bfloat16 is not supported on this GPU")
-
-
-def _fa2_only(mod):
-    if mod._FA_VARLEN_VERSION != "2":
-        pytest.skip(
-            f"register_autograd is only wired for FA2; got "
-            f"_FA_VARLEN_VERSION={mod._FA_VARLEN_VERSION!r}"
-        )
-
-
-def _padding_mask(batch, seqlen, valid_lens, device):
-    mask = torch.zeros(batch, seqlen, dtype=torch.bool, device=device)
-    for i, n in enumerate(valid_lens):
-        mask[i, :n] = True
-    return mask
-
-
-# --------------------------------------------------------------------------- #
-# flash_attn_no_pad (masked self-attention)                                    #
-# --------------------------------------------------------------------------- #
-
-
-@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
-def test_no_pad_inference_matches_original(no_pad_impls, dtype):
-    """No-grad path through the custom op is bit-identical to the original."""
-    _dtype_skip(dtype)
-    mod = no_pad_impls
-    torch.manual_seed(0)
-    device = torch.device("cuda")
-    b, s, h, d = 2, 64, 4, 64
-    qkv = torch.randn(b, s, 3, h, d, device=device, dtype=dtype)
-    mask = _padding_mask(b, s, [64, 48], device)
-
-    with torch.inference_mode():
-        out_ref = mod.flash_attn_no_pad(qkv, mask, causal=False, dropout_p=0.0)
-        out_test = mod.flash_attn_no_pad_compilable(qkv, mask, causal=False, dropout_p=0.0)
-    torch.testing.assert_close(out_test, out_ref, atol=0, rtol=0)
-
-
-@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
-def test_no_pad_training_backward_through_registered_autograd(no_pad_impls, dtype):
-    """FA2: grads flow through the registered op and match the original."""
-    _dtype_skip(dtype)
-    mod = no_pad_impls
-    _fa2_only(mod)
-    torch.manual_seed(0)
-    device = torch.device("cuda")
-    b, s, h, d = 2, 64, 4, 64
-    mask = _padding_mask(b, s, [64, 48], device)
-
-    qkv_ref = torch.randn(b, s, 3, h, d, device=device, dtype=dtype, requires_grad=True)
-    qkv_test = qkv_ref.detach().clone().requires_grad_(True)
-
-    out_ref = mod.flash_attn_no_pad(qkv_ref, mask, causal=False, dropout_p=0.0)
-    # Go through the compilable wrapper (which on FA2 unconditionally routes
-    # to the registered op — no carve-out).
-    out_test = mod.flash_attn_no_pad_compilable(qkv_test, mask, causal=False, dropout_p=0.0)
-
-    dout = torch.randn_like(out_ref)
-    (dqkv_ref,) = torch.autograd.grad((out_ref * dout).sum(), (qkv_ref,))
-    (dqkv_test,) = torch.autograd.grad((out_test * dout).sum(), (qkv_test,))
-
-    atol = rtol = 6e-3 if dtype == torch.float16 else 2e-2
-    torch.testing.assert_close(dqkv_test, dqkv_ref, atol=atol, rtol=rtol)
-
-
-def test_no_pad_forward_opcheck(no_pad_impls):
-    """Schema/fake-kernel consistency for the custom op (forward only)."""
-    torch.manual_seed(0)
-    device = torch.device("cuda")
-    b, s, h, d = 2, 64, 4, 64
-    qkv = torch.randn(b, s, 3, h, d, device=device, dtype=torch.float16)
-    mask = _padding_mask(b, s, [64, 48], device)
-    torch.library.opcheck(
-        torch.ops.fastvideo._flash_attn_no_pad_forward,
-        (qkv, mask, False, 0.0, None, False),
-    )
-
-
-def test_no_pad_opcheck_with_grad_inputs(no_pad_impls):
-    """FA2: full opcheck including ``test_autograd_registration`` — catches a
-    missing/inconsistent backward at unit-test time."""
-    mod = no_pad_impls
-    _fa2_only(mod)
-    torch.manual_seed(0)
-    device = torch.device("cuda")
-    b, s, h, d = 2, 64, 4, 64
-    qkv = torch.randn(b, s, 3, h, d, device=device, dtype=torch.float16, requires_grad=True)
-    mask = _padding_mask(b, s, [64, 48], device)
-    torch.library.opcheck(
-        torch.ops.fastvideo._flash_attn_no_pad_forward,
-        (qkv, mask, False, 0.0, None, False),
-    )
-
-
-# --------------------------------------------------------------------------- #
-# flash_attn_varlen_qk_no_pad (cross-attn / unequal q-k seqlen)                #
-# --------------------------------------------------------------------------- #
-
-
-@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
-def test_varlen_qk_inference_matches_original(no_pad_impls, dtype):
-    _dtype_skip(dtype)
-    mod = no_pad_impls
-    # The varlen-qk custom op's real forward goes through FA's varlen func; off
-    # FA2 that path is the pre-existing FA3 `dropout_p` carve-out (out of scope
-    # here), so skip cleanly like the autograd tests until that's fixed.
-    _fa2_only(mod)
-    torch.manual_seed(1)
-    device = torch.device("cuda")
-    b, sq, sk, h, d = 2, 48, 64, 4, 64
-    q = torch.randn(b, sq, h, d, device=device, dtype=dtype)
-    k = torch.randn(b, sk, h, d, device=device, dtype=dtype)
-    v = torch.randn(b, sk, h, d, device=device, dtype=dtype)
-    qmask = _padding_mask(b, sq, [48, 40], device)
-    kmask = _padding_mask(b, sk, [64, 56], device)
-
-    with torch.inference_mode():
-        out_ref = mod.flash_attn_varlen_qk_no_pad(q, k, v, qmask, kmask, causal=False, dropout_p=0.0)
-        out_test = mod.flash_attn_varlen_qk_no_pad_compilable(q, k, v, qmask, kmask, causal=False, dropout_p=0.0)
-    torch.testing.assert_close(out_test, out_ref, atol=0, rtol=0)
-
-
-@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
-def test_varlen_qk_training_backward_through_registered_autograd(no_pad_impls, dtype):
-    """FA2: grads through q, k, v all flow via the registered op and match."""
-    _dtype_skip(dtype)
-    mod = no_pad_impls
-    _fa2_only(mod)
-    torch.manual_seed(1)
-    device = torch.device("cuda")
-    b, sq, sk, h, d = 2, 48, 64, 4, 64
-    qmask = _padding_mask(b, sq, [48, 40], device)
-    kmask = _padding_mask(b, sk, [64, 56], device)
-
-    q_ref = torch.randn(b, sq, h, d, device=device, dtype=dtype, requires_grad=True)
-    k_ref = torch.randn(b, sk, h, d, device=device, dtype=dtype, requires_grad=True)
-    v_ref = torch.randn(b, sk, h, d, device=device, dtype=dtype, requires_grad=True)
-    q_test = q_ref.detach().clone().requires_grad_(True)
-    k_test = k_ref.detach().clone().requires_grad_(True)
-    v_test = v_ref.detach().clone().requires_grad_(True)
-
-    out_ref = mod.flash_attn_varlen_qk_no_pad(q_ref, k_ref, v_ref, qmask, kmask, causal=False, dropout_p=0.0)
-    out_test = mod.flash_attn_varlen_qk_no_pad_compilable(q_test, k_test, v_test, qmask, kmask,
-                                                          causal=False, dropout_p=0.0)
-
-    dout = torch.randn_like(out_ref)
-    dq_ref, dk_ref, dv_ref = torch.autograd.grad((out_ref * dout).sum(), (q_ref, k_ref, v_ref))
-    dq_test, dk_test, dv_test = torch.autograd.grad((out_test * dout).sum(), (q_test, k_test, v_test))
-
-    atol = rtol = 6e-3 if dtype == torch.float16 else 2e-2
-    torch.testing.assert_close(dq_test, dq_ref, atol=atol, rtol=rtol)
-    torch.testing.assert_close(dk_test, dk_ref, atol=atol, rtol=rtol)
-    torch.testing.assert_close(dv_test, dv_ref, atol=atol, rtol=rtol)
-
-
-def test_varlen_qk_forward_opcheck(no_pad_impls):
-    mod = no_pad_impls
-    _fa2_only(mod)
-    torch.manual_seed(1)
-    device = torch.device("cuda")
-    b, sq, sk, h, d = 2, 48, 64, 4, 64
-    q = torch.randn(b, sq, h, d, device=device, dtype=torch.float16)
-    k = torch.randn(b, sk, h, d, device=device, dtype=torch.float16)
-    v = torch.randn(b, sk, h, d, device=device, dtype=torch.float16)
-    qmask = _padding_mask(b, sq, [48, 40], device)
-    kmask = _padding_mask(b, sk, [64, 56], device)
-    torch.library.opcheck(
-        torch.ops.fastvideo._flash_attn_varlen_qk_no_pad_forward,
-        (q, k, v, qmask, kmask, False, 0.0, None, False),
-    )
-
-
-def test_varlen_qk_opcheck_with_grad_inputs(no_pad_impls):
-    """FA2: full opcheck with requires_grad inputs (autograd-registration check)."""
-    mod = no_pad_impls
-    _fa2_only(mod)
-    torch.manual_seed(1)
-    device = torch.device("cuda")
-    b, sq, sk, h, d = 2, 48, 64, 4, 64
-    q = torch.randn(b, sq, h, d, device=device, dtype=torch.float16, requires_grad=True)
-    k = torch.randn(b, sk, h, d, device=device, dtype=torch.float16, requires_grad=True)
-    v = torch.randn(b, sk, h, d, device=device, dtype=torch.float16, requires_grad=True)
-    qmask = _padding_mask(b, sq, [48, 40], device)
-    kmask = _padding_mask(b, sk, [64, 56], device)
-    torch.library.opcheck(
-        torch.ops.fastvideo._flash_attn_varlen_qk_no_pad_forward,
-        (q, k, v, qmask, kmask, False, 0.0, None, False),
-    )

fastvideo/tests/attention/test_sdpa_head_sizes.py

@@ -1,49 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-"""CPU tests: SDPA must not restrict attention head sizes.
-
-``torch.nn.functional.scaled_dot_product_attention`` is head-size
-agnostic, but ``SDPABackend.get_supported_head_sizes()`` used to declare
-a list copied from FlashAttentionBackend that omitted e.g. 80 (the head
-size of CLIP-L vision encoders such as matrixgame2's). That made
-capability validation flag SDPA as incompatible with head-80 layers even
-though it handles them fine.
-
-CPU-only; no GPU or flash-attn dependency.
-"""
-
-import torch
-
-from fastvideo.attention.backends.sdpa import SDPABackend, SDPAImpl, SDPAMetadata
-from fastvideo.platforms.cuda import CudaPlatformBase
-from fastvideo.platforms.interface import AttentionBackendEnum
-
-
-def test_sdpa_head_sizes_unrestricted() -> None:
-    """SDPA declares no head-size restriction (None), so 80 is allowed."""
-    assert SDPABackend.get_supported_head_sizes() is None
-    # Once the capability-validation API (#1494) lands, head-80 must pass.
-    validate = getattr(SDPABackend, "validate_compatibility", None)
-    if validate is not None:
-        assert validate(head_size=80, dtype=torch.bfloat16) is None
-
-
-def test_explicit_torch_sdpa_pin_head_80_resolves() -> None:
-    """An explicit TORCH_SDPA pin on a head-80 layer resolves to SDPABackend."""
-    cls_str = CudaPlatformBase.get_attn_backend_cls(AttentionBackendEnum.TORCH_SDPA, head_size=80, dtype=torch.bfloat16)
-    assert cls_str == "fastvideo.attention.backends.sdpa.SDPABackend"
-
-
-def test_sdpa_impl_computes_head_80_attention() -> None:
-    """SDPAImpl produces correct output for head_size=80 (not in the old list)."""
-    bs, seq, heads, head_dim = 2, 8, 2, 80
-    softmax_scale = head_dim**-0.5
-    impl = SDPAImpl(num_heads=heads, head_size=head_dim, causal=False, softmax_scale=softmax_scale)
-    torch.manual_seed(0)
-    q, k, v = (torch.randn(bs, seq, heads, head_dim) for _ in range(3))
-
-    out = impl.forward(q, k, v, SDPAMetadata(current_timestep=0))
-
-    assert out.shape == (bs, seq, heads, head_dim)
-    qt, kt, vt = (t.transpose(1, 2) for t in (q, k, v))
-    ref = torch.softmax(qt @ kt.transpose(-2, -1) * softmax_scale, dim=-1) @ vt
-    torch.testing.assert_close(out, ref.transpose(1, 2), rtol=1e-4, atol=1e-4)

fastvideo/tests/contract/test_modal_fa4_policy.py

@@ -1,58 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-"""Guard the Modal FA4 defaults that keep CI lanes on their intended backend.
-
-Pure text/AST analysis: no fastvideo imports, no torch, no Modal client.
-"""
-
-from __future__ import annotations
-
-import ast
-from pathlib import Path
-
-REPO_ROOT = Path(__file__).resolve().parents[3]
-MODAL_ROOT = REPO_ROOT / "fastvideo" / "tests" / "modal"
-PR_TEST = MODAL_ROOT / "pr_test.py"
-LAUNCH_L40S_JOB = MODAL_ROOT / "launch_l40s_job.py"
-SSIM_TEST = MODAL_ROOT / "ssim_test.py"
-
-
-def _function_strings(path: Path, function_name: str) -> str:
-    source = path.read_text(encoding="utf-8")
-    tree = ast.parse(source)
-    for node in tree.body:
-        if isinstance(node, ast.FunctionDef) and node.name == function_name:
-            return "\n".join(
-                child.value
-                for child in ast.walk(node)
-                if isinstance(child, ast.Constant)
-                and isinstance(child.value, str)
-            )
-    raise AssertionError(f"{function_name} not found in {path}")
-
-
-def test_generic_l40s_launcher_defaults_fa4_off():
-    source = LAUNCH_L40S_JOB.read_text(encoding="utf-8")
-    assert '"FASTVIDEO_FA4": os.environ.get("FASTVIDEO_FA4", "0")' in source
-
-
-def test_ssim_launcher_keeps_fa4_enabled_by_default():
-    source = SSIM_TEST.read_text(encoding="utf-8")
-    assert '"FASTVIDEO_FA4": os.environ.get("FASTVIDEO_FA4", "1")' in source
-
-
-def test_pr_model_load_and_training_lanes_disable_fa4():
-    lanes = {
-        "run_transformer_tests": "pytest ./fastvideo/tests/transformers -vs",
-        "run_training_tests": "pytest ./fastvideo/tests/training/Vanilla -srP",
-        "run_training_lora_tests": "pytest ./fastvideo/tests/training/lora/test_lora_training.py -srP",
-        "run_training_tests_VSA": "pytest ./fastvideo/tests/training/VSA -srP",
-        "run_distill_dmd_tests": "pytest ./fastvideo/tests/training/distill/test_distill_dmd.py -vs",
-        "run_self_forcing_tests": "pytest ./fastvideo/tests/training/self-forcing/test_self_forcing.py -vs",
-        "run_train_framework_tests": "pytest ./fastvideo/tests/train/models ./fastvideo/tests/train/methods -vs",
-        "seed_grad_norm_references": "pytest ./fastvideo/tests/train/methods -vs -rs",
-    }
-
-    for function_name, pytest_command in lanes.items():
-        function_strings = _function_strings(PR_TEST, function_name)
-        assert "FASTVIDEO_FA4=0" in function_strings
-        assert pytest_command in function_strings