diff --git a/.gitattributes b/.gitattributes index bb52f06..554d050 100644 --- a/.gitattributes +++ b/.gitattributes @@ -8,3 +8,14 @@ # it (the PowerShell --relock path would otherwise write CRLF), so the diff # stays the review surface. constraints-test.txt text eol=lf + +# Vendored upstream fork code (WhatDreamsCost-ComfyUI 2.0.2). These ship with +# the pack so the koolook Director runs/distributes, but they are upstream code +# — verbatim or near-verbatim (the JS carries only 2 retargeting edits; +# prompt_relay/patches are verbatim). Mark linguist-vendored so GitHub excludes +# them from language stats, and -diff so the large JS blob collapses in PR diffs. +# Our genuinely-authored files (ltx_director.py with the Koolook edits, +# keyframe_grid.py, k_ltx_av_bind_schedule.py) are intentionally NOT marked. +web/whatdreamscost_koolook/ltx_director.js linguist-vendored -diff +forks/whatdreamscost_koolook/versions/v2_0_2/prompt_relay.py linguist-vendored +forks/whatdreamscost_koolook/versions/v2_0_2/patches.py linguist-vendored diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml new file mode 100644 index 0000000..f8b219d --- /dev/null +++ b/.github/codeql/codeql-config.yml @@ -0,0 +1,12 @@ +name: "Koolook CodeQL config" + +# Vendored upstream code is NOT our security review surface. These are the same +# paths marked `linguist-vendored` in .gitattributes — CodeQL does not read +# linguist, so we restate the exclusion here. Keep this list in sync with +# .gitattributes (the vendored-upstream block). Our genuinely-authored files +# (ltx_director.py with the Koolook edits, keyframe_grid.py, k_*.py, the rest of +# web/sidebar/**) are intentionally NOT excluded and remain fully scanned. +paths-ignore: + - "web/whatdreamscost_koolook/ltx_director.js" + - "forks/whatdreamscost_koolook/versions/v2_0_2/prompt_relay.py" + - "forks/whatdreamscost_koolook/versions/v2_0_2/patches.py" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..394171d --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,56 @@ +name: "CodeQL" + +# Advanced setup (replaces GitHub "default setup"). We need a committed config +# so we can paths-ignore the vendored upstream WhatDreamsCost code — default +# setup does not read codeql-config.yml. See .github/codeql/codeql-config.yml. +# +# IMPORTANT: GitHub will not run this workflow while "default setup" for CodeQL +# is still enabled. Disable it once under Settings -> Code security -> CodeQL +# analysis -> Default setup -> Disable, then this workflow takes over. + +on: + push: + branches: [main] + pull_request: + branches: [main] + schedule: + - cron: "23 6 * * 1" + workflow_dispatch: + +permissions: + contents: read + +jobs: + analyze: + name: Analyze (${{ matrix.language }}) + runs-on: ubuntu-latest + permissions: + security-events: write + packages: read + actions: read + contents: read + strategy: + fail-fast: false + matrix: + include: + - language: actions + build-mode: none + - language: javascript-typescript + build-mode: none + - language: python + build-mode: none + steps: + - name: Checkout + uses: actions/checkout@v7.0.0 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + config-file: ./.github/codeql/codeql-config.yml + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{ matrix.language }}" diff --git a/.gitignore b/.gitignore index de2ecdc..92549a3 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,14 @@ docs/automations/LTX-2.3/audio-lipsync/runs/run-*/card.png # install, never the source tree, but ignored here as a defensive guard. web/_dev_build.json +# Converted/generated workflow JSONs dropped at the repo root during dev +# iteration (e.g. *.koolook.json node-swapped copies). Scratch deliverables, +# not package content. Root-level only (`/`), so the 30+ tracked JSON in +# subdirs — test fixtures, web/, .claude/, docs/runs/*/workflow.json — are +# untouched. `config.json` is the one tracked root-level JSON, kept trackable. +/*.json +!/config.json + # Local maintainer dev workspaces — not part of the published package. # These contain vendored 3rd-party clones and debugging scratchpads kept # only on the maintainer's disk. Anything in here would otherwise leak diff --git a/CHANGELOG.md b/CHANGELOG.md index bb1179c..915a8c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,47 @@ The format is inspired by Keep a Changelog and SemVer. ## [Unreleased] +### Added +- **LTX Director fork bumped to upstream v2.0.2** with a new + `forks/whatdreamscost_koolook/versions/v2_0_2/` namespace (pinned to upstream + commit `fe09f73`). This version is a **faithful replica of upstream 2.0.2** — + the older Koolook customizations (`relay_overrides`, `audio_transcript_json`, + per-segment sigma) are intentionally dropped as unused, so all latest 2.0.2 + features (Prompt Relay, audio, motion) behave exactly as upstream. The + canonical `LTXDirector__koolook` node ID rolls to this implementation; + `LTXDirector__koolook_v1_3_2` stays backed by v1.3.9, which remains on disk. + The timeline-editor web extension (`web/whatdreamscost_koolook/ltx_director.js`) + is now upstream 2.0.2's editor vendored verbatim and retargeted to the Koolook + node, so its UX matches upstream exactly (Add Video / Add IC Video / Retake + Mode / IC-LoRA track, correct widget visibility) — replacing the old + 1.3.9-era editor. The localStorage quota guard stays in the global + `web/koolook_draft_guard.js`. +- **Keyframe latent-grid snapping and ease controls (issue #258).** The v2.0.2 + node adds a `snap_keyframes_to_grid` toggle (default on) that snaps each + image keyframe to the center of its LTX latent-time bucket, so hard-pinned + keyframes land cleanly on a single latent frame and two pins never collide in + one bucket (the later pin is bumped to the next free bucket with a logged + warning). It also adds opt-in `keyframe_ease` / `ease_falloff` controls that + emit strength-ramped neighbor pins of the same pose, smoothing into and out + of hard keyframes without weakening the exact center pose. + +- **LTX A/V Bind Schedule node (`LTXAVBindSchedule`).** Decouples big motion + from lip-sync in LTX 2.3: a model patcher that scales the audio→video + cross-attention (`audio_to_video_attn`) by a per-step gain ramping from + `early_gain` (early/high-sigma steps, where coarse motion settles audio-blind) + up to full at low sigma (where lips bind). Targets the keyframe-transition jump + when a cut lands on an audio peak — one pass, no re-noise. `early_gain=1.0` + reproduces stock behavior. Ramp math is unit-tested; the model wiring is + validated by rendering against a live LTX 2.3 AV model. + +### Fixed +- **`dev-sync` from a git worktree.** `scripts/sync_to_dev.py` now resolves + `.env` with the worktree→main-repo fallback (the committed `.env` lives only + in the main checkout), matching `sync_to_dev_audio.py` and the documented + behavior. The fallback is now a shared `find_dotenv()` that both scripts use, + so they can't drift. Previously plain `dev-sync` failed with + "KOLOOK_COMFYUI_DEV_PATH not set" from any worktree. + ## [0.4.4] - 2026-06-23 ### Added diff --git a/__init__.py b/__init__.py index 27a0b4a..2ec2c0f 100644 --- a/__init__.py +++ b/__init__.py @@ -131,6 +131,7 @@ else: _merge_node_group("Easy Utility", ".k_easy_utility") _merge_node_group("Koolook Loop Status", ".k_loop_status") _merge_node_group("Koolook Publish Contract", ".k_publish_contract") + _merge_node_group("LTX A/V Bind Schedule", ".k_ltx_av_bind_schedule") # The VHS-dependent video wrappers degrade to empty mappings when # VideoHelperSuite is absent (they catch the import internally); the # guard also covers a hard ImportError should one ever slip through. diff --git a/docs/automations/LTX-2.3/audio-lipsync/CHEATSHEET.md b/docs/automations/LTX-2.3/audio-lipsync/CHEATSHEET.md index f3ef5e2..fe9d529 100644 --- a/docs/automations/LTX-2.3/audio-lipsync/CHEATSHEET.md +++ b/docs/automations/LTX-2.3/audio-lipsync/CHEATSHEET.md @@ -14,6 +14,13 @@ the `relay_overrides` knobs on the Koolook fork of `LTXDirector` (model-generated audio path). Goal: get `use_custom_audio=True` to look as clean as baseline. +As of the v2.0.2 fork, `LTXDirector__koolook` also exposes +`snap_keyframes_to_grid` (default **on**): it snaps each image keyframe to +its LTX latent-bucket center so hard pins stop "jumping" at bucket edges and +two pins never collide in one bucket. Resolution ceiling is one pin per +temporal-stride frames (~333 ms @ 24 fps). See +[`findings.md`](findings.md) and ComfyUI-Koolook#258. + ## Chat triggers (user-initiated only — never automatic) | Phrase | Script | What it does | diff --git a/docs/automations/LTX-2.3/audio-lipsync/findings.md b/docs/automations/LTX-2.3/audio-lipsync/findings.md index 2cf9d45..c8d3611 100644 --- a/docs/automations/LTX-2.3/audio-lipsync/findings.md +++ b/docs/automations/LTX-2.3/audio-lipsync/findings.md @@ -13,8 +13,83 @@ Anything in this file is **locked in** — don't sweep it again without a reason --- -*No findings yet. First entry appears once a knob's effect is stable across -≥ 3 confirming runs.* +## Audio→video binding must be sigma-scheduled, not full-on, to keep motion smooth AND lips synced + +**Finding.** In the LTX 2.3 AV transformer the audio→video coupling is one gated +cross-attention per block (`audio_to_video_attn`, gated by +`transformer_options["a2v_cross_attn"]`). Left full-on every denoise step, that +coupling fights hard keyframe pose changes that land on an audio peak → the +keyframe-boundary **jump**. The fix is the Koolook `LTXAVBindSchedule` node: it +scales the a2v output by a gain that **ramps from `early_gain` (early/high-sigma +steps) to 1.0 (late/low-sigma steps)**. + +**Promising knee: `early_gain=0.35, bind_start=0.25, bind_end=0.55`** (8-step base +pass) → almost-perfect lip-sync **and** smooth motion, no jumps on the 4-keyframe +goat clip (`Grayscale-BLK` v003 / 00020-1x, single stage). + +> ⚠️ **Not yet a clean validation — confound.** That clip's transitions +> (34/63/93) were deliberately placed on audio **lows**, so the absence of jumps +> may be the placement, not the node. The controlled test is still **pending**: +> same on-**peak** timeline, render `early_gain=1.0` (stock — should jump at the +> on-peak transition) vs `early_gain=0.35`. The node is only proven if `0.35` +> removes a jump that `1.0` shows on the *same* on-peak layout. Until then this +> is a strong lead, not a settled finding. + +**Why.** Lip-sync *timing* is itself coarse motion — decided in the early steps, +the same place pose transitions resolve. So the knob is a single trade axis: + +| `early_gain` | result | +|---|---| +| `0.0` (audio off early, e.g. `0.0 / 0.50 / 0.80`) | smooth motion, **lips never sync** (mouth posed audio-blind) | +| `1.0` (stock, full audio) | lips sync, **transitions jump** | +| **`~0.35`** (`0.35 / 0.25 / 0.55`) | **the knee — both** | + +A ~35% audio floor from step 0 (plus full binding by mid-denoise) is enough to +time the lips while staying low enough early that pose changes settle smoothly. +Tuning: lips soft → raise `early_gain` / lower `bind_start`; motion jumps → the +reverse. + +**Notes.** The global (scalar) ramp is the reliable lever. A per-frame +"transition-window" mode exists in the node but currently falls back to scalar +on the AV model (`grid_sizes` is `None` at the a2v call and the video token +stream is padded with appended keyframe-guide frames — e.g. `19 timeline + 4 +guide = 23` frames × 819 tokens), so mapping tokens→frames isn't reliable yet. +The architecturally-cleaner path for motion-rich lip-sync remains the IC-LoRA +Video track. Node + ramp math: `k_ltx_av_bind_schedule.py` (ComfyUI-Koolook#259). + +--- + +## Hard-pinned image keyframes must land on a latent-bucket center, one per bucket + +**Finding.** A hard-pinned LTX Director image keyframe is injected as a *single +latent frame* at the segment's left edge (`insert_frame = seg_start`), and its +effective time is quantized to the model's temporal stride `s` +(`s = scale_factors[0]`, 8 for LTXV) via `L(f) = 0 if f==0 else 1 + (f-1)//s`. +So a pin dropped near a bucket boundary rounds ambiguously (reads as a whole +bucket early/late), and two pins inside the same `s`-frame bucket collapse onto +one latent frame and fight. Either shows up as the keyframe-boundary **"jump"**. +Resolution is at most **one pin per `s` frames** (~333 ms / ~3 pins-per-sec @ +24 fps, s=8). + +**Confirmed across runs** (`Grayscale-BLK_h264_*`, same audio/seed/sampler): +3-pin layout that happened to land mid-bucket rendered smooth (00016, 00019); a +4-pin layout that pushed a switch near a bucket edge jumped (00018). Audio gaps +were ruled out — the 00018 audio had no leading/interior gap. + +**Fix (shipped).** `forks/whatdreamscost_koolook/versions/v2_0_2/` adds +`snap_keyframes_to_grid` (default on): image keyframes are snapped to their +latent-bucket center before the guide is built, and colliding pins are bumped to +the next free bucket with a logged warning. Snapping within a bucket doesn't +change which latent frame a pin lands on, so artistic timing is preserved — it +only removes the boundary ambiguity. Pure math + tests: +[`keyframe_grid.py`](../../../../forks/whatdreamscost_koolook/versions/v2_0_2/keyframe_grid.py), +[`test_whatdreamscost_v2_0_2_keyframe_grid.py`](../../../../tests/forks/test_whatdreamscost_v2_0_2_keyframe_grid.py). +Issue: ComfyUI-Koolook#258. + +**Animator note.** Viseme/pose *content* matching (mouth open on a vocal, etc.) +is the separate manual lever and stays the animator's job; the grid-snap only +fixes *placement*, and the per-pin resolution ceiling above bounds how finely a +still pin can track speech (use the IC-LoRA video track for sub-bucket motion). --- diff --git a/forks/THIRD_PARTY.md b/forks/THIRD_PARTY.md index 7009951..3adcf1e 100644 --- a/forks/THIRD_PARTY.md +++ b/forks/THIRD_PARTY.md @@ -266,6 +266,48 @@ below — GPL-3.0 §5(c) requires the whole work to be GPL-3.0. `scripts/loop_audio.config.json`. - **Last reviewed:** 2026-05-29 +### WhatDreamsCost/WhatDreamsCost-ComfyUI — v2.0.2 LTX Director subset (Koolook fork) + +- **Name:** WhatDreamsCost-ComfyUI +- **Upstream repo URL:** https://github.com/WhatDreamsCost/WhatDreamsCost-ComfyUI +- **Upstream commit/tag used:** `fe09f73756df202d08341c66b4dc5fc8d2acca22` + (pyproject/README version `2.0.2`; upstream does not tag releases). +- **License:** GPL-3.0 (verified 2026-06-22 from the upstream `LICENSE` + file). +- **Local path(s):** [`forks/whatdreamscost_koolook/versions/v2_0_2/`](whatdreamscost_koolook/versions/v2_0_2/) +- **What changed locally:** A faithful replica of upstream 2.0.2 — the older + Koolook customizations (`relay_overrides`, `audio_transcript_json`, + per-segment sigma) are **intentionally dropped** as unused, so every latest + 2.0.2 feature (Prompt Relay, audio, motion) behaves exactly as upstream. + - **`ltx_director.py` (upstream verbatim + 3 additive changes).** (1) the + node is namespaced as `LTXDirector__koolook` so it coexists with the + installed upstream copy; (2) **issue #258:** a `snap_keyframes_to_grid` + toggle (default on, appended last in the schema) snaps each image + keyframe's pixel position to the center of its LTX latent-time bucket + before the guide is built, so hard pins land on a single latent frame and + two pins never collide in one bucket (the later is bumped, with a logged + warning). Pixel positions in, pixel positions out — the upstream + `LTXDirectorGuide` re-derives latent indices, so it needs no change; and + (3) `keyframe_ease` / `ease_falloff` optionally emit strength-ramped + neighbor pins of the same pose to smooth into and out of hard keyframes. + Nothing else is modified. + - **`prompt_relay.py` (verbatim vendored).** Unmodified upstream 2.0.2; + imported by `ltx_director.py` for its Prompt Relay helpers. + - **`patches.py` (verbatim vendored).** Unmodified upstream `fe09f73`. + - **`keyframe_grid.py` (new, Koolook-original).** Pure stdlib latent-grid + snap/ease helpers; unit-tested in + [`tests/forks/test_whatdreamscost_v2_0_2_keyframe_grid.py`](../tests/forks/test_whatdreamscost_v2_0_2_keyframe_grid.py). +- **What was *not* forked:** Same policy as v1.3.9 — `LTXDirectorGuide` and + other WhatDreamsCost helper nodes remain runtime dependencies from the + user's installed upstream package. +- **Why the workflow-facing ID is stable:** the canonical + `LTXDirector__koolook` rolls to this clean 2.0.2 implementation (the package + entrypoint loads `v2_0_2` last). The legacy `LTXDirector__koolook_v1_3_2` ID + is not registered here; it stays backed by the v1.3.9 package for workflows + saved against it. v1.3.9 (with the older customizations) remains on disk for + rollback. +- **Last reviewed:** 2026-06-22 + ## De-vendored upstream code (untracked in v0.1.4 / v0.1.5) Six third-party trees were untracked from MAIN's git index in the diff --git a/forks/forks_manifest.yaml b/forks/forks_manifest.yaml index 91dadaa..d8d6d76 100644 --- a/forks/forks_manifest.yaml +++ b/forks/forks_manifest.yaml @@ -1,5 +1,5 @@ version: 1 -updated_at: "2026-05-28" +updated_at: "2026-06-22" external_forks: root_relative_to_main: "../ComfyUI-Forks" root_env_var: "KOLOOK_FORKS_DIR" @@ -113,6 +113,55 @@ entries: by this v1.3.9 implementation. The original v1.3.2 source remains on disk for attribution, review, and rollback. + - id: "whatdreamscost_koolook_v2_0_2_ltx_director" + name: "WhatDreamsCost Koolook v2.0.2 LTX Director subset" + source_repo: "https://github.com/WhatDreamsCost/WhatDreamsCost-ComfyUI.git" + source_ref: "2.0.2 (pyproject/README version on main; upstream does not tag releases)" + pinned_commit: "fe09f73756df202d08341c66b4dc5fc8d2acca22" + external_checkout: + relative_path_from_forks_root: "whatdreamscost-v2.0.2-koolook" + upstream_tracking_path_from_forks_root: "whatdreamscost-main-upstream" + default_branch: "main" + local_paths: + - "forks/whatdreamscost_koolook/__init__.py" + - "forks/whatdreamscost_koolook/versions/v2_0_2/__init__.py" + - "forks/whatdreamscost_koolook/versions/v2_0_2/UPSTREAM_PIN.yaml" + - "forks/whatdreamscost_koolook/versions/v2_0_2/ltx_director.py" + - "forks/whatdreamscost_koolook/versions/v2_0_2/prompt_relay.py" + - "forks/whatdreamscost_koolook/versions/v2_0_2/patches.py" + - "forks/whatdreamscost_koolook/versions/v2_0_2/keyframe_grid.py" + - "web/whatdreamscost_koolook/ltx_director.js" + status: "modified" + sync_state: "upstream-verbatim-plus-keyframe-snap" + license: "GPL-3.0" + license_verified_at: "2026-06-22" + exposed_node_ids: + - "LTXDirector__koolook" + notes: | + A faithful replica of upstream 2.0.2 `LTXDirector` — NOT a carry-forward + of the older Koolook customizations. relay_overrides, audio_transcript_json, + and the per-segment sigma patch are intentionally dropped (unused), so all + latest 2.0.2 features (Prompt Relay, audio, motion) behave exactly as + upstream. `ltx_director.py` is upstream 2.0.2 verbatim except for three + additive changes: the namespaced node ID `LTXDirector__koolook`, the + `snap_keyframes_to_grid` latent-bucket keyframe snap (default on, issue + #258) — image keyframes snap to the center of their LTX latent-time bucket + so hard pins land cleanly on one latent frame, with collision bumping + + warnings — and `keyframe_ease` / `ease_falloff`, which optionally emit + strength-ramped neighbor pins of the same pose to smooth into and out of + hard keyframes. `prompt_relay.py` and `patches.py` are vendored verbatim from + upstream fe09f73. The canonical `LTXDirector__koolook` rolls to this clean + 2.0.2 implementation (package entrypoint loads v2_0_2 last); the legacy + `LTXDirector__koolook_v1_3_2` ID stays backed by the v1_3_9 package. The + companion `LTXDirectorGuide` is upstream-unmodified and consumed from the + user's install. v1.3.9 (with the older customizations) remains on disk. + The timeline editor `web/whatdreamscost_koolook/ltx_director.js` is the + upstream 2.0.2 `js/ltx_director.js` vendored verbatim with only two + retargeting edits (unique extension name + bind to `LTXDirector__koolook`), + so the editor UX matches upstream 2.0.2 exactly; the old 1.3.9-era editor + and its now-obsolete base64-scrub deltas are replaced. localStorage quota + guarding stays in the global `web/koolook_draft_guard.js`. + - id: "easy_resize_inspiration" name: "Easy Resize (Koolook reimplementation, inspired by KJ Nodes)" source_repo: "https://github.com/kijai/ComfyUI-KJNodes" diff --git a/forks/whatdreamscost_koolook/__init__.py b/forks/whatdreamscost_koolook/__init__.py index c623587..a402515 100644 --- a/forks/whatdreamscost_koolook/__init__.py +++ b/forks/whatdreamscost_koolook/__init__.py @@ -11,17 +11,22 @@ from .versions.v1_3_2 import NODE_CLASS_MAPPINGS as V1_3_2_NODE_CLASS_MAPPINGS from .versions.v1_3_2 import NODE_DISPLAY_NAME_MAPPINGS as V1_3_2_NODE_DISPLAY_NAME_MAPPINGS from .versions.v1_3_9 import NODE_CLASS_MAPPINGS as V1_3_9_NODE_CLASS_MAPPINGS from .versions.v1_3_9 import NODE_DISPLAY_NAME_MAPPINGS as V1_3_9_NODE_DISPLAY_NAME_MAPPINGS +from .versions.v2_0_2 import NODE_CLASS_MAPPINGS as V2_0_2_NODE_CLASS_MAPPINGS +from .versions.v2_0_2 import NODE_DISPLAY_NAME_MAPPINGS as V2_0_2_NODE_DISPLAY_NAME_MAPPINGS NODE_CLASS_MAPPINGS = {} NODE_DISPLAY_NAME_MAPPINGS = {} -# v1.3.9 intentionally updates the legacy v1.3.2 workflow ID after the -# historical v1.3.2 mappings load. Saved workflows keep opening, but the -# implementation behind `LTXDirector__koolook_v1_3_2` is the upgraded -# compatibility alias, not byte-identical v1.3.2 code. +# Versions load oldest -> newest so the canonical stable IDs +# (`LTXDirector__koolook`, `LTXDirector__koolook_v1_3_2`) resolve to the +# newest implementation. Saved workflows keep opening; the implementation +# behind them is the latest upgraded version, not byte-identical older code. +# v2.0.2 is loaded last and therefore wins the shared IDs. NODE_CLASS_MAPPINGS.update(V1_3_2_NODE_CLASS_MAPPINGS) NODE_CLASS_MAPPINGS.update(V1_3_9_NODE_CLASS_MAPPINGS) +NODE_CLASS_MAPPINGS.update(V2_0_2_NODE_CLASS_MAPPINGS) NODE_DISPLAY_NAME_MAPPINGS.update(V1_3_2_NODE_DISPLAY_NAME_MAPPINGS) NODE_DISPLAY_NAME_MAPPINGS.update(V1_3_9_NODE_DISPLAY_NAME_MAPPINGS) +NODE_DISPLAY_NAME_MAPPINGS.update(V2_0_2_NODE_DISPLAY_NAME_MAPPINGS) __all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"] diff --git a/forks/whatdreamscost_koolook/versions/v2_0_2/UPSTREAM_PIN.yaml b/forks/whatdreamscost_koolook/versions/v2_0_2/UPSTREAM_PIN.yaml new file mode 100644 index 0000000..5fdae60 --- /dev/null +++ b/forks/whatdreamscost_koolook/versions/v2_0_2/UPSTREAM_PIN.yaml @@ -0,0 +1,33 @@ +source_repo: "https://github.com/WhatDreamsCost/WhatDreamsCost-ComfyUI.git" +source_ref: "2.0.2 (pyproject/README version on main; upstream does not tag releases)" +pinned_commit: "fe09f73756df202d08341c66b4dc5fc8d2acca22" +external_checkout_relative_to_forks_root: "whatdreamscost-v2.0.2-koolook" +upstream_tracking_relative_to_forks_root: "whatdreamscost-main-upstream" +external_work_branch: "main" +license: "GPL-3.0" +license_verified_at: "2026-06-22" +notes: | + A faithful replica of upstream 2.0.2 `LTXDirector` with only scoped additive + keyframe controls. None of the older Koolook customizations (relay_overrides, + audio_transcript_json, per-segment sigma) are carried into this version — it + is a clean 2.0.2 base so all latest upstream features (Prompt Relay, audio, + motion) behave exactly as upstream. + + This version folder ships: + - ltx_director.py (upstream 2.0.2 VERBATIM + three additive changes only: + the namespaced node ID `LTXDirector__koolook`, and the + `snap_keyframes_to_grid` latent-bucket keyframe snap from issue #258, + and `keyframe_ease` / `ease_falloff` strength-ramped neighbor pins). + - prompt_relay.py (VERBATIM upstream 2.0.2; imported by ltx_director.py). + - patches.py (VERBATIM upstream 2.0.2; imported by ltx_director.py). + - keyframe_grid.py (NEW, Koolook-original) pure latent-grid snap/ease helpers + used by ltx_director.py; unit-tested in tests/forks/. + + Node IDs: the canonical `LTXDirector__koolook` rolls to this clean 2.0.2 + implementation (package entrypoint loads v2_0_2 last). The legacy + `LTXDirector__koolook_v1_3_2` ID is NOT registered here; it stays backed by + the v1_3_9 package for workflows saved against it. The companion + `LTXDirectorGuide` is upstream-unmodified and consumed from the user's + install; the grid-snap operates on the pixel `insert_frames` this node emits, + so the guide node needs no change. v1.3.9 (with the older customizations) + remains on disk for rollback. diff --git a/forks/whatdreamscost_koolook/versions/v2_0_2/__init__.py b/forks/whatdreamscost_koolook/versions/v2_0_2/__init__.py new file mode 100644 index 0000000..7e468a2 --- /dev/null +++ b/forks/whatdreamscost_koolook/versions/v2_0_2/__init__.py @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +""" +WhatDreamsCost Koolook v2.0.2 mappings. + +A faithful replica of upstream WhatDreamsCost-ComfyUI 2.0.2 `LTXDirector`, +namespaced as `LTXDirector__koolook`, with two additive keyframe controls: +`snap_keyframes_to_grid` (issue #258) and `keyframe_ease` / `ease_falloff`. +No older Koolook customizations — Prompt Relay, audio, and every 2.0.2 feature +are upstream-exact. + +The package entrypoint loads this version last, so the canonical +`LTXDirector__koolook` resolves here. The legacy `LTXDirector__koolook_v1_3_2` +ID stays registered by the v1_3_9 package for workflows saved against it. +""" + +from .ltx_director import LTXDirector + + +NODE_CLASS_MAPPINGS = { + "LTXDirector__koolook": LTXDirector, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "LTXDirector__koolook": "LTX Director (Koolook)", +} + +__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"] diff --git a/forks/whatdreamscost_koolook/versions/v2_0_2/keyframe_grid.py b/forks/whatdreamscost_koolook/versions/v2_0_2/keyframe_grid.py new file mode 100644 index 0000000..c69de90 --- /dev/null +++ b/forks/whatdreamscost_koolook/versions/v2_0_2/keyframe_grid.py @@ -0,0 +1,177 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +""" +Latent-grid keyframe snapping for the Koolook LTX Director fork. + +Original ComfyUI-Koolook code (not derived from upstream WhatDreamsCost). +Authored 2026-06-22. License: GPL-3.0-or-later (matches the pack). + +Background +---------- +LTX encodes time with a temporal stride ``s`` (``scale_factors[0]`` at +runtime; 8 for LTXV). A guide image pinned at pixel frame ``f`` is injected +into a single latent frame at index:: + + L(f) = 0 if f <= 0 # causal first frame + = 1 + (f - 1)//s otherwise + +So a pin's effective time is quantized to ~``s``-frame buckets. Two +consequences this module addresses: + +1. A pin dropped near a bucket boundary rounds ambiguously and can read as + placed a whole bucket early/late. Snapping each pin to its bucket + *center* removes the ambiguity without changing which latent frame it + lands on (any pixel in the same bucket maps to the same ``L``). +2. Two pins inside one bucket collapse to the same latent frame and fight + (the later overwrites the earlier). We bump the later pin to the next + free bucket and surface a warning instead of silently clobbering. + +This is a placement fix only: pose/phoneme *content* matching is the +animator's job; strength stays whatever the caller passes (no softening). +""" + +from __future__ import annotations + +import math +from typing import List, Tuple + + +def latent_index(frame: int, stride: int) -> int: + """Latent frame index that pixel ``frame`` lands in (causal first frame).""" + if frame <= 0: + return 0 + return 1 + (frame - 1) // stride + + +def bucket_center(index: int, stride: int) -> int: + """Pixel frame at the center of latent bucket ``index``. + + ``latent_index(bucket_center(i, s), s) == i`` for every ``i >= 0``. + Bucket 0 is the lone causal frame 0; bucket ``i >= 1`` spans pixels + ``[1 + (i-1)*s, i*s]`` and its center is the lower-middle pixel. + """ + if index <= 0: + return 0 + return (index - 1) * stride + 1 + (stride - 1) // 2 + + +def latent_count_for_duration(duration_frames: int, stride: int) -> int: + """Latent frame count for an LTX timeline after the ``stride*n+1`` ceiling. + + LTXV latent generation rounds pixel duration up to the next valid + ``stride*n+1`` length. The snapper must use that ceiled temporal length too; + using the floor count drops the final valid latent bucket for non-conforming + durations such as 120 frames at stride 8. + """ + if stride < 1: + raise ValueError(f"stride must be >= 1, got {stride}") + return 1 + int(math.ceil(max(0, int(duration_frames) - 1) / float(stride))) + + +def snap_keyframes_to_grid( + insert_frames: List[int], + stride: int, + latent_length: int, +) -> Tuple[List[int], List[str]]: + """Snap guide-image pixel positions to latent-bucket centers, 1 per bucket. + + Parameters + ---------- + insert_frames: + Pixel-frame positions of the image guides, in timeline order. Frame + ``0`` (the identity/start pin on the causal first latent frame) is + passed through untouched. + stride: + Temporal stride ``s`` (``scale_factors[0]``). Must be >= 1. + latent_length: + Number of latent frames available. Highest assignable latent index + is ``latent_length - 1``; pins that cannot find a free bucket at or + below it are left on their (clamped) target and a warning is added. + + Returns + ------- + (snapped_frames, warnings) + ``snapped_frames`` is the same length/order as ``insert_frames``. + ``warnings`` is a (possibly empty) list of human-readable strings. + """ + if stride < 1: + raise ValueError(f"stride must be >= 1, got {stride}") + + warnings: List[str] = [] + max_index = max(0, latent_length - 1) + + # Resolve in chronological order so an earlier pin keeps its bucket and a + # colliding later pin is the one that moves. + order = sorted(range(len(insert_frames)), key=lambda i: insert_frames[i]) + + used: set[int] = set() + snapped: List[int] = [0] * len(insert_frames) + + for pos in order: + f = insert_frames[pos] + target = latent_index(f, stride) + target = min(target, max_index) + + if target in used: + bumped = target + while bumped in used and bumped < max_index: + bumped += 1 + if bumped in used: + warnings.append( + f"keyframe at frame {f} collides on latent frame {target} " + f"and no free bucket remains below the clip end " + f"(latent_length={latent_length}, stride={stride}); " + f"too many pins for this duration — left overlapping." + ) + else: + warnings.append( + f"keyframe at frame {f} shared latent frame {target} with an " + f"earlier pin; bumped to latent frame {bumped} " + f"(~{bucket_center(bumped, stride)} px). Pins resolve at most " + f"one per {stride} frames." + ) + target = bumped + + used.add(target) + snapped[pos] = bucket_center(target, stride) + + return snapped, warnings + + +def expand_keyframe_ease(insert_frames, strengths, stride, ease, falloff, max_frame): + """Expand each hard keyframe into a strength-ramped cluster (ease in/out). + + Keeps every center pin at its original (exact) strength, and adds neighbor + pins one latent-bucket apart (``+/- k*stride`` pixel frames) at decreasing + strength ``center * falloff**k`` for ``k in 1..ease``. The model is then + softly pulled toward the pose approaching and leaving the locked frame + instead of snapping to a dead stop — smoothing the "stop/dissolve/stop" + without softening the exact pose at the center. + + Pure list math (frames/strengths only). Returns ``(idxs, frames, strengths)`` + parallel lists where ``idxs[i]`` indexes the ORIGINAL keyframe, so the caller + reuses the same image tensor. Neighbors that fall off the clip or onto a + frame already claimed by a center/neighbor are skipped (no clobbering). + """ + n = len(insert_frames) + if ease <= 0 or n == 0: + return list(range(n)), list(insert_frames), [float(s) for s in strengths] + + stride = max(1, int(stride)) + claimed = set(int(f) for f in insert_frames) # centers own their frames + out_idx, out_f, out_s = [], [], [] + # centers first (exact) + for i, (f, s) in enumerate(zip(insert_frames, strengths)): + out_idx.append(i); out_f.append(int(f)); out_s.append(float(s)) + # then ramped neighbors + for i, (f, s) in enumerate(zip(insert_frames, strengths)): + f = int(f); s = float(s) + for k in range(1, int(ease) + 1): + ns = s * (float(falloff) ** k) + for nf in (f - k * stride, f + k * stride): + if nf < 0 or nf > max_frame: + continue + if nf in claimed: + continue + claimed.add(nf) + out_idx.append(i); out_f.append(nf); out_s.append(ns) + return out_idx, out_f, out_s diff --git a/forks/whatdreamscost_koolook/versions/v2_0_2/ltx_director.py b/forks/whatdreamscost_koolook/versions/v2_0_2/ltx_director.py new file mode 100644 index 0000000..39ec948 --- /dev/null +++ b/forks/whatdreamscost_koolook/versions/v2_0_2/ltx_director.py @@ -0,0 +1,1433 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# +# Derived from WhatDreamsCost-ComfyUI `ltx_director.py` +# (https://github.com/WhatDreamsCost/WhatDreamsCost-ComfyUI) at commit +# fe09f73756df202d08341c66b4dc5fc8d2acca22 (pyproject version 2.0.2). +# Vendored VERBATIM from upstream 2.0.2 except for three additive Koolook changes, +# modified by ComfyUI-Koolook on 2026-06-22: +# - namespaced node ID `LTXDirector__koolook` (coexists with installed upstream) +# - `snap_keyframes_to_grid` latent-bucket keyframe snap (issue #258) +# - `keyframe_ease` / `ease_falloff` strength-ramped neighbor pins +# No other behavior is changed: Prompt Relay, audio, and all 2.0.2 features are +# upstream-exact. License: GPL-3.0-or-later (matches the ComfyUI-Koolook pack). + +import logging +import asyncio +import json +import base64 +import io as _io +import math + +import numpy as np +import torch +import torch.nn.functional as F +import av +from PIL import Image + +import os +import platform +import folder_paths +import comfy.model_management +from server import PromptServer +from aiohttp import web + +from comfy_api.latest import io + +from .prompt_relay import ( + get_raw_tokenizer, + map_token_indices, + build_segments, + create_mask_fn, + distribute_segment_lengths, +) + +from .patches import detect_model_type, apply_patches +from .keyframe_grid import latent_count_for_duration as _latent_count_for_duration +from .keyframe_grid import snap_keyframes_to_grid as _snap_keyframes_to_grid +from .keyframe_grid import expand_keyframe_ease as _expand_keyframe_ease + +log = logging.getLogger(__name__) + +# Setup global event loop exception handler to silence ConnectionResetError (WinError 10054/10053) on Windows +try: + loop = None + try: + loop = asyncio.get_event_loop() + except RuntimeError: + try: + loop = asyncio.get_event_loop_policy().get_event_loop() + except Exception: + pass + + if loop is not None: + old_handler = loop.get_exception_handler() + + def silence_connection_reset_handler(loop, context): + exception = context.get('exception') + if (isinstance(exception, (ConnectionResetError, ConnectionAbortedError)) or + (isinstance(exception, OSError) and getattr(exception, 'winerror', None) in (10054, 10053))): + # Suppress WinError 10054 and WinError 10053 tracebacks in logging + return + if old_handler: + old_handler(loop, context) + else: + loop.default_exception_handler(context) + + loop.set_exception_handler(silence_connection_reset_handler) +except Exception: + pass + +# Custom socket type shared with LTXSequencer +GuideData = io.Custom("GUIDE_DATA") +MotionGuideData = io.Custom("MOTION_GUIDE_DATA") + +# --- File Check Endpoint for Deduplication --- +@PromptServer.instance.routes.get("/ltx_director_check_file") +async def ltx_director_check_file(request): + filename = request.query.get("filename", "") + file_size = request.query.get("size", "") + if not filename: + return web.json_response({"exists": False}) + + upload_dir = folder_paths.get_input_directory() + temp_dir = os.path.join(upload_dir, "whatdreamscost") + + # 1. Check if the exact filename exists in whatdreamscost or root input dir + possible_paths = [ + os.path.join(temp_dir, filename), + os.path.join(upload_dir, filename) + ] + + found_path = None + for p in possible_paths: + if os.path.exists(p) and os.path.isfile(p): + if file_size: + try: + if os.path.getsize(p) == int(file_size): + found_path = p + break + except ValueError: + found_path = p + break + else: + found_path = p + break + + if found_path: + rel_name = os.path.relpath(found_path, upload_dir).replace('\\', '/') + return web.json_response({"exists": True, "name": rel_name}) + + # 2. Suffix search if exact match not found + base_name = os.path.basename(filename) + suffix = f"_{base_name}" + try: + for search_dir in [temp_dir, upload_dir]: + if os.path.exists(search_dir): + for f_name in os.listdir(search_dir): + if f_name.endswith(suffix) or f_name == base_name: + pot_path = os.path.join(search_dir, f_name) + if os.path.isfile(pot_path): + if file_size: + try: + if os.path.getsize(pot_path) == int(file_size): + rel_name = os.path.relpath(pot_path, upload_dir).replace('\\', '/') + return web.json_response({"exists": True, "name": rel_name}) + except ValueError: + pass + else: + rel_name = os.path.relpath(pot_path, upload_dir).replace('\\', '/') + return web.json_response({"exists": True, "name": rel_name}) + except Exception as e: + log.warning(f"[LTXDirector] Error listing input directory: {e}") + + return web.json_response({"exists": False}) + + +def read_wav_peaks(wav_path): + import wave + peaks = [] + with wave.open(wav_path, 'rb') as w: + n_frames = w.getnframes() + if n_frames > 0: + frames_bytes = w.readframes(n_frames) + samples = np.frombuffer(frames_bytes, dtype=np.int16) + num_peaks = 200 + step = max(1, len(samples) // num_peaks) + for i in range(num_peaks): + chunk = samples[i * step : (i + 1) * step] + if len(chunk) > 0: + max_val = np.max(np.abs(chunk)) / 32767.0 + peaks.append(float(max_val)) + else: + peaks.append(0.0) + else: + peaks = [0.0] * 200 + return peaks + + +def extract_audio_from_video(video_path): + import wave + try: + base, _ = os.path.splitext(video_path) + output_wav = base + "_extracted_audio.wav" + + # Check if already exists, is not empty, and has the correct 44100Hz sample rate + if os.path.exists(output_wav) and os.path.getsize(output_wav) > 44: + try: + with wave.open(output_wav, 'rb') as w_check: + if w_check.getframerate() == 44100: + peaks = read_wav_peaks(output_wav) + input_dir = folder_paths.get_input_directory() + rel_output = os.path.relpath(output_wav, input_dir).replace('\\', '/') + return rel_output, peaks + except Exception: + pass + + # Decode the video using PyAV + with av.open(video_path) as container: + if not container.streams.audio: + return None, None + stream = container.streams.audio[0] + + # Setup resampler to 44100Hz, Mono, signed 16-bit integer (s16) + resampler = av.AudioResampler( + format='s16', + layout='mono', + rate=44100, + ) + + audio_bytes = bytearray() + + for frame in container.decode(stream): + for resampled_frame in resampler.resample(frame): + arr = resampled_frame.to_ndarray() + audio_bytes.extend(arr.tobytes()) + + # Flush resampler + for resampled_frame in resampler.resample(None): + arr = resampled_frame.to_ndarray() + audio_bytes.extend(arr.tobytes()) + + if not audio_bytes: + return None, None + + # Write WAV file + with wave.open(output_wav, 'wb') as w: + w.setnchannels(1) + w.setsampwidth(2) # 16-bit + w.setframerate(44100) + w.writeframes(audio_bytes) + + # Calculate peaks + peaks = [] + samples = np.frombuffer(audio_bytes, dtype=np.int16) + num_peaks = 200 + step = max(1, len(samples) // num_peaks) + for i in range(num_peaks): + chunk = samples[i * step : (i + 1) * step] + if len(chunk) > 0: + max_val = np.max(np.abs(chunk)) / 32767.0 + peaks.append(float(max_val)) + else: + peaks.append(0.0) + + input_dir = folder_paths.get_input_directory() + rel_output = os.path.relpath(output_wav, input_dir).replace('\\', '/') + return rel_output, peaks + except Exception as e: + print(f"[LTXDirector] Server audio extraction failed: {e}") + return None, None + + +def get_audio_peaks(audio_path): + import wave + # If it is already a WAV file, read peaks directly + _, ext = os.path.splitext(audio_path) + if ext.lower() == ".wav": + try: + return read_wav_peaks(audio_path) + except Exception: + pass # fallback to PyAV + + # Use PyAV to decode and resample the audio file + try: + with av.open(audio_path) as container: + if not container.streams.audio: + return None + stream = container.streams.audio[0] + resampler = av.AudioResampler( + format='s16', + layout='mono', + rate=8000, + ) + audio_bytes = bytearray() + for frame in container.decode(stream): + for resampled_frame in resampler.resample(frame): + arr = resampled_frame.to_ndarray() + audio_bytes.extend(arr.tobytes()) + for resampled_frame in resampler.resample(None): + arr = resampled_frame.to_ndarray() + audio_bytes.extend(arr.tobytes()) + + if not audio_bytes: + return None + + peaks = [] + samples = np.frombuffer(audio_bytes, dtype=np.int16) + num_peaks = 200 + step = max(1, len(samples) // num_peaks) + for i in range(num_peaks): + chunk = samples[i * step : (i + 1) * step] + if len(chunk) > 0: + max_val = np.max(np.abs(chunk)) / 32767.0 + peaks.append(float(max_val)) + else: + peaks.append(0.0) + return peaks + except Exception as e: + print(f"[LTXDirector] Failed to get audio peaks via PyAV: {e}") + return None + + +@PromptServer.instance.routes.get("/ltx_director_get_audio") +async def ltx_director_get_audio(request): + filename = request.query.get("filename") + if not filename: + return web.json_response({"error": "Missing filename"}, status=400) + + upload_dir = folder_paths.get_input_directory() + + clean_filename = filename.replace('\\', '/') + file_path = os.path.join(upload_dir, clean_filename) + if not os.path.exists(file_path): + basename = os.path.basename(clean_filename) + temp_path = os.path.join(upload_dir, "whatdreamscost", basename) + if os.path.exists(temp_path): + file_path = temp_path + else: + file_path = os.path.join(upload_dir, basename) + + if not os.path.exists(file_path) or not os.path.isfile(file_path): + return web.json_response({"error": "File not found"}, status=404) + + _, ext = os.path.splitext(file_path) + is_audio = ext.lower() in [".wav", ".mp3", ".ogg", ".flac", ".m4a"] + + if is_audio: + peaks = None + try: + peaks = get_audio_peaks(file_path) + except Exception as e: + print(f"[LTXDirector] Failed to get audio peaks for audio file: {e}") + + rel_path = os.path.relpath(file_path, upload_dir).replace('\\', '/') + return web.json_response({ + "audio_file": rel_path, + "peaks": peaks + }) + + audio_file, peaks = None, None + try: + loop = asyncio.get_event_loop() + audio_file, peaks = await loop.run_in_executor(None, extract_audio_from_video, file_path) + except Exception as e: + print(f"[LTXDirector] Error extracting audio: {e}") + + return web.json_response({ + "audio_file": audio_file, + "peaks": peaks + }) + + +@PromptServer.instance.routes.get("/ltx_director_open_folder") +async def ltx_director_open_folder(request): + upload_dir = os.path.join(folder_paths.get_input_directory(), "whatdreamscost") + os.makedirs(upload_dir, exist_ok=True) + try: + if hasattr(os, "startfile"): + os.startfile(upload_dir) + else: + import webbrowser + webbrowser.open(os.path.abspath(upload_dir)) + return web.json_response({"success": True}) + except Exception as e: + print(f"[LTXDirector] Failed to open workspace folder: {e}") + return web.json_response({"success": False, "error": str(e)}, status=500) + + +def _read_and_write_file_chunk(file, file_path, mode): + chunk_bytes = file.file.read() + with open(file_path, mode) as f: + f.write(chunk_bytes) + + +# --- LTX Director Chunked Video Upload Endpoint --- +# Bypasses the 413 Payload Too Large error for large video files. +# This endpoint is self-contained and independent of any other node. +@PromptServer.instance.routes.post("/ltx_director_upload_chunk") +async def ltx_director_upload_chunk(request): + post = await request.post() + file = post.get("file") + filename = post.get("filename") + chunk_index = int(post.get("chunk_index")) + total_chunks = int(post.get("total_chunks")) + + upload_dir = os.path.join(folder_paths.get_input_directory(), "whatdreamscost") + os.makedirs(upload_dir, exist_ok=True) + + # Sanitize filename to prevent path traversal attacks (e.g. ../../etc/passwd) + filename = os.path.basename(filename) + file_path = os.path.join(upload_dir, filename) + + # Belt-and-suspenders: confirm the resolved path is still inside the upload directory + if not os.path.realpath(file_path).startswith(os.path.realpath(upload_dir)): + return web.json_response({"error": "Invalid filename"}, status=400) + + # Append chunk to file (write fresh on first chunk, append on subsequent) + mode = "ab" if chunk_index > 0 else "wb" + + # Offload the blocking read/write disk I/O to a thread executor + loop = asyncio.get_event_loop() + await loop.run_in_executor(None, _read_and_write_file_chunk, file, file_path, mode) + + if chunk_index == total_chunks - 1: + audio_file, peaks = None, None + try: + audio_file, peaks = await loop.run_in_executor(None, extract_audio_from_video, file_path) + except Exception as e: + print(f"[LTXDirector] Error in final chunk audio extraction: {e}") + + return web.json_response({ + "name": f"whatdreamscost/{filename}", + "audio_file": audio_file, + "peaks": peaks + }) + return web.json_response({"status": "ok"}) + + + +def _load_image_tensor(seg: dict) -> torch.Tensor: + """Decode an image from the ComfyUI input folder (if imageFile provided) or fallback to base64 + to a ComfyUI-style image tensor of shape [1, H, W, 3], float32 in [0, 1].""" + if seg.get("imageFile"): + file_path = os.path.join(folder_paths.get_input_directory(), seg["imageFile"]) + if os.path.exists(file_path): + img = Image.open(file_path).convert("RGB") + arr = np.array(img, dtype=np.float32) / 255.0 + return torch.from_numpy(arr).unsqueeze(0) + + b64_str = seg.get("imageB64", "") + if not b64_str or b64_str.startswith("/view?"): + return torch.zeros((1, 512, 512, 3), dtype=torch.float32) + + if "," in b64_str: + b64_str = b64_str.split(",", 1)[1] + + try: + img_bytes = base64.b64decode(b64_str) + img = Image.open(_io.BytesIO(img_bytes)).convert("RGB") + arr = np.array(img, dtype=np.float32) / 255.0 + return torch.from_numpy(arr).unsqueeze(0) + except: + return torch.zeros((1, 512, 512, 3), dtype=torch.float32) + +def _load_video_tensor(seg: dict, frame_rate: float) -> torch.Tensor: + """Extracts a sequence of frames from a video file based on the segment's trim parameters, + and returns them as an [N, H, W, 3] float32 tensor.""" + file_path = os.path.join(folder_paths.get_input_directory(), seg.get("imageFile", "")) + + if not os.path.exists(file_path): + return torch.zeros((1, 512, 512, 3), dtype=torch.float32) + + trim_start_frames = float(seg.get("trimStart", 0)) + length_frames = float(seg.get("length", 1)) + start_sec = trim_start_frames / frame_rate + + frames = [] + try: + with av.open(file_path) as container: + stream = container.streams.video[0] + stream.thread_type = "AUTO" + + # Seek slightly before target to hit a keyframe + if stream.time_base: + seek_pts = int((max(0, start_sec - 0.5)) / float(stream.time_base)) + else: + seek_pts = int((max(0, start_sec - 0.5)) * av.time_base) + + container.seek(seek_pts, stream=stream, backward=True) + + for frame in container.decode(stream): + frame_time = frame.time + if frame_time is None and frame.pts is not None and stream.time_base: + frame_time = float(frame.pts * stream.time_base) + + if frame_time is None: + frame_time = 0.0 + + if frame_time < start_sec - 0.01: + continue + + frames.append(frame.to_ndarray(format='rgb24')) + + if len(frames) >= int(length_frames): + break + except Exception as e: + log.warning(f"[PromptRelay] Video extract error: {e}") + + if not frames: + return torch.zeros((1, 512, 512, 3), dtype=torch.float32) + + frames_np = np.array(frames, dtype=np.float32) / 255.0 + return torch.from_numpy(frames_np) + +def _resize_image(tensor: torch.Tensor, target_w: int, target_h: int, method: str, divisible_by: int) -> torch.Tensor: + """Resize an [N, H, W, 3] float32 tensor to target dimensions using the given method, + then snap the final dimensions to be divisible by `divisible_by`.""" + + def snap(val, div): + return max(div, (val // div) * div) + + tw = snap(target_w, divisible_by) + th = snap(target_h, divisible_by) + + N, H, W, C = tensor.shape + if H == th and W == tw: + return tensor + + t_nchw = tensor.permute(0, 3, 1, 2) + + if method == "stretch to fit": + resized = F.interpolate(t_nchw, size=(th, tw), mode="bilinear", align_corners=False) + + elif method == "maintain aspect ratio": + ratio = min(tw / W, th / H) + new_w = snap(int(W * ratio), divisible_by) + new_h = snap(int(H * ratio), divisible_by) + resized = F.interpolate(t_nchw, size=(new_h, new_w), mode="bilinear", align_corners=False) + + elif method == "pad" or method == "pad green": + ratio = min(tw / W, th / H) + new_w = snap(int(W * ratio), divisible_by) + new_h = snap(int(H * ratio), divisible_by) + inner = F.interpolate(t_nchw, size=(new_h, new_w), mode="bilinear", align_corners=False) + + pad_l = (tw - new_w) // 2 + pad_t = (th - new_h) // 2 + + if method == "pad green": + resized = torch.zeros((N, C, th, tw), dtype=t_nchw.dtype, device=t_nchw.device) + # #66FF00 is roughly R: 102/255, G: 255/255, B: 0 + resized[:, 0, :, :] = 102 / 255.0 + resized[:, 1, :, :] = 1.0 + resized[:, 2, :, :] = 0.0 + resized[:, :, pad_t:pad_t+new_h, pad_l:pad_l+new_w] = inner + else: + resized = F.pad(inner, (pad_l, tw - new_w - pad_l, pad_t, th - new_h - pad_t), mode="constant", value=0) + + elif method == "crop": + ratio = max(tw / W, th / H) + new_w = int(W * ratio) + new_h = int(H * ratio) + inner = F.interpolate(t_nchw, size=(new_h, new_w), mode="bilinear", align_corners=False) + + left = (new_w - tw) // 2 + top = (new_h - th) // 2 + resized = inner[:, :, top:top+th, left:left+tw] + + else: + resized = F.interpolate(t_nchw, size=(th, tw), mode="bilinear", align_corners=False) + + return resized.permute(0, 2, 3, 1) + + +def _compress_image(tensor: torch.Tensor, crf: int) -> torch.Tensor: + """Apply H.264 compression artefacts to an [N, H, W, 3] float32 tensor (ComfyUI image format). + crf=0 means no compression. Uses PyAV to encode/decode frames in-memory.""" + if crf == 0: + return tensor + + N, H, W, C = tensor.shape + + # Dimensions must be even for H.264 + h = (H // 2) * 2 + w = (W // 2) * 2 + + # uint8 [N, H, W, 3] + tensor_bytes = (tensor[:, :h, :w, :] * 255.0).byte().cpu().numpy() + + try: + buf = _io.BytesIO() + container = av.open(buf, mode="w", format="mp4") + stream = container.add_stream("libx264", rate=24) + stream.width = w + stream.height = h + stream.pix_fmt = "yuv420p" + stream.options = {"crf": str(crf), "preset": "ultrafast"} + + for i in range(N): + frame = av.VideoFrame.from_ndarray(tensor_bytes[i], format="rgb24") + for pkt in stream.encode(frame): + container.mux(pkt) + + for pkt in stream.encode(None): + container.mux(pkt) + + container.close() + + buf.seek(0) + container_r = av.open(buf, mode="r") + decoded = [frame_r.to_ndarray(format="rgb24") for frame_r in container_r.decode(video=0)] + container_r.close() + + if not decoded: + return tensor + + decoded_np = np.stack(decoded).astype(np.float32) / 255.0 + + # Re-embed into original tensor shape (may have been cropped by even-rounding) + out = tensor.clone() + dec_N = min(N, len(decoded)) + out[:dec_N, :h, :w] = torch.from_numpy(decoded_np[:dec_N]).to(tensor.device, tensor.dtype) + + return out + + except Exception as e: + log.warning("[PromptRelay] img_compression encode/decode failed: %s", e) + return tensor + + +def _build_combined_audio(timeline_data_str: str, start_frame: int, duration_frames: int, frame_rate: float, override_audio: bool = False) -> dict: + """Parses timeline JSON, loads/trims audio directly from memory using PyAV, + and aligns to a global timeline yielding ComfyUI's format. + Output length explicitly mimics the timeline's duration_frames length.""" + target_sr = 44100 + total_samples = max(1, int(math.ceil(duration_frames / frame_rate * target_sr))) + empty_audio = {"waveform": torch.zeros((1, 2, total_samples), dtype=torch.float32), "sample_rate": target_sr} + + if not timeline_data_str: + return empty_audio + + try: + data = json.loads(timeline_data_str) + is_retake = data.get("retakeMode", False) + if is_retake and data.get("retakeVideo"): + retake_vid = data.get("retakeVideo") + audio_segs = [{ + "videoFile": retake_vid.get("imageFile") or retake_vid.get("fileName"), + "audioFile": retake_vid.get("imageFile") or retake_vid.get("fileName"), + "start": 0, + "length": retake_vid.get("videoDurationFrames", duration_frames), + "trimStart": 0 + }] + override_audio = True + elif override_audio: + audio_segs = data.get("motionSegments", []) + else: + audio_segs = data.get("audioSegments", []) + except Exception: + return empty_audio + + if not audio_segs: + return empty_audio + + out_waveform = torch.zeros((2, total_samples), dtype=torch.float32) + + for seg in audio_segs: + buffer = None + file_key = "videoFile" if override_audio else "audioFile" + if seg.get(file_key): + file_path = os.path.join(folder_paths.get_input_directory(), seg[file_key]) + if not os.path.exists(file_path): + # Try fallback under whatdreamscost subfolder + basename = os.path.basename(seg[file_key]) + fallback_path = os.path.join(folder_paths.get_input_directory(), "whatdreamscost", basename) + if os.path.exists(fallback_path): + file_path = fallback_path + + if os.path.exists(file_path): + with open(file_path, "rb") as f: + buffer = _io.BytesIO(f.read()) + + if not override_audio and not buffer and seg.get("audioB64"): + b64 = seg.get("audioB64") + if "," in b64: + b64 = b64.split(",", 1)[1] + try: + audio_bytes = base64.b64decode(b64) + buffer = _io.BytesIO(audio_bytes) + except: + pass + + if not buffer: + continue + + try: + clip_frames = [] + + # Use PyAV to decode directly from memory buffer + with av.open(buffer) as container: + if not container.streams.audio: + continue + stream = container.streams.audio[0] + + # Setup resampler to ensure output is 44.1kHz, Stereo, Float32 Planar + resampler = av.AudioResampler( + format='fltp', + layout='stereo', + rate=target_sr, + ) + + for frame in container.decode(stream): + for resampled_frame in resampler.resample(frame): + # to_ndarray() on fltp gives shape (channels, samples) + arr = resampled_frame.to_ndarray() + clip_frames.append(torch.from_numpy(arr)) + + # Flush the resampler to get any remaining samples + for resampled_frame in resampler.resample(None): + arr = resampled_frame.to_ndarray() + clip_frames.append(torch.from_numpy(arr)) + + if not clip_frames: + continue + + # Concatenate all frame blocks along the samples dimension (dim 1) + waveform = torch.cat(clip_frames, dim=1) # Shape: [2, total_clip_samples] + + # Calculate interactive trim boundaries + trim_start_frames = float(seg.get("trimStart", 0)) + length_frames = float(seg.get("length", 1)) + start_frames = float(seg.get("start", 0)) + + if start_frames + length_frames <= start_frame: + continue + + offset = max(0, start_frame - start_frames) + trim_start_frames += offset + length_frames = max(1, length_frames - offset) + start_frames = max(0, start_frames - start_frame) + + start_sample_src = int(trim_start_frames / frame_rate * target_sr) + length_samples = int(length_frames / frame_rate * target_sr) + end_sample_src = start_sample_src + length_samples + + if start_sample_src < 0: start_sample_src = 0 + if end_sample_src > waveform.shape[1]: + end_sample_src = waveform.shape[1] + + actual_length = end_sample_src - start_sample_src + if actual_length <= 0: continue + + # Extract the correct segment of the audio + clip_waveform = waveform[:, start_sample_src:end_sample_src] + + # Position onto the timeline + start_sample_dst = int(start_frames / frame_rate * target_sr) + + if start_sample_dst >= out_waveform.shape[1]: + continue + + end_sample_dst = start_sample_dst + actual_length + + # Clip any trailing overflow so we don't index past the timeline bounds + if end_sample_dst > out_waveform.shape[1]: + actual_length = out_waveform.shape[1] - start_sample_dst + clip_waveform = clip_waveform[:, :actual_length] + end_sample_dst = start_sample_dst + actual_length + + if actual_length <= 0: + continue + + # Additive composite (allows clips overlapping to sum together naturally) + out_waveform[:, start_sample_dst:end_sample_dst] += clip_waveform + + except Exception as e: + log.warning("[PromptRelay] Audio process error for segment %s: %s", seg.get("fileName"), e) + continue + + return {"waveform": out_waveform.unsqueeze(0), "sample_rate": target_sr} + + +def _convert_to_latent_lengths(pixel_lengths, temporal_stride, latent_frames): + """Convert pixel-space segment lengths to integer latent-space lengths using the + largest-remainder method. Targets the full `latent_frames` when the pixel sum looks + like full coverage (within one stride of latent_frames * stride). Otherwise targets + round(total_pixel / temporal_stride) so partial-coverage timelines stay partial. + """ + if not pixel_lengths: + return [] + total_pixel = sum(pixel_lengths) + if total_pixel <= 0: + return [1] * len(pixel_lengths) + + naive_total = max(1, round(total_pixel / temporal_stride)) + target_total = min(latent_frames, naive_total) + # Within one frame of full → user clearly intended full coverage; pin to latent_frames. + if target_total >= latent_frames - 1: + target_total = latent_frames + + exact = [p * target_total / total_pixel for p in pixel_lengths] + result = [int(e) for e in exact] + diff = target_total - sum(result) + if diff > 0: + order = sorted(range(len(exact)), key=lambda i: -(exact[i] - int(exact[i]))) + for k in range(diff): + result[order[k % len(order)]] += 1 + + # Ensure every segment has ≥ 1 latent frame (steal from the largest if needed). + for i in range(len(result)): + if result[i] < 1: + max_idx = max(range(len(result)), key=lambda j: result[j]) + if result[max_idx] > 1: + result[max_idx] -= 1 + result[i] = 1 + + return result + + +def _encode_relay(model, clip, latent, global_prompt, local_prompts, segment_lengths, epsilon): + for name, val in (("global_prompt", global_prompt), + ("local_prompts", local_prompts), + ("segment_lengths", segment_lengths)): + if val is None: + raise ValueError( + f"PromptRelay: '{name}' arrived as None. " + "Likely causes: a stale workflow JSON saved with null, the timeline " + "editor's web extension failing to load, or an upstream node returning None. " + "Set the field to an empty string or fix the upstream connection." + ) + + # Split prompts but do NOT filter out empty ones yet, so we can detect them + locals_list = [p.strip() for p in local_prompts.split("|")] + + # If there are no visual segments on the timeline (e.g., only using IC-LoRA motion track), + # bypass the local prompt chunking entirely and just use the global prompt. + if not locals_list or (len(locals_list) == 1 and not locals_list[0]): + log.info("[PromptRelay] No local segments found. Using global prompt exclusively.") + conditioning = clip.encode_from_tokens_scheduled(clip.tokenize(global_prompt)) + return model.clone(), conditioning + + # Check if any specific segment is empty and apply fallbacks + for i, p in enumerate(locals_list): + if not p: + fallback = global_prompt.strip() if global_prompt else "video" + if not fallback: + fallback = "video" + locals_list[i] = fallback + + arch, patch_size, temporal_stride = detect_model_type(model) + + samples = latent["samples"] + latent_frames = samples.shape[2] + tokens_per_frame = (samples.shape[3] // patch_size[1]) * (samples.shape[4] // patch_size[2]) + + parsed_lengths = None + if segment_lengths.strip(): + pixel_lengths = [int(float(x.strip())) for x in segment_lengths.split(",") if x.strip()] + parsed_lengths = _convert_to_latent_lengths(pixel_lengths, temporal_stride, latent_frames) + + raw_tokenizer = get_raw_tokenizer(clip) + full_prompt, token_ranges = map_token_indices(raw_tokenizer, global_prompt, locals_list) + + log.info("[PromptRelay] Global: tokens [0:%d] (%d tokens)", token_ranges[0][0], token_ranges[0][0]) + for i, (s, e) in enumerate(token_ranges): + log.info("[PromptRelay] Segment %d: tokens [%d:%d] (%d tokens)", i, s, e, e - s) + + conditioning = clip.encode_from_tokens_scheduled(clip.tokenize(full_prompt)) + + effective_lengths = distribute_segment_lengths(len(locals_list), latent_frames, parsed_lengths) + + log.info( + "[PromptRelay] Latent: %d frames, %d tokens/frame, segments: %s", + latent_frames, tokens_per_frame, effective_lengths, + ) + + q_token_idx = build_segments(token_ranges, effective_lengths, epsilon, None) + mask_fn = create_mask_fn(q_token_idx, tokens_per_frame, latent_frames) + + patched = model.clone() + apply_patches(patched, arch, mask_fn) + + return patched, conditioning + + +class LTXDirector(io.ComfyNode): + """WYSIWYG timeline variant — segments and lengths come from a visual editor in the node UI.""" + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LTXDirector__koolook", + display_name="LTX Director (Koolook)", + category="Koolook/PromptRelay", + description=( + "Same as Prompt Relay Encode, but local prompts and segment lengths are edited " + "visually as draggable blocks on a timeline. The duration_frames input only sets the " + "timeline scale (pixel space) — actual frame count is still read from the latent." + ), + inputs=[ + io.Model.Input("model"), + io.Clip.Input("clip"), + io.Vae.Input("audio_vae", optional=True, tooltip="Optional. Connect an Audio VAE to generate audio latents."), + io.Latent.Input("optional_latent", optional=True, tooltip="Optional. Connect a latent to override the auto-generated one."), + io.String.Input( + "global_prompt", multiline=True, default="", force_input=True, optional=True, + tooltip="Conditions the entire video. Anchors persistent characters, objects, and scene context.", + ), + io.Float.Input( + "start_second", default=0.0, min=0.0, max=1000.0, step=0.01, + tooltip="Start time in seconds of the timeline generation.", + ), + io.Float.Input( + "end_second", default=5.0, min=0.0, max=1000.0, step=0.01, + tooltip="End time in seconds of the timeline generation.", + ), + io.Float.Input( + "duration_seconds", default=5.0, min=0.1, max=1000.0, step=0.01, + tooltip="Total timeline duration in seconds (computed/synced from frames).", + ), + io.Int.Input( + "start_frame", default=0, min=0, max=10000, step=1, + tooltip="Start frame of the timeline generation.", + ), + io.Int.Input( + "end_frame", default=120, min=1, max=10000, step=1, + tooltip="End frame of the timeline generation.", + ), + io.Int.Input( + "duration_frames", default=120, min=1, max=10000, step=1, + tooltip="Total timeline length in pixel-space frames. Used by the editor for visual scale only.", + ), + io.String.Input( + "timeline_data", default="", + tooltip="JSON state of the timeline editor (auto-managed; do not edit by hand).", + ), + io.Boolean.Input( + "use_custom_audio", default=False, optional=True, + tooltip="Toggle between using timeline audio (ON) and generating audio from scratch (OFF).", + ), + io.Boolean.Input( + "use_custom_motion", default=True, optional=True, + tooltip="Toggle between using timeline motion guidance (ON) and ignoring motion video segments (OFF).", + ), + io.Boolean.Input( + "inpaint_audio", default=True, optional=True, + tooltip="Toggle whether empty gaps in the audio track are inpainted with generated audio.", + ), + io.String.Input( + "local_prompts", multiline=True, default="", + tooltip="Auto-populated from the timeline editor.", + ), + io.String.Input( + "segment_lengths", default="", + tooltip="Auto-populated from the timeline editor (pixel-space frame counts).", + ), + io.Float.Input( + "epsilon", default=0.001, min=0.0001, max=0.99, step=0.0001, + tooltip="Penalty decay parameter. Values below ~0.1 all produce sharp boundaries (paper default 0.001). For softer transitions, try 0.5 or higher.", + ), + io.Float.Input( + "frame_rate", default=24, min=1, max=240, step=1, optional=True, + tooltip="Frames per second — only affects how time is displayed in the timeline editor when time_units is set to 'seconds'.", + ), + io.Combo.Input( + "display_mode", options=["frames", "seconds"], default="seconds", optional=True, + tooltip="Display the ruler, segment ranges, length input, and total in frames or seconds. Internal storage is always pixel-space frames.", + ), + io.String.Input( + "guide_strength", default="", + tooltip="Auto-populated from the timeline editor (comma-separated guide strengths for image segments).", + ), + io.Int.Input( + "custom_width", default=0, min=0, max=8192, step=1, optional=True, + tooltip="Target output width for all image segments. Set to 0 to use the original image width.", + ), + io.Int.Input( + "custom_height", default=0, min=0, max=8192, step=1, optional=True, + tooltip="Target output height for all image segments. Set to 0 to use the original image height.", + ), + io.Combo.Input( + "resize_method", + options=["maintain aspect ratio", "stretch to fit", "pad", "pad green", "crop"], + default="maintain aspect ratio", + optional=True, + tooltip="How to resize image segments to fit the target dimensions.", + ), + io.Int.Input( + "divisible_by", default=32, min=1, max=256, step=1, optional=True, + tooltip="Snap the final output image dimensions to be divisible by this number (e.g. 32 for LTX).", + ), + io.Int.Input( + "img_compression", default=18, min=0, max=100, step=1, optional=True, + tooltip="H.264 CRF compression to apply to each guide image. 0 = no compression, higher = more artefacts.", + ), + io.Boolean.Input( + "override_audio", default=False, optional=True, + tooltip="Use the audio from the IC-LoRA video instead of using the audio track.", + ), + io.Boolean.Input( + "snap_keyframes_to_grid", default=True, optional=True, + tooltip=( + "Koolook (issue #258): snap each image keyframe to the center " + "of its LTX latent-time bucket so hard pins land cleanly on one " + "latent frame, and bump pins that collide in the same bucket. " + "Off = use raw timeline positions." + ), + ), + io.Int.Input( + "keyframe_ease", default=0, min=0, max=4, optional=True, + tooltip=( + "Koolook: ease in/out of each hard pin. 0 = off (single " + "frozen frame -> the robotic stop/dissolve). 1-2 adds " + "strength-ramped neighbor pins of the SAME pose one latent " + "bucket apart, so the model glides into and out of the " + "locked pose instead of snapping. Center pose stays exact; " + "costs extra guide frames (more compute)." + ), + ), + io.Float.Input( + "ease_falloff", default=0.5, min=0.0, max=1.0, step=0.05, optional=True, + tooltip=( + "Strength multiplier per ease step: neighbor k gets " + "center_strength * falloff**k. Lower = quicker release " + "(less dwell), higher = gentler/longer ease." + ), + ), + ], + outputs=[ + io.Model.Output(display_name="model"), + io.Conditioning.Output(display_name="positive"), + io.Latent.Output(display_name="video_latent", tooltip="Auto-generated LTXV empty latent (only populated when no latent is connected)."), + io.Latent.Output(display_name="audio_latent", tooltip="Auto-generated audio latent (uses custom audio if enabled)."), + GuideData.Output(display_name="guide_data"), + MotionGuideData.Output(display_name="motion_guide_data"), + io.Float.Output(display_name="frame_rate", tooltip="The frame rate used for the timeline."), + io.Audio.Output(display_name="combined_audio", tooltip="Combined timeline audio layout."), + ], + ) + + @classmethod + def execute(cls, model, clip, start_second, end_second, duration_seconds, start_frame, end_frame, duration_frames, + timeline_data, local_prompts, segment_lengths, global_prompt="", guide_strength="", epsilon=1e-3, + frame_rate=24, display_mode="seconds", + custom_width=768, custom_height=512, resize_method="maintain aspect ratio", + divisible_by=32, img_compression=0, audio_vae=None, optional_latent=None, + use_custom_audio=False, inpaint_audio=True, use_custom_motion=True, override_audio=False, + snap_keyframes_to_grid=True, keyframe_ease=0, ease_falloff=0.5) -> io.NodeOutput: + + # Parse timeline data + try: + tdata = json.loads(timeline_data) if timeline_data else {} + except Exception as e: + log.error(f"[LTXDirector] execute timeline_data parse error: {e}") + tdata = {} + + is_retake_mode = tdata.get("retakeMode", False) + is_retake_active = is_retake_mode and tdata.get("retakeVideo") is not None + + # Extract global_prompt from timeline_data if not connected/empty + if not global_prompt: + if is_retake_mode: + global_prompt = tdata.get("retake_global_prompt", "") + else: + global_prompt = tdata.get("global_prompt", "") + + log.info(f"[LTXDirector] execute RECEIVED global_prompt: {repr(global_prompt)}") + + # --- Build guide_data from image segments FIRST (to derive output dimensions) --- + guide_data = {"images": [], "insert_frames": [], "strengths": [], "frame_rate": frame_rate} + derived_w, derived_h = custom_width, custom_height + try: + img_segs = [ + s for s in tdata.get("segments", []) + if s.get("type", "image") in ("image", "video") + and (s.get("imageFile") or s.get("imageB64")) + and int(s.get("start", 0)) < start_frame + duration_frames + and int(s.get("start", 0)) + int(s.get("length", 1)) > start_frame + ] + img_segs.sort(key=lambda s: s["start"]) + + strengths = [] + if guide_strength.strip(): + strengths = [float(x.strip()) for x in guide_strength.split(",") if x.strip()] + + for idx, seg in enumerate(img_segs): + seg_start = int(seg.get("start", 0)) + offset = max(0, start_frame - seg_start) + + if seg.get("type") == "video": + if offset > 0: + seg["trimStart"] = float(seg.get("trimStart", 0)) + offset + seg["length"] = max(1, int(seg.get("length", 1)) - offset) + tensor = _load_video_tensor(seg, float(frame_rate)) + else: + tensor = _load_image_tensor(seg) + + # Apply resize + src_h, src_w = tensor.shape[1], tensor.shape[2] + + def snap(val, div): + return max(div, (val // div) * div) + + if custom_width > 0 and custom_height > 0: + # Both dimensions set — apply selected resize_method (pad, crop, stretch, maintain AR) + tensor = _resize_image(tensor, custom_width, custom_height, resize_method, divisible_by) + elif custom_width > 0: + # Width only — scale height from AR, snap both, then resize to exact dimensions + tgt_w = snap(custom_width, divisible_by) + tgt_h = snap(int(src_h * tgt_w / src_w), divisible_by) + tensor = _resize_image(tensor, tgt_w, tgt_h, "stretch to fit", divisible_by) + elif custom_height > 0: + # Height only — scale width from AR, snap both, then resize to exact dimensions + tgt_h = snap(custom_height, divisible_by) + tgt_w = snap(int(src_w * tgt_h / src_h), divisible_by) + tensor = _resize_image(tensor, tgt_w, tgt_h, "stretch to fit", divisible_by) + else: + # Both zero — keep original dimensions, just snap to divisible_by + tensor = _resize_image(tensor, src_w, src_h, "maintain aspect ratio", divisible_by) + + + # Apply compression + if img_compression > 0: + tensor = _compress_image(tensor, img_compression) + + # Record dimensions of the first processed image for latent generation + if idx == 0: + derived_h = tensor.shape[1] + derived_w = tensor.shape[2] + + if seg.get("isEndFrame"): + insert_frame = max(0, seg_start + int(seg.get("length", 1)) - 1 - start_frame) + else: + insert_frame = max(0, seg_start - start_frame) + strength = strengths[idx] if idx < len(strengths) else 1.0 + guide_data["images"].append(tensor) + guide_data["insert_frames"].append(insert_frame) + guide_data["strengths"].append(float(strength)) + + # If no images were loaded from the timeline, create a dummy image at strength 0 + # to prevent artifacts in text-to-video mode. + if not guide_data["images"] and optional_latent is None: + src_w = derived_w if derived_w > 0 else 768 + src_h = derived_h if derived_h > 0 else 512 + + # If there's an IC-LoRA video or retake base video on the timeline, extract its dimensions for accurate aspect ratio scaling + tdata_motion = json.loads(timeline_data) if timeline_data else {} + found_dims = False + + # Check for retake base video first + is_retake = tdata_motion.get("retakeMode", False) + retake_vid = tdata_motion.get("retakeVideo") or {} + retake_file = retake_vid.get("imageFile", "") if isinstance(retake_vid, dict) else "" + if is_retake and retake_file: + r_path = os.path.join(folder_paths.get_input_directory(), retake_file) + if not os.path.exists(r_path): + basename = os.path.basename(retake_file) + fallback_path = os.path.join(folder_paths.get_input_directory(), "whatdreamscost", basename) + if os.path.exists(fallback_path): + r_path = fallback_path + if os.path.exists(r_path): + try: + with av.open(r_path) as container: + stream = container.streams.video[0] + src_w = stream.width or stream.codec_context.width + src_h = stream.height or stream.codec_context.height + found_dims = True + except: + pass + + # Fallback to normal motion segments + if not found_dims: + for mseg in tdata_motion.get("motionSegments", []): + v_file = mseg.get("videoFile") + if v_file: + v_path = os.path.join(folder_paths.get_input_directory(), v_file) + if not os.path.exists(v_path): + basename = os.path.basename(v_file) + fallback_path = os.path.join(folder_paths.get_input_directory(), "whatdreamscost", basename) + if os.path.exists(fallback_path): + v_path = fallback_path + if os.path.exists(v_path): + try: + with av.open(v_path) as container: + stream = container.streams.video[0] + src_w = stream.width or stream.codec_context.width + src_h = stream.height or stream.codec_context.height + found_dims = True + break + except: + pass + + # Create a dummy tensor of the exact source dimensions + tensor = torch.zeros((1, src_h, src_w, 3), dtype=torch.float32) + + def snap(val, div): + return max(div, (val // div) * div) + + # Route the dummy tensor through the exact same resizing pipeline + if custom_width > 0 and custom_height > 0: + tensor = _resize_image(tensor, custom_width, custom_height, resize_method, divisible_by) + elif custom_width > 0: + tgt_w = snap(custom_width, divisible_by) + tgt_h = snap(int(src_h * tgt_w / src_w), divisible_by) + tensor = _resize_image(tensor, tgt_w, tgt_h, "stretch to fit", divisible_by) + elif custom_height > 0: + tgt_h = snap(custom_height, divisible_by) + tgt_w = snap(int(src_w * tgt_h / src_h), divisible_by) + tensor = _resize_image(tensor, tgt_w, tgt_h, "stretch to fit", divisible_by) + else: + tensor = _resize_image(tensor, src_w, src_h, "maintain aspect ratio", divisible_by) + + guide_data["images"].append(tensor) + guide_data["insert_frames"].append(0) + guide_data["strengths"].append(0.0) + + derived_w = tensor.shape[2] + derived_h = tensor.shape[1] + + except Exception as e: + log.warning("[PromptRelay] Could not build guide_data: %s", e) + + # Koolook (issue #258): snap image keyframes to latent-bucket centers so + # hard pins land on one latent frame and don't collide. Pixel in, pixel out + # — the downstream guide node re-derives latent indices. + if snap_keyframes_to_grid and guide_data["insert_frames"]: + try: + _arch, _patch_size, _stride = detect_model_type(model) + _stride = max(1, int(_stride)) + _latent_count = _latent_count_for_duration(duration_frames, _stride) + _snapped, _warns = _snap_keyframes_to_grid( + guide_data["insert_frames"], _stride, _latent_count + ) + guide_data["insert_frames"] = _snapped + for _w in _warns: + log.warning("[LTXDirector] keyframe grid-snap: %s", _w) + except Exception as _e: + log.warning("[LTXDirector] keyframe grid-snap skipped: %s", _e) + + # Koolook: ease in/out of each hard pin (smooth the robotic stop/dissolve) + # by emitting strength-ramped neighbor pins of the same pose. Center pose + # stays exact; default keyframe_ease=0 is a no-op. + if keyframe_ease and int(keyframe_ease) > 0 and guide_data["images"]: + try: + _arch, _ps, _stride = detect_model_type(model) + _stride = max(1, int(_stride)) + _idxs, _frames, _strs = _expand_keyframe_ease( + guide_data["insert_frames"], guide_data["strengths"], + _stride, int(keyframe_ease), float(ease_falloff), int(duration_frames), + ) + guide_data["images"] = [guide_data["images"][i] for i in _idxs] + guide_data["insert_frames"] = _frames + guide_data["strengths"] = _strs + log.info( + "[LTXDirector] keyframe ease: expanded to %d pins " + "(ease=%d, falloff=%.2f)", len(_idxs), int(keyframe_ease), float(ease_falloff), + ) + except Exception as _e: + log.warning("[LTXDirector] keyframe ease skipped: %s", _e) + + # --- Auto-generate LTXV latent if none was provided --- + # Apply the community 8n+1 rule directly to the timeline's duration_frames: + # int(ceil(((duration_frames) - 1) / 8) * 8) + 1 + # This ensures we get AT LEAST the requested frames, snapped to LTXV's requirements. + ltxv_length = int(math.ceil((duration_frames - 1) / 8.0) * 8) + 1 + + if optional_latent is None: + latent_w = max(32, (derived_w // 32) * 32) + latent_h = max(32, (derived_h // 32) * 32) + # LTXV temporal: ((length - 1) // 8) + 1 latent frames; invert to get pixel frames -> length + latent_t = ((ltxv_length - 1) // 8) + 1 + samples = torch.zeros( + [1, 128, latent_t, latent_h // 32, latent_w // 32], + device=comfy.model_management.intermediate_device(), + ) + latent = {"samples": samples} + log.info( + "[PromptRelay] Auto-generated LTXV latent: %dx%d, %d pixel frames (%d latent frames)", + latent_w, latent_h, ltxv_length, latent_t, + ) + else: + latent = optional_latent + + patched, conditioning = _encode_relay( + model, clip, latent, global_prompt, local_prompts, segment_lengths, epsilon, + ) + + # --- Build Audio Output --- + audio_out = _build_combined_audio(timeline_data, start_frame, ltxv_length, float(frame_rate), override_audio=override_audio) + + # --- Audio Latent Generation --- + audio_latent = {} + + if audio_vae is not None: + # Helper to generate empty latent + def get_empty_latent(): + # Support both raw AudioVAE objects and ComfyUI VAE wrappers. + inner = getattr(audio_vae, "first_stage_model", audio_vae) + z_channels = audio_vae.latent_channels + audio_freq = inner.latent_frequency_bins + num_audio_latents = inner.num_of_latents_from_frames(ltxv_length, float(frame_rate)) + audio_latents = torch.zeros( + (1, z_channels, num_audio_latents, audio_freq), + device=comfy.model_management.intermediate_device(), + ) + return {"samples": audio_latents, "type": "audio"} + + if use_custom_audio or override_audio or is_retake_active: + try: + if audio_out is not None: + # 1. Encode audio waveform into latent space + waveform = audio_out["waveform"] + if waveform.ndim == 2: + waveform = waveform.unsqueeze(0) + if waveform.ndim != 3: + raise ValueError( + f"Expected custom audio waveform with 2 or 3 dims, got shape {tuple(waveform.shape)}" + ) + + # Wrapped ComfyUI VAE expects (batch, samples, channels); + # raw AudioVAE expects a dict with waveform in (batch, channels, samples). + if hasattr(audio_vae, "first_stage_model"): + latent_samples = audio_vae.encode(waveform.movedim(1, -1)) + else: + latent_samples = audio_vae.encode({ + "waveform": waveform, + "sample_rate": audio_out["sample_rate"], + }) + + if latent_samples.numel() == 0: + raise ValueError("Encoded audio latent is empty (0 elements).") + + # 2. Create a 3D gap mask [B, F, H] to avoid accidental broadcasting to the 5D video latent + # which also has 128 channels. A 4D audio mask [1, 128, F, H] confuses ComfyUI's KSampler + # into masking the video latent as well, causing black frames. + B, C, F_len, H_len = latent_samples.shape + + if is_retake_active: + gap_mask = torch.zeros((B, F_len, H_len), dtype=torch.float32, device=latent_samples.device) + + retake_start = float(tdata.get("retakeStart", 0)) + retake_len = float(tdata.get("retakeLength", 0)) + + overlap_start = max(start_frame, retake_start) + overlap_end = min(start_frame + ltxv_length, retake_start + retake_len) + + if overlap_end > overlap_start: + rel_start = overlap_start - start_frame + rel_len = overlap_end - overlap_start + + start_sec = rel_start / float(frame_rate) + len_sec = rel_len / float(frame_rate) + total_sec = ltxv_length / float(frame_rate) + + start_idx = int((start_sec / total_sec) * F_len) + end_idx = int(((start_sec + len_sec) / total_sec) * F_len) + + start_idx = max(0, min(F_len, start_idx)) + end_idx = max(0, min(F_len, end_idx)) + + gap_mask[:, start_idx:end_idx, :] = 1.0 + else: + gap_mask = torch.ones((B, F_len, H_len), dtype=torch.float32, device=latent_samples.device) + + audio_segs_key = "motionSegments" if override_audio else "audioSegments" + file_key = "videoFile" if override_audio else "audioFile" + for seg in tdata.get(audio_segs_key, []): + if not seg.get(file_key): + continue + + seg_start = float(seg.get("start", 0)) + seg_len = float(seg.get("length", 1)) + + if seg_start + seg_len <= start_frame or seg_start >= start_frame + ltxv_length: + continue + + offset = max(0, start_frame - seg_start) + seg_len = max(1.0, seg_len - offset) + seg_start = max(0, seg_start - start_frame) + + start_sec = seg_start / float(frame_rate) + len_sec = seg_len / float(frame_rate) + total_sec = ltxv_length / float(frame_rate) + + start_idx = int((start_sec / total_sec) * F_len) + end_idx = int(((start_sec + len_sec) / total_sec) * F_len) + gap_mask[:, start_idx:end_idx, :] = 0.0 + + if inpaint_audio: + # Generate new audio in the gaps, preserve custom audio segments + mask = gap_mask + else: + # Preserve the entire audio latent (no generation). + # We use a 3D zeros mask to prevent video blackouts. + mask = torch.zeros((B, F_len, H_len), dtype=torch.float32, device=latent_samples.device) + + audio_latent = { + "samples": latent_samples, + "type": "audio", + "noise_mask": mask + } + log.info("[PromptRelay] Generated custom audio latent with dynamic noise mask.") + else: + raise ValueError("No audio waveform to encode.") + except Exception as e: + log.error("[PromptRelay] Failed to generate custom audio latent: %s", e) + raise e + else: + # Generate empty latent + try: + audio_latent = get_empty_latent() + log.info("[PromptRelay] Auto-generated empty audio latent.") + except Exception as e: + log.error("[PromptRelay] Could not generate empty audio latent: %s", e) + raise e + + # --- Motion guide output from timeline video segments --- + motion_guide_data = {"segments": [], "frame_rate": float(frame_rate), "duration_frames": int(duration_frames), "resize_method": resize_method} + try: + tdata = json.loads(timeline_data) if timeline_data else {} + if use_custom_motion: + motion_segments = tdata.get("motionSegments", []) + else: + motion_segments = [] + for seg in motion_segments: + seg_start = int(seg.get("start", 0)) + length = int(seg.get("length", 1)) + if seg_start >= start_frame + duration_frames or seg_start + length <= start_frame: + continue + if not seg.get("videoFile"): + continue + + offset = max(0, start_frame - seg_start) + new_start = max(0, seg_start - start_frame) + + # Trim length so it doesn't extend beyond duration_frames + clipped_len = min(length - offset, duration_frames - new_start) + if clipped_len <= 0: + continue + + clean = dict(seg) + clean["start"] = new_start + clean["length"] = clipped_len + clean["trimStart"] = float(seg.get("trimStart", 0)) + offset + motion_guide_data["segments"].append(clean) + except Exception as e: + log.warning("[LTXDirector] Could not build motion_guide_data: %s", e) + + # Inject raw timeline details for downstream masking in Retake Mode + guide_data["timeline_data"] = timeline_data + guide_data["start_frame"] = start_frame + guide_data["duration_frames"] = duration_frames + guide_data["resize_method"] = resize_method + + return io.NodeOutput(patched, conditioning, latent, audio_latent, guide_data, motion_guide_data, float(frame_rate), audio_out) + + +NODE_CLASS_MAPPINGS = { + "LTXDirector__koolook": LTXDirector, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "LTXDirector__koolook": "LTX Director (Koolook)", +} diff --git a/forks/whatdreamscost_koolook/versions/v2_0_2/patches.py b/forks/whatdreamscost_koolook/versions/v2_0_2/patches.py new file mode 100644 index 0000000..f2655f5 --- /dev/null +++ b/forks/whatdreamscost_koolook/versions/v2_0_2/patches.py @@ -0,0 +1,195 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# +# Vendored VERBATIM from WhatDreamsCost-ComfyUI `patches.py` +# (https://github.com/WhatDreamsCost/WhatDreamsCost-ComfyUI) at commit +# fe09f73756df202d08341c66b4dc5fc8d2acca22 (pyproject version 2.0.2). +# Unmodified; `ltx_director.py` imports detect_model_type / apply_patches. +# License: GPL-3.0-or-later (matches the ComfyUI-Koolook pack). + +import logging +import types + +import comfy.ldm.modules.attention + +log = logging.getLogger(__name__) + + +def _masked_attention(q, k, v, heads, mask, transformer_options={}, **kwargs): + # Bypass wrap_attn (sage/etc may ignore masks) by calling attention_pytorch directly. + return comfy.ldm.modules.attention.attention_pytorch( + q, k, v, heads, mask=mask, + _inside_attn_wrapper=True, + transformer_options=transformer_options, + **kwargs, + ) + + +def _wan_t2v_forward(self, mask_fn, x, context, transformer_options={}, **kwargs): + q = self.norm_q(self.q(x)) + k = self.norm_k(self.k(context)) + v = self.v(context) + + mask = mask_fn(q.shape[1], k.shape[1], q.dtype, q.device, transformer_options) + if mask is not None: + x = _masked_attention(q, k, v, heads=self.num_heads, mask=mask, + transformer_options=transformer_options) + else: + x = comfy.ldm.modules.attention.optimized_attention( + q, k, v, heads=self.num_heads, transformer_options=transformer_options, + ) + return self.o(x) + + +def _wan_i2v_forward(self, mask_fn, x, context, context_img_len, transformer_options={}, **kwargs): + context_img = context[:, :context_img_len] + context_text = context[:, context_img_len:] + + q = self.norm_q(self.q(x)) + + k_img = self.norm_k_img(self.k_img(context_img)) + v_img = self.v_img(context_img) + img_x = comfy.ldm.modules.attention.optimized_attention( + q, k_img, v_img, heads=self.num_heads, transformer_options=transformer_options, + ) + + k = self.norm_k(self.k(context_text)) + v = self.v(context_text) + + mask = mask_fn(q.shape[1], k.shape[1], q.dtype, q.device, transformer_options) + if mask is not None: + x = _masked_attention(q, k, v, heads=self.num_heads, mask=mask, + transformer_options=transformer_options) + else: + x = comfy.ldm.modules.attention.optimized_attention( + q, k, v, heads=self.num_heads, transformer_options=transformer_options, + ) + + return self.o(x + img_x) + + +def _make_masked_override(prev_override): + """transformer_options override that routes mask-bearing attention calls through + attention_pytorch (sage/etc. drop arbitrary masks). Chains to a prior override + when no mask is present so we don't clobber other backends.""" + def override(func, *args, **kwargs): + if kwargs.get("mask") is not None: + return comfy.ldm.modules.attention.attention_pytorch(*args, **kwargs) + if prev_override is not None: + return prev_override(func, *args, **kwargs) + return func(*args, **kwargs) + return override + + +def debug_log(msg): + pass + + +def _make_ltx_mask_wrapper(underlying, mask_fn, attr): + """Wrap an existing LTX cross-attn forward (the default `CrossAttention.forward` + or another node's patch — e.g. KJNodes NAG), injecting PromptRelay's additive + mask via the `mask` kwarg the upstream signature already accepts. + + `underlying` must already be bound to its module — callable as + `underlying(x, context=..., mask=..., ...)`. + """ + def wrapped(_self, x, context=None, mask=None, pe=None, k_pe=None, transformer_options={}): + debug_log(f"wrapped called: x.shape={list(x.shape)} context.shape={list(context.shape) if context is not None else None} mask_is_none={mask is None}") + if context is not None: + opts = {**transformer_options, "promptrelay_attn_type": attr} + pr_mask = mask_fn(x.shape[1], context.shape[1], x.dtype, x.device, opts) + debug_log(f"mask_fn returned pr_mask_is_none={pr_mask is None}") + if pr_mask is not None: + debug_log(f"pr_mask info: shape={list(pr_mask.shape)} min={pr_mask.min().item()} max={pr_mask.max().item()} sum={pr_mask.sum().item()}") + mask = pr_mask if mask is None else mask + pr_mask + + if mask is not None: + prev = transformer_options.get("optimized_attention_override") + transformer_options = { + **transformer_options, + "optimized_attention_override": _make_masked_override(prev), + } + + return underlying( + x, context=context, mask=mask, pe=pe, k_pe=k_pe, + transformer_options=transformer_options, + ) + + wrapped._promptrelay_wrapper = True + return wrapped + + +class _CrossAttnPatch: + """Descriptor that binds (impl, mask_fn) as a method onto a Wan cross-attn module.""" + + def __init__(self, impl, mask_fn): + self.impl = impl + self.mask_fn = mask_fn + + def __get__(self, obj, objtype=None): + impl, mask_fn = self.impl, self.mask_fn + + def wrapped(self_module, *args, **kwargs): + return impl(self_module, mask_fn, *args, **kwargs) + + return types.MethodType(wrapped, obj) + + +def detect_model_type(model): + """Return (arch, patch_size, temporal_stride) for latent geometry. + + temporal_stride is the VAE's pixel→latent temporal compression factor, + used to convert user-facing pixel frame counts to latent frames. + """ + diff_model = model.model.diffusion_model + + if hasattr(diff_model, "patch_size") and not hasattr(diff_model, "patchifier"): + return "wan", tuple(diff_model.patch_size), 4 + + if hasattr(diff_model, "patchifier"): + return "ltx", (1, 1, 1), int(diff_model.vae_scale_factors[0]) + + raise ValueError( + f"Unsupported model type: {type(diff_model).__name__}. " + f"Currently supports Wan and LTX models." + ) + + +def _check_unpatched(model_clone, key): + if key in getattr(model_clone, "object_patches", {}): + raise RuntimeError( + f"PromptRelay: cross-attention forward at '{key}' is already patched by " + "another node. Stacking is not supported for this architecture — remove " + "the conflicting node." + ) + + +def apply_patches(model_clone, arch, mask_fn): + diffusion_model = model_clone.get_model_object("diffusion_model") + + if arch == "wan": + from comfy.ldm.wan.model import WanI2VCrossAttention + for idx, block in enumerate(diffusion_model.blocks): + key = f"diffusion_model.blocks.{idx}.cross_attn.forward" + _check_unpatched(model_clone, key) + cross_attn = block.cross_attn + impl = _wan_i2v_forward if isinstance(cross_attn, WanI2VCrossAttention) else _wan_t2v_forward + model_clone.add_object_patch(key, _CrossAttnPatch(impl, mask_fn).__get__(cross_attn, cross_attn.__class__)) + return + + if arch == "ltx": + to = model_clone.model_options["transformer_options"] + to["promptrelay_mask_fn"] = mask_fn + + for idx, block in enumerate(diffusion_model.transformer_blocks): + for attr in ("attn2", "audio_attn2"): + module = getattr(block, attr, None) + if module is None: + continue + key = f"diffusion_model.transformer_blocks.{idx}.{attr}.forward" + # get_model_object returns the prior patch if present, else the default bound forward. + underlying = model_clone.get_model_object(key) + wrapper = _make_ltx_mask_wrapper(underlying, mask_fn, attr) + model_clone.add_object_patch(key, types.MethodType(wrapper, module)) + return + + raise ValueError(f"Unknown model arch: {arch}") diff --git a/forks/whatdreamscost_koolook/versions/v2_0_2/prompt_relay.py b/forks/whatdreamscost_koolook/versions/v2_0_2/prompt_relay.py new file mode 100644 index 0000000..0b8351f --- /dev/null +++ b/forks/whatdreamscost_koolook/versions/v2_0_2/prompt_relay.py @@ -0,0 +1,262 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# +# Vendored VERBATIM from WhatDreamsCost-ComfyUI `prompt_relay.py` +# (https://github.com/WhatDreamsCost/WhatDreamsCost-ComfyUI) at commit +# fe09f73756df202d08341c66b4dc5fc8d2acca22 (pyproject version 2.0.2). +# Unmodified; `ltx_director.py` imports its Prompt Relay helpers. +# License: GPL-3.0-or-later (matches the ComfyUI-Koolook pack). + +import logging +import math +import torch + +log = logging.getLogger(__name__) + + +def build_temporal_cost(q_token_idx, Lq, Lk, device, dtype, tokens_per_frame): + """Gaussian penalty matrix [Lq, Lk] for video cross-attention (integer frame indexing).""" + offset = torch.zeros(Lq, Lk, device=device, dtype=dtype) + query_frames = torch.arange(Lq, device=device, dtype=torch.long) // tokens_per_frame + + for seg in q_token_idx: + local = seg["local_token_idx"].to(device=device) + d = (query_frames.float()[:, None] - seg["midpoint"]).abs() + strength = seg.get("strength", 1.0) + cost = strength * (torch.relu(d - seg["window"]) ** 2) / (2 * seg["sigma"] ** 2) + offset[:, local] = cost.to(offset.dtype) + + return offset + + +def build_temporal_cost_scaled(q_token_idx, Lq, Lk, device, dtype, latent_frames, is_audio=False): + """Penalty matrix for queries that don't map to integer frames (e.g. LTXAV audio tokens).""" + offset = torch.zeros(Lq, Lk, device=device, dtype=dtype) + query_frames = torch.arange(Lq, device=device, dtype=torch.float32) * latent_frames / Lq + + for seg in q_token_idx: + local = seg["local_token_idx"].to(device=device) + d = (query_frames[:, None] - seg["midpoint"]).abs() + if is_audio: + sigma_val = seg.get("sigma_audio", seg["sigma"]) + window_val = seg.get("window_audio", seg["window"]) + strength_val = seg.get("strength_audio", 1.0) + else: + sigma_val = seg["sigma"] + window_val = seg["window"] + strength_val = seg.get("strength", 1.0) + cost = strength_val * (torch.relu(d - window_val) ** 2) / (2 * sigma_val ** 2) + offset[:, local] = cost.to(offset.dtype) + + return offset + + +def debug_log(msg): + pass + + +def create_mask_fn(q_token_idx, fallback_tokens_per_frame, latent_frames): + """Closure: mask_fn(Lq, Lk, dtype, device, transformer_options) -> additive mask or None. + + Takes shapes/dtype/device instead of tensors so callers can compute the mask + without first materializing q/k projections — required so PromptRelay can + wrap an existing cross-attn forward (e.g. KJNodes NAG) instead of replacing it. + """ + cache = {} + max_token_idx = max(int(seg["local_token_idx"].max().item()) for seg in q_token_idx) + 1 + + def mask_fn(Lq, Lk, dtype, device, transformer_options): + debug_log(f"mask_fn check: Lq={Lq} Lk={Lk} max_token_idx={max_token_idx} cond_or_uncond={transformer_options.get('cond_or_uncond', [])}") + if Lq == Lk: + debug_log("mask_fn: Lq == Lk, returning None") + return None + + # Only apply on conditional pass — not unconditional (negative prompt) + cond_or_uncond = transformer_options.get("cond_or_uncond", []) + if 1 in cond_or_uncond and 0 not in cond_or_uncond: + debug_log("mask_fn: unconditional pass, returning None") + return None + + grid_sizes = transformer_options.get("grid_sizes", None) + attn_type = transformer_options.get("promptrelay_attn_type", "attn2") + is_audio = (attn_type == "audio_attn2") + + if is_audio: + mode = "scaled" + video_lq = -1 + else: + if grid_sizes is not None: + video_tpf = int(grid_sizes[1]) * int(grid_sizes[2]) + else: + if Lq % latent_frames == 0: + video_tpf = Lq // latent_frames + else: + video_tpf = fallback_tokens_per_frame + video_lq = latent_frames * video_tpf + + # Skip cross-modal attention — text keys are padded to a fixed length ≥ max_token_idx and != video_lq + if Lk == video_lq or Lk < max_token_idx: + debug_log(f"mask_fn: Lk == video_lq ({Lk == video_lq}) or Lk < max_token_idx ({Lk < max_token_idx}), returning None") + return None + + mode = "video" if Lq == video_lq else "scaled" + + key = (Lq, Lk, mode, device) + if key not in cache: + if mode == "video": + cost = build_temporal_cost(q_token_idx, Lq, Lk, device, dtype, video_tpf) + else: + cost = build_temporal_cost_scaled(q_token_idx, Lq, Lk, device, dtype, latent_frames, is_audio=is_audio) + log.info( + "[PromptRelay] Built penalty matrix (%s): Lq=%d, Lk=%d, nonzero=%d/%d", + mode, Lq, Lk, (cost > 0).sum().item(), cost.numel(), + ) + debug_log(f"Built penalty matrix ({mode}): Lq={Lq}, Lk={Lk}, key={key}") + cache[key] = -cost + + return cache[key].to(dtype) + + return mask_fn + + +def build_segments(token_ranges, segment_lengths, epsilon=1e-3, relay_options=None): + """Per-segment metadata for the temporal penalty. + + relay_options (optional dict) overrides per-stream knobs: + video_strength, video_window_scale, + audio_epsilon, audio_strength, audio_window_scale + Audio knobs only affect architectures whose cross-attention takes the scaled + (non-integer-frame) path — currently LTX audio_attn2. + """ + # Paper uses constant sigma = 1/ln(1/epsilon) regardless of segment length + sigma = 1.0 / math.log(1.0 / epsilon) if 0 < epsilon < 1 else 0.1448 + + opts = relay_options or {} + v_strength = opts.get("video_strength", 1.0) + v_window_scale = opts.get("video_window_scale", 1.0) + a_epsilon = opts.get("audio_epsilon") + a_strength = opts.get("audio_strength", 1.0) + a_window_scale = opts.get("audio_window_scale", 1.0) + + if a_epsilon is not None and 0 < a_epsilon < 1: + sigma_audio = 1.0 / math.log(1.0 / a_epsilon) + else: + sigma_audio = sigma + + if relay_options: + log.info( + "[PromptRelay] Advanced options active — video: strength=%.3f window_scale=%.3f | " + "audio: epsilon=%s strength=%.3f window_scale=%.3f", + v_strength, v_window_scale, + f"{a_epsilon:.4f}" if a_epsilon is not None else "inherit", + a_strength, a_window_scale, + ) + + q_token_idx = [] + frame_cursor = 0 + + for (tok_start, tok_end), L in zip(token_ranges, segment_lengths): + if L <= 0: + frame_cursor += L + continue + midpoint = (2 * frame_cursor + L) // 2 + base_window = max(L // 2 - 2, 0) + q_token_idx.append({ + "local_token_idx": torch.arange(tok_start, tok_end), + "midpoint": midpoint, + "window": max(base_window * v_window_scale, 0.0), + "sigma": sigma, + "strength": v_strength, + "window_audio": max(base_window * a_window_scale, 0.0), + "sigma_audio": sigma_audio, + "strength_audio": a_strength, + }) + frame_cursor += L + + return q_token_idx + + +def get_raw_tokenizer(clip): + """Extract the raw SPiece/HF tokenizer from a ComfyUI CLIP object.""" + tokenizer_wrapper = clip.tokenizer + for attr_name in dir(tokenizer_wrapper): + if attr_name.startswith("_"): + continue + inner = getattr(tokenizer_wrapper, attr_name, None) + if inner is not None and hasattr(inner, "tokenizer"): + return inner.tokenizer + + raise RuntimeError( + f"Could not find raw tokenizer on CLIP object. " + f"Known attributes: {[a for a in dir(tokenizer_wrapper) if not a.startswith('_')]}" + ) + + +def map_token_indices(raw_tokenizer, global_prompt, local_prompts): + """Tokenize global + space-prefixed locals; return (full_prompt, per-local token ranges). + + Uses incremental tokenization to avoid SentencePiece context-dependency issues. + """ + prefixed_locals = [" " + lp for lp in local_prompts] + full_prompt = global_prompt + "".join(prefixed_locals) + + # Detect if the tokenizer appends EOS dynamically + has_eos = getattr(raw_tokenizer, "add_eos", False) + if not has_eos: + try: + test_res = raw_tokenizer("test") + if isinstance(test_res, dict) and "input_ids" in test_res: + ids = test_res["input_ids"] + elif hasattr(test_res, "input_ids"): + ids = test_res.input_ids + elif isinstance(test_res, list): + ids = test_res + else: + ids = [] + + if ids: + eos_id = getattr(raw_tokenizer, "eos_token_id", None) + if eos_id is not None and ids[-1] == eos_id: + has_eos = True + elif ids[-1] == 1: + has_eos = True + except Exception: + pass + + eos_adj = 1 if has_eos else 0 + + prev_len = len(raw_tokenizer(global_prompt)["input_ids"]) - eos_adj + token_ranges = [] + built = global_prompt + + for plp in prefixed_locals: + built += plp + cur_len = len(raw_tokenizer(built)["input_ids"]) - eos_adj + if cur_len <= prev_len: + raise ValueError(f"Local prompt produced no tokens: '{plp.strip()}'") + token_ranges.append((prev_len, cur_len)) + prev_len = cur_len + + return full_prompt, token_ranges + + +def distribute_segment_lengths(num_segments, latent_frames, specified_lengths=None): + """Validate or auto-distribute segment frame counts, capped to fit within latent_frames.""" + if specified_lengths: + if len(specified_lengths) != num_segments: + raise ValueError( + f"Number of segment_lengths ({len(specified_lengths)}) " + f"must match number of local prompts ({num_segments})" + ) + lengths = specified_lengths + else: + # ceil division — matches reference implementation + step = -(-latent_frames // num_segments) + lengths = [step] * num_segments + + effective = [] + cursor = 0 + for L in lengths: + end = min(cursor + L, latent_frames) + effective.append(max(end - cursor, 0)) + cursor = end + return effective diff --git a/k_ltx_av_bind_schedule.py b/k_ltx_av_bind_schedule.py new file mode 100644 index 0000000..df964f2 --- /dev/null +++ b/k_ltx_av_bind_schedule.py @@ -0,0 +1,297 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# +# ComfyUI-Koolook — LTX A/V Bind Schedule node. +# Copyright (C) 2026 ComfyUI-Koolook contributors (kforgelabs). +"""LTX A/V Bind Schedule (Koolook) — decouple big motion from lip-sync. + +LTX 2.3's audio-video transformer (`comfy/ldm/lightricks/av_model.py`, +`BasicAVTransformerBlock`) injects audio into the video stream through a single +gated cross-attention per block:: + + a2v_out = self.audio_to_video_attn(vx_scaled, context=ax_scaled, ...) + vx.addcmul_(a2v_out, gate_out_a2v) + +That audio->video term makes the lips follow the audio — and also fights a hard +keyframe pose change when a transition lands on an audio peak (the jump). This +node scales ``a2v_out`` by a **gain** so we control where/when audio binds. + +Two modes (selected by whether ``transition_frames`` is set): + +* **Global ramp** (``transition_frames`` empty): gain ramps over the denoise + from ``early_gain`` (high sigma — motion settles audio-blind) up to 1.0 (low + sigma — lips bind). Decouples motion from lips, but lip-sync timing is also + coarse motion, so holding audio out globally desyncs the mouth. + +* **Transition windows** (``transition_frames`` set, e.g. "34,63,93" with + ``total_frames``): audio binds at FULL strength everywhere (lips sync across + the whole clip) EXCEPT in a ±``transition_window`` band around each keyframe + transition, where the sigma ramp applies — so only those moments are + motion-protected early and rejoined late. This breaks the motion/lip-sync + trade-off: smooth transitions AND synced lips. + +``early_gain=1.0`` (or no node) reproduces stock behavior. + +NOTE: the pure ramp (`a2v_gain`) and per-frame mask (`windowed_frame_gains`) are +unit-tested; the model wiring is validated by rendering on a machine with the +LTX 2.3 AV model loaded (not in CI). +""" +from __future__ import annotations + +import types +import logging + +log = logging.getLogger(__name__) + +_GAIN_KEY = "nghtdrp_a2v_gain" # per-step sigma-ramp gain, set by the unet wrapper + + +def a2v_gain(sigma, sigma_max, bind_start=0.55, bind_end=0.75, early_gain=0.0): + """Audio->video gain for the current ``sigma`` (the sigma ramp). + + Progress runs 0 at the start of denoise (``sigma == sigma_max``) to 1 at the + end (``sigma -> 0``). Holds at ``early_gain`` until ``bind_start``, smoothsteps + to 1.0 by ``bind_end`` (progress fractions in [0, 1]). + """ + if sigma_max <= 0.0: + return 1.0 + progress = 1.0 - (float(sigma) / float(sigma_max)) + progress = 0.0 if progress < 0.0 else (1.0 if progress > 1.0 else progress) + if bind_end <= bind_start: + return 1.0 if progress >= bind_end else early_gain + if progress <= bind_start: + return early_gain + if progress >= bind_end: + return 1.0 + t = (progress - bind_start) / (bind_end - bind_start) + s = t * t * (3.0 - 2.0 * t) # smoothstep + return early_gain + (1.0 - early_gain) * s + + +def windowed_frame_gains(latent_frames, transition_idxs, window, sigma_gain): + """Per-latent-frame a2v gain list. + + 1.0 everywhere (full audio -> lips sync), dipping toward ``sigma_gain`` inside + a ``+/-window`` band around each transition latent index, tapered smoothly so + there's no hard edge. At a transition's center the gain is ``sigma_gain`` (so + the sigma ramp still protects motion early / rejoins late there); at the band + edge it's back to 1.0. + """ + if latent_frames <= 0: + return [] + window = max(0, int(window)) + gains = [1.0] * latent_frames + if not transition_idxs: + # no windows -> uniform sigma_gain (degenerates to the global ramp) + return [float(sigma_gain)] * latent_frames + for f in range(latent_frames): + best = 1.0 + for ti in transition_idxs: + dist = abs(f - int(ti)) + if dist <= window: + if window == 0: + taper = 0.0 + else: + u = dist / window # 0 center -> 1 edge + taper = u * u * (3.0 - 2.0 * u) # smoothstep + gv = sigma_gain + (1.0 - sigma_gain) * taper + if gv < best: + best = gv + gains[f] = best + return gains + + +def _parse_frames(s): + out = [] + for tok in str(s or "").replace(";", ",").split(","): + tok = tok.strip() + if not tok: + continue + try: + out.append(float(tok)) + except ValueError: + pass + return out + + +def _make_a2v_scale_method(underlying, transition_fracs, window_frac, diag): + """Bound-method wrapper around audio_to_video_attn.forward: scale its output + by the per-step (and optionally per-frame) gain. ``diag`` is a shared 1-elem + list used to log geometry once per render.""" + + def wrapped(_self_module, *args, **kwargs): + out = underlying(*args, **kwargs) + to = kwargs.get("transformer_options", {}) or {} + sigma_gain = float(to.get(_GAIN_KEY, 1.0)) + grid = to.get("grid_sizes", None) + + def _diag(applied): + if not diag[0]: + diag[0] = True + log.info( + "[LTXAVBindSchedule] a2v fired: out.shape=%s grid_sizes=%s " + "transition_fracs=%s window_frac=%.4f -> %s", + tuple(getattr(out, "shape", ())), grid, transition_fracs, + window_frac, applied, + ) + + # Global mode (no transition windows): uniform scalar gain. + if not transition_fracs: + _diag("global scalar") + return out if sigma_gain == 1.0 else out * sigma_gain + + # Windowed mode: need the latent grid to map tokens -> frames. + try: + latent_frames = int(grid[0]) + tpf = int(grid[1]) * int(grid[2]) + except Exception: + _diag("FALLBACK scalar (no grid_sizes)") + return out if sigma_gain == 1.0 else out * sigma_gain + + if latent_frames <= 0 or tpf <= 0 or out.shape[1] != latent_frames * tpf: + _diag(f"FALLBACK scalar (shape mismatch: out[1]={out.shape[1]} vs " + f"{latent_frames}*{tpf}={latent_frames * tpf})") + return out if sigma_gain == 1.0 else out * sigma_gain + _diag(f"per-frame (latent_frames={latent_frames}, tpf={tpf})") + + ti = sorted({max(0, min(latent_frames - 1, round(fr * (latent_frames - 1)))) + for fr in transition_fracs}) + wl = max(1, round(window_frac * latent_frames)) + fg = windowed_frame_gains(latent_frames, ti, wl, sigma_gain) + if all(g == 1.0 for g in fg): + return out + # expand per-frame -> per-token (frame-major) and scale + gain_tok = out.new_tensor(fg).repeat_interleave(tpf).view(1, -1, 1) + return out * gain_tok + + return wrapped + + +class LTXAVBindSchedule: + """Schedule LTX 2.3 audio->video binding across denoise and/or in time.""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "model": ("MODEL", {"tooltip": "LTX 2.3 audio-video model."}), + "early_gain": ("FLOAT", { + "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01, + "tooltip": "Audio->video strength in the early (high-sigma) steps " + "where big motion resolves. 0 = audio-blind motion; " + "1 = stock. In windowed mode this is the floor INSIDE " + "the transition bands only.", + }), + "bind_start": ("FLOAT", { + "default": 0.55, "min": 0.0, "max": 1.0, "step": 0.01, + "tooltip": "Denoise progress (0=start,1=end) where audio begins " + "ramping in.", + }), + "bind_end": ("FLOAT", { + "default": 0.75, "min": 0.0, "max": 1.0, "step": 0.01, + "tooltip": "Denoise progress where audio binding reaches full.", + }), + }, + "optional": { + "transition_frames": ("STRING", { + "default": "", + "tooltip": "Comma-separated keyframe transition frames (timeline " + "frames, e.g. '34,63,93'). Empty = global ramp (audio " + "gated everywhere). Set = WINDOWED: audio binds fully " + "everywhere except a band around these frames.", + }), + "total_frames": ("INT", { + "default": 0, "min": 0, "max": 100000, "step": 1, + "tooltip": "Clip length in timeline frames (e.g. 146). Required " + "for windowed mode to map frames -> latent grid.", + }), + "transition_window": ("INT", { + "default": 8, "min": 0, "max": 100000, "step": 1, + "tooltip": "Half-width (timeline frames) of the audio-protected " + "band around each transition.", + }), + }, + } + + RETURN_TYPES = ("MODEL",) + RETURN_NAMES = ("model",) + FUNCTION = "patch" + CATEGORY = "Koolook/LTX" + DESCRIPTION = ( + "Decouple big motion from lip-sync in LTX 2.3 by gating the audio->video " + "cross-attention — globally by sigma, or (windowed) only around keyframe " + "transitions so lips stay synced everywhere else. One pass, no re-noise." + ) + + def patch(self, model, early_gain, bind_start, bind_end, + transition_frames="", total_frames=0, transition_window=8): + m = model.clone() + + try: + sigma_max = float(m.get_model_object("model_sampling").sigma_max) + except Exception as e: # pragma: no cover - live model + log.warning("[LTXAVBindSchedule] no model_sampling.sigma_max: %s", e) + sigma_max = 1.0 + + # windowed-mode geometry (fractions are stride-independent) + frames = _parse_frames(transition_frames) + if frames and total_frames and total_frames > 0: + transition_fracs = [min(1.0, max(0.0, f / float(total_frames))) for f in frames] + window_frac = max(0.0, transition_window / float(total_frames)) + mode = f"windowed ({len(transition_fracs)} transitions, +/-{transition_window}f)" + else: + transition_fracs = [] + window_frac = 0.0 + mode = "global ramp" + + patched = 0 + diag = [False] # shared one-shot diagnostic flag across blocks + try: + dm = m.get_model_object("diffusion_model") + for idx, block in enumerate(getattr(dm, "transformer_blocks", None) or []): + if getattr(block, "audio_to_video_attn", None) is None: + continue + key = f"diffusion_model.transformer_blocks.{idx}.audio_to_video_attn.forward" + underlying = m.get_model_object(key) + wrapper = _make_a2v_scale_method(underlying, transition_fracs, window_frac, diag) + m.add_object_patch(key, types.MethodType(wrapper, block.audio_to_video_attn)) + patched += 1 + except Exception as e: # pragma: no cover - live model + log.warning("[LTXAVBindSchedule] block patch failed: %s", e) + + if patched == 0: + log.warning( + "[LTXAVBindSchedule] no audio_to_video_attn blocks found — is this an " + "LTX 2.3 audio-video model? Model passed through unchanged." + ) + return (m,) + + prev = m.model_options.get("model_function_wrapper") + + def unet_wrapper(model_function, params): + try: + sigma = float(params["timestep"].max()) + except Exception: + sigma = sigma_max + gain = a2v_gain(sigma, sigma_max, bind_start, bind_end, early_gain) + c = params["c"] + to = dict(c.get("transformer_options", {})) + to[_GAIN_KEY] = gain + c = {**c, "transformer_options": to} + params = {**params, "c": c} + if prev is not None: + return prev(model_function, params) + return model_function(params["input"], params["timestep"], **params["c"]) + + m.set_model_unet_function_wrapper(unet_wrapper) + log.info("[LTXAVBindSchedule] patched %d a2v blocks; mode=%s; early_gain=%.2f " + "bind=%.2f..%.2f", patched, mode, early_gain, bind_start, bind_end) + return (m,) + + +NODE_CLASS_MAPPINGS = {"LTXAVBindSchedule": LTXAVBindSchedule} +NODE_DISPLAY_NAME_MAPPINGS = {"LTXAVBindSchedule": "LTX A/V Bind Schedule (Koolook)"} + +__all__ = [ + "NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", + "a2v_gain", "windowed_frame_gains", +] diff --git a/scripts/sync_to_dev.py b/scripts/sync_to_dev.py index 7f94c04..e4c257c 100644 --- a/scripts/sync_to_dev.py +++ b/scripts/sync_to_dev.py @@ -162,6 +162,7 @@ RUNTIME_PATHS: tuple[str, ...] = ( "k_easy_utility.py", "k_easy_wan22_prompt.py", "k_loop_status.py", + "k_ltx_av_bind_schedule.py", "k_publish_contract.py", "k_video_combine.py", "k_video_load.py", @@ -199,6 +200,37 @@ def load_dotenv(env_path: Path) -> None: os.environ.setdefault(key.strip(), value.strip().strip('"').strip("'")) +def find_dotenv() -> Path | None: + """Locate the `.env` to load. + + Worktree root first, then the **main repo root** when running from a git + worktree (where `.git` is a file pointing at + `
/.git/worktrees/`). The committed `.env` lives only in the + main checkout, so without this fallback `dev-sync` is unusable from a + worktree. Mirrors `scripts/make_card.py` and the documented behavior in + the dev-sync docs. + """ + direct = REPO_ROOT / ".env" + if direct.exists(): + return direct + git_marker = REPO_ROOT / ".git" + if not git_marker.is_file(): + return None + try: + content = git_marker.read_text(encoding="utf-8").strip() + except OSError: + return None + if not content.startswith("gitdir:"): + return None + gitdir = Path(content.split(":", 1)[1].strip()) + if "worktrees" not in gitdir.parts: + return None + idx = gitdir.parts.index("worktrees") + main_repo_root = Path(*gitdir.parts[:idx]).parent + candidate = main_repo_root / ".env" + return candidate if candidate.exists() else None + + def _ignore(_dir: str, names: list[str]) -> list[str]: return [n for n in names if n in DIR_EXCLUDES] @@ -337,7 +369,9 @@ def main() -> int: ) args = parser.parse_args() - load_dotenv(REPO_ROOT / ".env") + env_path = find_dotenv() + if env_path is not None: + load_dotenv(env_path) target_str = os.environ.get("KOLOOK_COMFYUI_DEV_PATH") if not target_str: diff --git a/scripts/sync_to_dev_audio.py b/scripts/sync_to_dev_audio.py index 26cf84e..85b1a3c 100644 --- a/scripts/sync_to_dev_audio.py +++ b/scripts/sync_to_dev_audio.py @@ -134,28 +134,6 @@ def remove_stale_paths(target: Path, *, dry_run: bool, verbose: bool) -> int: return removed -def _find_dotenv() -> Path | None: - direct = _dev.REPO_ROOT / ".env" - if direct.exists(): - return direct - git_marker = _dev.REPO_ROOT / ".git" - if not git_marker.is_file(): - return None - try: - content = git_marker.read_text(encoding="utf-8").strip() - except OSError: - return None - if not content.startswith("gitdir:"): - return None - gitdir = Path(content.split(":", 1)[1].strip()) - if "worktrees" not in gitdir.parts: - return None - idx = gitdir.parts.index("worktrees") - main_repo_root = Path(*gitdir.parts[:idx]).parent - candidate = main_repo_root / ".env" - return candidate if candidate.exists() else None - - def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -191,7 +169,7 @@ def main() -> int: ) args = parser.parse_args() - env_path = _find_dotenv() + env_path = _dev.find_dotenv() if env_path is not None: _dev.load_dotenv(env_path) diff --git a/tests/forks/test_whatdreamscost_v2_0_2_keyframe_grid.py b/tests/forks/test_whatdreamscost_v2_0_2_keyframe_grid.py new file mode 100644 index 0000000..247167a --- /dev/null +++ b/tests/forks/test_whatdreamscost_v2_0_2_keyframe_grid.py @@ -0,0 +1,197 @@ +"""Tests for the v2_0_2 latent-grid keyframe snapper (issue #258). + +The snapper lives in +``forks/whatdreamscost_koolook/versions/v2_0_2/keyframe_grid.py`` and is +deliberately pure stdlib, so we load it by file path — the same isolation +trick the other fork tests use to avoid triggering the package +``__init__`` (which imports ``ltx_director`` -> ``comfy_api``, unavailable +inside pytest). +""" +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import pytest + + +_MOD_PATH = ( + Path(__file__).resolve().parents[2] + / "forks" + / "whatdreamscost_koolook" + / "versions" + / "v2_0_2" + / "keyframe_grid.py" +) +_spec = importlib.util.spec_from_file_location( + "whatdreamscost_v2_0_2_keyframe_grid", _MOD_PATH +) +_mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_mod) + +latent_index = _mod.latent_index +bucket_center = _mod.bucket_center +latent_count_for_duration = _mod.latent_count_for_duration +snap = _mod.snap_keyframes_to_grid + +S = 8 # LTXV temporal stride + + +# --- latent_index ----------------------------------------------------------- + +@pytest.mark.parametrize( + "frame,expected", + [ + (0, 0), # causal first frame + (-5, 0), # clamp negatives to the first frame + (1, 1), # first frame of bucket 1 + (8, 1), # last frame of bucket 1 + (9, 2), # first frame of bucket 2 + (16, 2), + (33, 5), + (40, 5), + ], +) +def test_latent_index(frame, expected): + assert latent_index(frame, S) == expected + + +# --- bucket_center round-trips ---------------------------------------------- + +@pytest.mark.parametrize("i", range(0, 20)) +def test_bucket_center_round_trips(i): + # Any pin placed at a bucket's center maps back to that exact bucket. + assert latent_index(bucket_center(i, S), S) == i + + +def test_bucket_center_values(): + assert bucket_center(0, S) == 0 + assert bucket_center(1, S) == 4 + assert bucket_center(5, S) == 36 + + +# --- snapping --------------------------------------------------------------- + +def test_frame_zero_passthrough(): + snapped, warns = snap([0], S, latent_length=20) + assert snapped == [0] + assert warns == [] + + +def test_boundary_pin_snaps_to_center_same_latent_frame(): + # A pin near a bucket edge keeps its latent frame but moves to center. + (snapped,), warns = snap([33], S, latent_length=20) + assert latent_index(33, S) == latent_index(snapped, S) == 5 + assert snapped == 36 + assert warns == [] + + +def test_distinct_buckets_untouched_latent_indices(): + frames = [0, 39, 86] # the known-good 00016 layout + snapped, warns = snap(frames, S, latent_length=20) + assert [latent_index(f, S) for f in snapped] == [0, 5, 11] + assert warns == [] + + +def test_output_order_matches_input_order(): + frames = [86, 0, 39] # deliberately unsorted + snapped, _ = snap(frames, S, latent_length=20) + # position-for-position, each output is the snap of the same input + assert latent_index(snapped[0], S) == latent_index(86, S) + assert snapped[1] == 0 + assert latent_index(snapped[2], S) == latent_index(39, S) + + +def test_collision_bumps_later_pin_and_warns(): + # Two pins in the same bucket (latent 1): 4 and 6 both -> L1. + frames = [4, 6] + snapped, warns = snap(frames, S, latent_length=20) + indices = sorted(latent_index(f, S) for f in snapped) + assert indices == [1, 2] # second bumped to the next bucket + assert len(warns) == 1 + assert "bumped" in warns[0] + + +def test_collision_exhaustion_warns_when_no_free_bucket(): + # latent_length=2 -> only latent indices {0, 1} exist. Three pins all in + # bucket 1 cannot all fit; the last has nowhere to go. + frames = [2, 3, 4] + snapped, warns = snap(frames, S, latent_length=2) + # everything clamps to the max index (1); at least one exhaustion warning + assert all(latent_index(f, S) <= 1 for f in snapped) + assert any("no free bucket" in w for w in warns) + + +def test_invalid_stride_raises(): + with pytest.raises(ValueError): + snap([0, 8], 0, latent_length=20) + + +def test_latent_count_for_duration_uses_ltxv_ceiling_rule(): + # 120 is not 8n+1. LTXV rounds it to 121 pixel frames, i.e. 16 latent + # frames, so the final bucket (index 15) is valid for snapping. + assert latent_count_for_duration(120, S) == 16 + snapped, warns = snap([119], S, latent_length=latent_count_for_duration(120, S)) + assert snapped == [bucket_center(15, S)] + assert latent_index(snapped[0], S) == 15 + assert warns == [] + + +def test_latent_count_rejects_invalid_stride(): + with pytest.raises(ValueError): + latent_count_for_duration(120, 0) + + +def test_empty_input(): + snapped, warns = snap([], S, latent_length=20) + assert snapped == [] + assert warns == [] + + +# --- expand_keyframe_ease: strength-ramped neighbor clusters --- + +ease = _mod.expand_keyframe_ease + + +def test_ease_zero_is_passthrough(): + idxs, frames, strs = ease([0, 40], [1.0, 1.0], 8, 0, 0.5, 145) + assert idxs == [0, 1] + assert frames == [0, 40] + assert strs == [1.0, 1.0] + + +def test_ease_keeps_centers_exact_first(): + idxs, frames, strs = ease([40], [1.0], 8, 1, 0.5, 145) + # center emitted first, exact + assert idxs[0] == 0 and frames[0] == 40 and strs[0] == 1.0 + + +def test_ease_adds_ramped_neighbors_one_bucket_apart(): + idxs, frames, strs = ease([40], [1.0], 8, 1, 0.5, 145) + # neighbors at 32 and 48, strength 0.5, reusing image index 0 + pairs = sorted((f, s) for f, s in zip(frames[1:], strs[1:])) + assert pairs == [(32, 0.5), (48, 0.5)] + assert all(i == 0 for i in idxs) + + +def test_ease_two_levels_falloff(): + idxs, frames, strs = ease([40], [1.0], 8, 2, 0.5, 145) + m = {f: s for f, s in zip(frames, strs)} + assert m[40] == 1.0 + assert m[32] == 0.5 and m[48] == 0.5 # k=1 + assert m[24] == 0.25 and m[56] == 0.25 # k=2 + + +def test_ease_skips_out_of_range(): + # center at 0: left neighbor (-8) dropped, right (8) kept + idxs, frames, strs = ease([0], [1.0], 8, 1, 0.5, 145) + assert sorted(frames) == [0, 8] + + +def test_ease_skips_collision_with_other_center(): + # centers 40 and 48 (one bucket apart): 40's +8 == 48 (a center) -> skipped + idxs, frames, strs = ease([40, 48], [1.0, 1.0], 8, 1, 0.5, 145) + # both centers present + assert frames.count(40) == 1 and frames.count(48) == 1 + # no duplicate frame entries + assert len(frames) == len(set(frames)) diff --git a/tests/nodes/test_ltx_av_bind_schedule.py b/tests/nodes/test_ltx_av_bind_schedule.py new file mode 100644 index 0000000..bd86aad --- /dev/null +++ b/tests/nodes/test_ltx_av_bind_schedule.py @@ -0,0 +1,127 @@ +"""Unit tests for the LTX A/V Bind Schedule gain ramp. + +Only the pure `a2v_gain` ramp is tested here — it's stdlib-only. The model +wiring (object-patch on audio_to_video_attn + the unet wrapper) is validated by +rendering against a live LTX 2.3 AV model, not in CI. Loaded by file path to +avoid importing the package __init__ (which pulls in ComfyUI). +""" +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import pytest + + +_MOD = ( + Path(__file__).resolve().parents[2] / "k_ltx_av_bind_schedule.py" +) +_spec = importlib.util.spec_from_file_location("k_ltx_av_bind_schedule_under_test", _MOD) +_m = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_m) +gain = _m.a2v_gain + +SMAX = 10.0 + + +def test_early_steps_use_early_gain(): + # sigma at the top of the schedule -> progress 0 -> early_gain + assert gain(SMAX, SMAX, 0.55, 0.75, 0.0) == 0.0 + assert gain(SMAX, SMAX, 0.55, 0.75, 0.2) == pytest.approx(0.2) + + +def test_late_steps_full_bind(): + # sigma -> 0 -> progress 1 -> full audio binding + assert gain(0.0, SMAX, 0.55, 0.75, 0.0) == 1.0 + assert gain(0.01 * SMAX, SMAX, 0.55, 0.75, 0.0) == 1.0 + + +def test_monotonic_non_decreasing_as_sigma_falls(): + # walk sigma from max down to 0; gain must never decrease + prev = -1.0 + for i in range(101): + sigma = SMAX * (1.0 - i / 100.0) # SMAX -> 0 + g = gain(sigma, SMAX, 0.55, 0.75, 0.0) + assert g >= prev - 1e-9 + prev = g + + +def test_holds_early_gain_before_bind_start(): + # progress 0.3 (sigma=0.7*SMAX) is below bind_start=0.55 -> early_gain + assert gain(0.7 * SMAX, SMAX, 0.55, 0.75, 0.1) == pytest.approx(0.1) + + +def test_full_at_and_after_bind_end(): + # progress 0.75 (sigma=0.25*SMAX) == bind_end -> 1.0 + assert gain(0.25 * SMAX, SMAX, 0.55, 0.75, 0.0) == 1.0 + + +def test_smoothstep_midpoint_between_bounds(): + # midpoint of the ramp: progress 0.65 -> smoothstep(0.5)=0.5 -> halfway + g = gain((1.0 - 0.65) * SMAX, SMAX, 0.55, 0.75, 0.0) + assert g == pytest.approx(0.5, abs=1e-6) + + +def test_zero_sigma_max_is_safe(): + assert gain(5.0, 0.0, 0.55, 0.75, 0.0) == 1.0 + + +def test_degenerate_bounds_step(): + # bind_end <= bind_start -> hard step at bind_end (progress >= 0.6 -> 1.0) + assert gain(0.5 * SMAX, SMAX, 0.6, 0.6, 0.0) == 0.0 # progress 0.5 < 0.6 + assert gain(0.3 * SMAX, SMAX, 0.6, 0.6, 0.0) == 1.0 # progress 0.7 >= 0.6 + + +def test_early_gain_one_is_stock_behavior(): + # early_gain=1.0 -> gain is 1.0 everywhere (node becomes a no-op) + for frac in (0.0, 0.3, 0.6, 1.0): + assert gain((1.0 - frac) * SMAX, SMAX, 0.55, 0.75, 1.0) == pytest.approx(1.0) + + +# --- windowed_frame_gains: per-frame mask (transition-window mode) --- + +wfg = _m.windowed_frame_gains + + +def test_windowed_full_audio_outside_bands(): + # transition at latent frame 5, window 1, sigma_gain 0.0 -> only 4,5,6 dip + g = wfg(20, [5], 1, 0.0) + assert g[5] == 0.0 # center fully suppressed + assert g[0] == 1.0 and g[10] == 1.0 # far frames untouched (lips sync) + assert all(g[f] == 1.0 for f in range(20) if abs(f - 5) > 1) + + +def test_windowed_center_is_sigma_gain(): + g = wfg(20, [5], 2, 0.3) + assert g[5] == pytest.approx(0.3) # center floor = sigma_gain + assert g[3] == pytest.approx(1.0) # at edge distance==window -> ~1.0 + assert g[7] == pytest.approx(1.0) + + +def test_windowed_tapers_between_center_and_edge(): + g = wfg(40, [20], 4, 0.0) + # monotonic rise from center outward on each side + assert g[20] < g[21] < g[22] < g[23] <= g[24] + assert g[24] == pytest.approx(1.0, abs=1e-6) + + +def test_windowed_multiple_transitions_take_min(): + g = wfg(30, [5, 6], 1, 0.0) + # overlapping bands: frames 4..7 affected, 5 and 6 fully suppressed + assert g[5] == 0.0 and g[6] == 0.0 + assert g[10] == 1.0 + + +def test_windowed_no_transitions_is_uniform_sigma_gain(): + g = wfg(10, [], 3, 0.4) + assert g == [pytest.approx(0.4)] * 10 + + +def test_windowed_empty_latent_frames(): + assert wfg(0, [5], 2, 0.0) == [] + + +def test_windowed_sigma_gain_one_is_noop(): + # late steps: sigma_gain==1 -> even transition frames are full (lips rejoin) + g = wfg(20, [5], 2, 1.0) + assert all(x == pytest.approx(1.0) for x in g) diff --git a/tests/scripts/test_sync_to_dev.py b/tests/scripts/test_sync_to_dev.py index 09618f4..f1de444 100644 --- a/tests/scripts/test_sync_to_dev.py +++ b/tests/scripts/test_sync_to_dev.py @@ -98,3 +98,32 @@ def test_dev_sync_ships_manager_visible_package_metadata(): installed version. """ assert "pyproject.toml" in sync_to_dev.RUNTIME_PATHS + + +# --- find_dotenv: worktree -> main-repo .env fallback (dev-sync from a worktree) --- + +def test_find_dotenv_prefers_worktree_env(tmp_path, monkeypatch): + wt = tmp_path / "wt" + wt.mkdir() + (wt / ".env").write_text("X=1\n", encoding="utf-8") + monkeypatch.setattr(sync_to_dev, "REPO_ROOT", wt) + assert sync_to_dev.find_dotenv() == wt / ".env" + + +def test_find_dotenv_falls_back_to_main_repo_from_worktree(tmp_path, monkeypatch): + main = tmp_path / "ComfyUI-Koolook" + (main / ".git" / "worktrees" / "wt").mkdir(parents=True) + (main / ".env").write_text("KOLOOK_COMFYUI_DEV_PATH=/somewhere\n", encoding="utf-8") + wt = main / ".claude" / "worktrees" / "wt" + wt.mkdir(parents=True) + gitdir = main / ".git" / "worktrees" / "wt" + (wt / ".git").write_text(f"gitdir: {gitdir.as_posix()}\n", encoding="utf-8") + monkeypatch.setattr(sync_to_dev, "REPO_ROOT", wt) + assert sync_to_dev.find_dotenv() == main / ".env" + + +def test_find_dotenv_none_when_no_env_and_not_a_worktree(tmp_path, monkeypatch): + plain = tmp_path / "plain" + plain.mkdir() + monkeypatch.setattr(sync_to_dev, "REPO_ROOT", plain) + assert sync_to_dev.find_dotenv() is None diff --git a/tests/sidebar/test_ltx_director_draft_guard.py b/tests/sidebar/test_ltx_director_draft_guard.py index 36f915b..e621faa 100644 --- a/tests/sidebar/test_ltx_director_draft_guard.py +++ b/tests/sidebar/test_ltx_director_draft_guard.py @@ -1,4 +1,12 @@ -"""Frontend contract checks for the LTX Director timeline extension.""" +"""Frontend contract checks for the LTX Director timeline extension. + +As of the v2.0.2 fork, web/whatdreamscost_koolook/ltx_director.js is the +upstream 2.0.2 timeline editor vendored VERBATIM and retargeted to the Koolook +node id, rather than the old (1.3.9-era) editor carrying Koolook media-scrub +deltas. Upstream 2.0.2 persists media via imageFile/audioFile + /api/view URLs, +so those scrub deltas are obsolete; the localStorage quota guard still lives +exclusively in the global web/koolook_draft_guard.js. +""" from __future__ import annotations from pathlib import Path @@ -8,11 +16,10 @@ REPO_ROOT = Path(__file__).resolve().parents[2] LTX_DIRECTOR_JS = REPO_ROOT / "web" / "whatdreamscost_koolook" / "ltx_director.js" -def test_ltx_director_no_longer_embeds_the_draft_guard() -> None: - """The quota guard moved to web/koolook_draft_guard.js (all key - generations, every page). A stale copy here would double-install the - localStorage wrapper and drift behind the global guard's key coverage, - so its absence is a contract, not an accident. +def test_ltx_director_does_not_embed_the_draft_guard() -> None: + """The quota guard lives in web/koolook_draft_guard.js (all keys, every + page). The node editor must not re-install a localStorage wrapper or it + would double-install and drift behind the global guard's key coverage. """ source = LTX_DIRECTOR_JS.read_text(encoding="utf-8") @@ -23,27 +30,25 @@ def test_ltx_director_no_longer_embeds_the_draft_guard() -> None: assert "web/koolook_draft_guard.js" in source # pointer for archaeology -def test_timeline_serialization_drops_preview_only_media() -> None: +def test_ltx_director_js_is_retargeted_to_the_koolook_node() -> None: + """The vendored upstream editor must bind to the Koolook node id under a + unique extension name, never the bare upstream `LTXDirector` (that would + collide with the installed upstream WhatDreamsCost extension). + """ source = LTX_DIRECTOR_JS.read_text(encoding="utf-8") - assert "function serializeTimelineSegment(seg)" in source - assert "imageB64" in source - assert "audioB64" in source - assert "segments: sortedSegments.map(s => serializeTimelineSegment(s))" in source - assert "audioSegments: (this.timeline.audioSegments || []).map(s => serializeTimelineSegment(s))" in source + assert 'name: "LTXDirector_Koolook"' in source + assert 'nodeData.name === "LTXDirector__koolook"' in source + # must not bind the bare upstream node type + assert 'nodeData.name === "LTXDirector"' not in source + assert 'name: "LTXDirector",' not in source -def test_timeline_rehydrates_image_previews_from_saved_image_file() -> None: +def test_ltx_director_js_is_vendored_from_pinned_upstream() -> None: + """Provenance header pins the exact upstream commit it was vendored from, + so the editor version is auditable against forks_manifest.yaml. + """ source = LTX_DIRECTOR_JS.read_text(encoding="utf-8") - assert "function inputImageUrl(imageFile)" in source - assert "function imagePreviewSrc(seg)" in source - assert "function scrubPersistedPreviewMedia(seg)" in source - assert 'replace(/\\\\/g, "/")' in source - assert "if (seg.imageFile && seg.imageB64) delete seg.imageB64;" in source - assert "if (seg.audioFile && seg.audioB64) delete seg.audioB64;" in source - assert "p.segments.map(scrubPersistedPreviewMedia)" in source - assert "p.audioSegments.map(scrubPersistedPreviewMedia)" in source - assert "const src = imagePreviewSrc(seg);" in source - assert "seg._imagePreviewSrc = src;" in source - assert "seg.imgObj.src = src;" in source + assert "Vendored VERBATIM from WhatDreamsCost-ComfyUI" in source + assert "fe09f73756df202d08341c66b4dc5fc8d2acca22" in source diff --git a/tests/test_vendored_exclusions.py b/tests/test_vendored_exclusions.py new file mode 100644 index 0000000..0ccf612 --- /dev/null +++ b/tests/test_vendored_exclusions.py @@ -0,0 +1,26 @@ +"""Guards for vendored-upstream exclusions. + +Only the WhatDreamsCost 2.0.2 files added as upstream replicas should be hidden +from GitHub language stats / CodeQL. Older fork versions carry Koolook-authored +customizations, so broad ``versions/*`` exclusions would hide real review +surface. +""" +from __future__ import annotations + +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def test_whatdreamscost_vendored_exclusions_are_version_scoped() -> None: + gitattributes = (REPO_ROOT / ".gitattributes").read_text(encoding="utf-8") + codeql_config = ( + REPO_ROOT / ".github" / "codeql" / "codeql-config.yml" + ).read_text(encoding="utf-8") + + for text in (gitattributes, codeql_config): + assert "versions/*/prompt_relay.py" not in text + assert "versions/*/patches.py" not in text + assert "versions/v2_0_2/prompt_relay.py" in text + assert "versions/v2_0_2/patches.py" in text diff --git a/web/whatdreamscost_koolook/ltx_director.js b/web/whatdreamscost_koolook/ltx_director.js index 8b5847c..51b221d 100644 --- a/web/whatdreamscost_koolook/ltx_director.js +++ b/web/whatdreamscost_koolook/ltx_director.js @@ -1,3 +1,18 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +// Vendored VERBATIM from WhatDreamsCost-ComfyUI js/ltx_director.js at commit +// fe09f73756df202d08341c66b4dc5fc8d2acca22 (version 2.0.2), with ONLY two +// retargeting edits by ComfyUI-Koolook on 2026-06-22: +// - extension name 'LTXDirector' -> 'LTXDirector_Koolook' (avoid collision +// with the installed upstream WhatDreamsCost extension) +// - node guard nodeData.name 'LTXDirector' -> 'LTXDirector__koolook' +// The timeline editor UX is otherwise byte-for-byte upstream 2.0.2. +// +// localStorage draft-quota guarding is intentionally NOT embedded here; it +// lives in the global web/koolook_draft_guard.js (covers all draft keys on +// every page). 2.0.2 persists timeline media via imageFile/audioFile + the +// /api/view URL, so the old koolook base64-scrub deltas are no longer needed. +// License: GPL-3.0-or-later (matches the ComfyUI-Koolook pack). + const { app } = window.comfyAPI.app; const { api } = window.comfyAPI.api; @@ -5,59 +20,60 @@ const { api } = window.comfyAPI.api; const RULER_HEIGHT = 24; const BLOCK_HEIGHT = 160; // Increased to make the image timeline area much taller const AUDIO_TRACK_HEIGHT = 80; -const CANVAS_HEIGHT = RULER_HEIGHT + BLOCK_HEIGHT + AUDIO_TRACK_HEIGHT; +const MOTION_TRACK_HEIGHT = 80; // used as Motion Guide track height +const CANVAS_HEIGHT = RULER_HEIGHT + BLOCK_HEIGHT + MOTION_TRACK_HEIGHT + AUDIO_TRACK_HEIGHT; const HANDLE_HIT_PX = 14; const MIN_SEGMENT_LENGTH = 6; const MAX_THUMBNAIL_DIM = 512; // Increased to maintain quality for taller images -const HIDDEN_WIDGET_NAMES = ["timeline_data", "local_prompts", "segment_lengths", "guide_strength", "audio_data", "use_custom_audio"]; + +const HIDDEN_WIDGET_NAMES = ["timeline_data", "local_prompts", "segment_lengths", "guide_strength", "audio_data", "use_custom_audio", "inpaint_audio", "use_custom_motion", "override_audio"]; function hideWidget(w) { if (!w) return; - if (!w._origType && w.type !== "hidden") w._origType = w.type; - // We don't set w.type = "hidden" anymore because it causes rendering issues in Nodes 2.0. - // Instead we use the computeSize = () => [0,0] trick which works in both V1 and V2. + w.hidden = true; if (!w.options) w.options = {}; w.options.hidden = true; - w.computeSize = () => [0, 0]; + + // Use computeSize and draw overrides to safely collapse in LiteGraph + // without triggering ComfyUI's "convert to input slot" auto-behavior. + if (!window.LiteGraph || !window.LiteGraph.vueNodesMode) { + w.computeSize = () => [0, -4]; // -4 cancels out ComfyUI's hardcoded 4px widget padding + if (!w._hiddenDrawHooked) { + w._origDraw = w.hasOwnProperty('draw') ? w.draw : undefined; + w._hiddenDrawHooked = true; + } + w.draw = () => { }; + } + if (w.element) w.element.style.display = "none"; + if (w.callback) w.callback(w.value); +} + +function showWidget(w) { + if (!w) return; + + w.hidden = false; + if (w.options) w.options.hidden = false; + + if (!window.LiteGraph || !window.LiteGraph.vueNodesMode) { + delete w.computeSize; + if (w._hiddenDrawHooked) { + if (w._origDraw !== undefined) { + w.draw = w._origDraw; + } else { + delete w.draw; + } + delete w._hiddenDrawHooked; + } + } + + if (w.element) w.element.style.display = ""; + if (w.callback) w.callback(w.value); } function clamp(v, min, max) { return Math.max(min, Math.min(max, v)); } -function serializeTimelineSegment(seg) { - const { imgObj, imageB64, audioB64, _audioBuffer, _file, _imagePreviewSrc, ...rest } = seg || {}; - return rest; -} - -function inputImageUrl(imageFile) { - const normalized = String(imageFile || "").replace(/\\/g, "/"); - if (!normalized) return ""; - const parts = normalized.split("/"); - const filename = parts.pop(); - const subfolder = parts.join("/"); - if (!filename) return ""; - return api.apiURL(`/view?filename=${encodeURIComponent(filename)}&type=input&subfolder=${encodeURIComponent(subfolder)}`); -} - -function imagePreviewSrc(seg) { - if (!seg) return ""; - return seg.imageB64 || seg._imagePreviewSrc || inputImageUrl(seg.imageFile); -} - -function scrubPersistedPreviewMedia(seg) { - if (!seg || typeof seg !== "object") return seg; - if (seg.imageFile && seg.imageB64) delete seg.imageB64; - if (seg.audioFile && seg.audioB64) delete seg.audioB64; - delete seg.imgObj; - delete seg._audioBuffer; - delete seg._file; - return seg; -} - -// The Comfy draft-cache quota guard that used to live here is global now: -// web/koolook_draft_guard.js (covers all draft key generations). - // --- Modern Dark/Grey UI CSS (ComfyUI Match) --- const STYLES = ` .pr-wrapper { @@ -102,18 +118,25 @@ const STYLES = ` gap: 6px; transition: all 0.2s ease; } - .pr-btn:hover { + .pr-btn:hover:not(:disabled) { background: #333; border-color: #555; } - .pr-btn-danger:hover { + .pr-btn.toggle-on { + background: #1c222d; + border-color: #283142; + color: #e0e0e0; + } + .pr-btn.toggle-on:hover:not(:disabled) { + background: #2a3445; + border-color: #3b4b66; + } + .pr-btn-danger:hover:not(:disabled) { background: #4a1515; border-color: #cc4444; color: #ffaaaa; } .pr-canvas { - border-radius: 6px; - border: 1px solid #111; background: #2a2a2a; cursor: pointer; width: 100%; @@ -127,24 +150,70 @@ const STYLES = ` flex-grow: 1; /* Automatically scales to fill node height */ min-height: 40px; } - .pr-prompt-area { + .pr-prompt-wrapper { + position: relative; width: 100%; height: 100%; background: #222; - color: #e0e0e0; border: 1px solid #111; border-radius: 6px; - padding: 8px; + box-sizing: border-box; + transition: border-color 0.2s ease, opacity 0.2s ease; + overflow: hidden; + } + .pr-prompt-wrapper.focus-active { + border-color: #888; + } + .pr-wrapper.has-focus .pr-prompt-wrapper:not(.focus-active), + .pr-wrapper:has(.pr-prompt-wrapper.focus-active) .pr-prompt-wrapper:not(.focus-active) { + opacity: 0.65; + } + .pr-prompt-label { + position: absolute; + top: 5px; + left: 8px; + font-size: 9px; + font-weight: bold; + color: #666; + text-transform: uppercase; + letter-spacing: 0.5px; + pointer-events: none; + user-select: none; + z-index: 5; + } + .pr-prompt-area { + position: absolute; + top: 20px; + left: 0; + width: 100%; + height: calc(100% - 20px); + background: transparent; + color: #e0e0e0; + border: none; + padding: 0 8px 8px 8px; resize: none; /* Removed the manual resize corner handle */ font-size: 12px; line-height: 1.4; box-sizing: border-box; outline: none; - transition: border-color 0.2s ease; } .pr-prompt-area:focus { border-color: #888; } + .pr-motion-info { + width: 100%; + height: 100%; + background: #181818; + color: #aaa; + border: 1px solid #111; + border-radius: 6px; + padding: 10px; + font-size: 12px; + line-height: 1.6; + box-sizing: border-box; + display: none; + } + .pr-motion-info span { color: #fff; font-weight: 500; } .pr-audio-info { width: 100%; height: 100%; @@ -197,6 +266,8 @@ const STYLES = ` color: #fff; white-space: nowrap; margin-left: auto; + user-select: none; + -webkit-user-select: none; } .pr-strength-slider { -webkit-appearance: none; @@ -231,6 +302,8 @@ const STYLES = ` width: 52px; text-align: center; padding: 3px; + user-select: none; + -webkit-user-select: none; } .pr-strength-input::-webkit-outer-spin-button, .pr-strength-input::-webkit-inner-spin-button { @@ -334,6 +407,8 @@ const STYLES = ` width: 100%; overflow-x: auto; overflow-y: hidden; + padding-bottom: 10px; + box-sizing: content-box; } .pr-timeline-viewport::-webkit-scrollbar { height: 10px; @@ -343,12 +418,12 @@ const STYLES = ` border-radius: 5px; } .pr-timeline-viewport::-webkit-scrollbar-thumb { - background: #444 + background: #444; border-radius: 5px; border: 1px solid #000; } .pr-timeline-viewport::-webkit-scrollbar-thumb:hover { - background: #666 + background: #666; border-color: #000; } .pr-zoom-controls { @@ -385,12 +460,16 @@ const STYLES = ` font-size: 12px; color: #aaa; font-family: monospace; + user-select: none; + -webkit-user-select: none; } .pr-timecode { font-size: 14px; font-weight: bold; color: #e0e0e0; font-family: monospace; + user-select: none; + -webkit-user-select: none; } .pr-settings-menu { position: fixed; @@ -403,7 +482,7 @@ const STYLES = ` gap: 8px; z-index: 9999; box-shadow: 0 4px 20px rgba(0,0,0,0.7); - min-width: 220px; + min-width: 250px; } .pr-settings-title { font-size: 11px; @@ -488,8 +567,10 @@ const STYLES = ` } .pr-settings-toggle-btn { width: 100%; + box-sizing: border-box; + margin: 0; background: #252525; - color: #aaa; + color: #fff; border: 1px solid #333; border-radius: 4px; padding: 5px 8px; @@ -500,7 +581,7 @@ const STYLES = ` } .pr-settings-toggle-btn:hover { background: #2e2e2e; - color: #ccc; + color: #fff; border-color: #555; } .pr-settings-close-btn { @@ -526,7 +607,7 @@ const STYLES = ` border-radius: 6px; padding: 2px; width: 110px; - height: 22px; + height: 25px; align-items: center; box-sizing: border-box; } @@ -535,7 +616,10 @@ const STYLES = ` text-align: center; font-size: 10px; font-weight: 500; - line-height: 18px; + display: flex; + align-items: center; + justify-content: center; + height: 100%; cursor: pointer; border-radius: 4px; color: #888; @@ -545,22 +629,25 @@ const STYLES = ` background: #333; color: #fff; } - .pr-segment:hover:not(.active) { - color: #ccc; + .pr-settings-divider { + border-top: 1px solid #333; + margin: 4px 0; } `; -if (!document.getElementById("prompt-relay-styles")) { - const styleEl = document.createElement("style"); +let styleEl = document.getElementById("prompt-relay-styles"); +if (!styleEl) { + styleEl = document.createElement("style"); styleEl.id = "prompt-relay-styles"; - styleEl.textContent = STYLES; document.head.appendChild(styleEl); } +styleEl.textContent = STYLES; // --- Icons --- const ICONS = { upload: ``, audio: ``, + motion: ``, trash: ``, text: ``, play: ``, @@ -570,17 +657,78 @@ const ICONS = { plus: ``, fit: ``, gear: ``, - close: `` + close: ``, + start: ``, + end: ``, + mark: ``, + help: ``, + magnet: `` }; // --- Data Models --- function parseInitial(jsonStr) { - let parsed = { segments: [], audioSegments: [] }; + let parsed = { + segments: [], + motionSegments: [], + audioSegments: [], + global_prompt: "", + retake_global_prompt: "", + mainTrackEnabled: true, + audioTrackEnabled: true, + motionTrackEnabled: true, + propHeight: 90, + globalPropHeight: 60, + showFilenames: true, + overrideAudio: false, + inpaint_audio: true, + retakeMode: false, + retakeStart: 24, + retakeLength: 48, + retakePrompt: "", + retakeStrength: 1.0, + retakeVideo: null, + normalStartFrame: 0, + normalDurationFrames: 120 + }; try { if (jsonStr) { const p = JSON.parse(jsonStr); - if (Array.isArray(p.segments)) parsed.segments = p.segments.map(scrubPersistedPreviewMedia); - if (Array.isArray(p.audioSegments)) parsed.audioSegments = p.audioSegments.map(scrubPersistedPreviewMedia); + if (p.global_prompt !== undefined) parsed.global_prompt = p.global_prompt; + if (p.retake_global_prompt !== undefined) parsed.retake_global_prompt = p.retake_global_prompt; + if (p.mainTrackEnabled !== undefined) parsed.mainTrackEnabled = p.mainTrackEnabled; + if (p.audioTrackEnabled !== undefined) parsed.audioTrackEnabled = p.audioTrackEnabled; + if (p.motionTrackEnabled !== undefined) parsed.motionTrackEnabled = p.motionTrackEnabled; + if (p.propHeight !== undefined) parsed.propHeight = p.propHeight; + if (p.globalPropHeight !== undefined) parsed.globalPropHeight = p.globalPropHeight; + if (p.showFilenames !== undefined) parsed.showFilenames = p.showFilenames; + if (p.overrideAudio !== undefined) parsed.overrideAudio = p.overrideAudio; + if (p.inpaint_audio !== undefined) parsed.inpaint_audio = p.inpaint_audio; + if (p.retakeMode !== undefined) parsed.retakeMode = p.retakeMode; + if (p.retakeStart !== undefined) parsed.retakeStart = p.retakeStart; + if (p.retakeLength !== undefined) parsed.retakeLength = p.retakeLength; + if (p.retakePrompt !== undefined) parsed.retakePrompt = p.retakePrompt; + if (p.retakeStrength !== undefined) parsed.retakeStrength = p.retakeStrength; + if (p.retakeVideo !== undefined) parsed.retakeVideo = p.retakeVideo; + if (p.normalStartFrame !== undefined) parsed.normalStartFrame = p.normalStartFrame; + if (p.normalDurationFrames !== undefined) parsed.normalDurationFrames = p.normalDurationFrames; + if (Array.isArray(p.segments)) { + parsed.segments = p.segments.map(s => { + const { imgObj, videoEl, _isSeeking, thumbnails, _extractingThumbs, _sSecs, _lSecs, _tSecs, _dSecs, _uploading, _blobUrl, ...rest } = s; + return rest; + }); + } + if (Array.isArray(p.motionSegments)) { + parsed.motionSegments = p.motionSegments.map(s => { + const { videoEl, _isSeeking, thumbnails, _extractingThumbs, _sSecs, _lSecs, _tSecs, _dSecs, _uploading, _blobUrl, ...rest } = s; + return rest; + }); + } + if (Array.isArray(p.audioSegments)) { + parsed.audioSegments = p.audioSegments.map(s => { + const { _sSecs, _lSecs, _tSecs, _dSecs, _uploading, _decoding, ...rest } = s; + return rest; + }); + } } } catch (e) { } @@ -594,6 +742,16 @@ function parseInitial(jsonStr) { if (!seg.id) { seg.id = Date.now().toString() + Math.random().toString(36).substr(2, 5); } + if (seg.isEndFrame === undefined) { + seg.isEndFrame = false; + } + } + + for (let seg of parsed.motionSegments) { + if (!seg.id) { + seg.id = Date.now().toString() + Math.random().toString(36).substr(2, 5); + } + if (seg.trimStart === undefined) seg.trimStart = 0; } for (let seg of parsed.audioSegments) { @@ -615,13 +773,22 @@ class TimelineEditor { // Track heights (dynamic) this.rulerHeight = RULER_HEIGHT; this.blockHeight = BLOCK_HEIGHT; + this.motionTrackHeight = MOTION_TRACK_HEIGHT; this.audioTrackHeight = AUDIO_TRACK_HEIGHT; this.canvasHeight = CANVAS_HEIGHT; // Core data - this.timeline = { segments: [], audioSegments: [] }; - this.selectionType = "image"; // "image" or "audio" - this.selectedIndex = -1; + this.timeline = { segments: [], motionSegments: [], audioSegments: [] }; + this.selectionType = "image"; // "image", "motion", or "audio" + this.selectedSegmentIds = []; + this._selectedIndex = -1; + this._audioTrackWasEnabledBeforeOverride = false; + + // Selection box tracking + this._isSelectingBox = false; + this._selectBoxStart = null; + this._selectBoxCurrent = null; + this._selectBoxInitialSelectedIds = null; // Interactions this._isDragging = false; @@ -648,6 +815,9 @@ class TimelineEditor { this.playbackStartFrame = 0; this._playLoopId = null; + // File handling + this.currentFileHandle = null; + // --- Ghost dragging state --- this._ghostSegmentId = null; this._ghostTrack = null; @@ -658,6 +828,10 @@ class TimelineEditor { this._gapMenuDismisser = null; // Attach to Python widgets + this.startFramesWidget = this.node.widgets.find(w => w.name === "start_frame"); + this.startSecondsWidget = this.node.widgets.find(w => w.name === "start_second"); + this.endFramesWidget = this.node.widgets.find(w => w.name === "end_frame"); + this.endSecondsWidget = this.node.widgets.find(w => w.name === "end_second"); this.durationFramesWidget = this.node.widgets.find(w => w.name === "duration_frames"); this.durationSecondsWidget = this.node.widgets.find(w => w.name === "duration_seconds"); this.frameRateWidget = this.node.widgets.find(w => w.name === "frame_rate"); @@ -667,14 +841,68 @@ class TimelineEditor { this.guideStrengthWidget = this.node.widgets.find(w => w.name === "guide_strength"); this.displayModeWidget = this.node.widgets.find(w => w.name === "display_mode"); + // Track the last-known frame rate so we can compute the rescale ratio + // inside the frameRateWidget callback (the widget value is already updated + // to the new value before the callback fires, so we can't read "old" from it). + this._prevFrameRate = this.getFrameRate(); + this._prevStartFrames = this.getStartFrames(); + this._prevStartSeconds = this.startSecondsWidget ? this.startSecondsWidget.value : 0; + + console.log("[LTXDirector debug] Constructor: timelineDataWidget value:", this.timelineDataWidget?.value); this.timeline = parseInitial(this.timelineDataWidget?.value); - this.loadImages(); + this.retakeMode = this.timeline.retakeMode === true; + if (this.retakeMode) { + if (this.timeline.retake_global_prompt) { + if (!this.node.properties) this.node.properties = {}; + this.node.properties.global_prompt = this.timeline.retake_global_prompt; + } + } else { + if (this.timeline.global_prompt) { + if (!this.node.properties) this.node.properties = {}; + this.node.properties.global_prompt = this.timeline.global_prompt; + } + } + console.log("[LTXDirector debug] Constructor: parsed timeline:", JSON.stringify(this.timeline)); + + // Treat this.timeline (from timeline_data widget) as the absolute source of truth! + this.mainTrackEnabled = this.timeline.mainTrackEnabled !== false; + this.audioTrackEnabled = this.timeline.audioTrackEnabled !== false; + this.motionTrackEnabled = this.timeline.motionTrackEnabled !== false; + + // Sync the properties dictionary too so they match + this.node.properties.mainTrackEnabled = this.mainTrackEnabled; + this.node.properties.audioTrackEnabled = this.audioTrackEnabled; + this.node.properties.motionTrackEnabled = this.motionTrackEnabled; + if (this.timeline.showFilenames !== undefined) { + this.node.properties.showFilenames = this.timeline.showFilenames; + } + if (this.timeline.overrideAudio !== undefined) { + this.node.properties.overrideAudio = this.timeline.overrideAudio; + } + if (this.timeline.inpaint_audio !== undefined) { + this.node.properties.inpaint_audio = this.timeline.inpaint_audio; + } + + // Sync widgets to match the timeline data + const inpaintWidget = this.node.widgets?.find(w => w.name === "inpaint_audio"); + if (inpaintWidget && this.timeline.inpaint_audio !== undefined) { + inpaintWidget.value = this.timeline.inpaint_audio; + } + const overrideWidget = this.node.widgets?.find(w => w.name === "override_audio"); + if (overrideWidget && this.timeline.overrideAudio !== undefined) { + overrideWidget.value = this.timeline.overrideAudio; + } + + this._audioTrackWasEnabledBeforeOverride = this.node.properties.audioTrackWasEnabledBeforeOverride || false; + this.loadMedia(); this.createDOM(); + this.updateRetakeUIState(); if (this.timeline.segments.length > 0) { this.selectedIndex = 0; } this.updateUIFromSelection(); + this.syncWidgetsAndUI(); this.commitChanges(true); // Hide settings widgets by default to reduce node clutter. // Deferred so all widget types are finalized before we touch them. @@ -682,14 +910,162 @@ class TimelineEditor { let isSyncing = false; + // --- Start Callbacks --- + const origStartFramesCallback = this.startFramesWidget?.callback; + if (this.startFramesWidget) { + this.startFramesWidget.callback = (...args) => { + if (origStartFramesCallback) origStartFramesCallback.apply(this.startFramesWidget, args); + + if (!isSyncing && this.startSecondsWidget && this.durationFramesWidget && this.endFramesWidget) { + isSyncing = true; + + let newStartFrames = this.getStartFrames(); + const endFrame = this.endFramesWidget.value || 1; + let newDurationFrames = Math.max(1, endFrame - newStartFrames); + + if (newDurationFrames <= 1) { + newStartFrames = endFrame - 1; + this.startFramesWidget.value = newStartFrames; + newDurationFrames = 1; + } + + this.startSecondsWidget.value = parseFloat((newStartFrames / this.getFrameRate()).toFixed(3)); + + this.durationFramesWidget.value = newDurationFrames; + if (this.durationSecondsWidget) { + this.durationSecondsWidget.value = parseFloat((newDurationFrames / this.getFrameRate()).toFixed(3)); + } + + this._prevStartFrames = newStartFrames; + this._prevStartSeconds = this.startSecondsWidget.value; + + isSyncing = false; + } + + this.commitChanges(); + }; + } + + const origStartSecondsCallback = this.startSecondsWidget?.callback; + if (this.startSecondsWidget) { + this.startSecondsWidget.callback = (...args) => { + if (origStartSecondsCallback) origStartSecondsCallback.apply(this.startSecondsWidget, args); + + if (!isSyncing && this.startFramesWidget && this.durationSecondsWidget && this.endFramesWidget) { + isSyncing = true; + + let newStartSeconds = this.startSecondsWidget.value; + let newStartFrames = Math.max(0, Math.round(newStartSeconds * this.getFrameRate())); + + const endFrame = this.endFramesWidget.value || 1; + let newDurationFrames = Math.max(1, endFrame - newStartFrames); + + if (newDurationFrames <= 1) { + newStartFrames = endFrame - 1; + newStartSeconds = newStartFrames / this.getFrameRate(); + this.startSecondsWidget.value = parseFloat(newStartSeconds.toFixed(3)); + newDurationFrames = 1; + } + + this.startFramesWidget.value = newStartFrames; + + this.durationFramesWidget.value = newDurationFrames; + if (this.durationSecondsWidget) { + this.durationSecondsWidget.value = parseFloat((newDurationFrames / this.getFrameRate()).toFixed(3)); + } + + this._prevStartFrames = newStartFrames; + this._prevStartSeconds = this.startSecondsWidget.value; + + isSyncing = false; + } + + this.commitChanges(); + }; + } + + // --- End Callbacks --- + const origEndFramesCallback = this.endFramesWidget?.callback; + if (this.endFramesWidget) { + this.endFramesWidget.callback = (...args) => { + if (origEndFramesCallback) origEndFramesCallback.apply(this.endFramesWidget, args); + + if (!isSyncing && this.endSecondsWidget && this.durationFramesWidget && this.startFramesWidget) { + isSyncing = true; + + let newEndFrames = this.endFramesWidget.value; + const startFrame = this.startFramesWidget.value || 0; + let newDurationFrames = Math.max(1, newEndFrames - startFrame); + + if (newDurationFrames <= 1) { + newEndFrames = startFrame + 1; + this.endFramesWidget.value = newEndFrames; + newDurationFrames = 1; + } + + this.endSecondsWidget.value = parseFloat((newEndFrames / this.getFrameRate()).toFixed(3)); + + this.durationFramesWidget.value = newDurationFrames; + if (this.durationSecondsWidget) { + this.durationSecondsWidget.value = parseFloat((newDurationFrames / this.getFrameRate()).toFixed(3)); + } + + isSyncing = false; + } + + this.commitChanges(); + }; + } + + const origEndSecondsCallback = this.endSecondsWidget?.callback; + if (this.endSecondsWidget) { + this.endSecondsWidget.callback = (...args) => { + if (origEndSecondsCallback) origEndSecondsCallback.apply(this.endSecondsWidget, args); + + if (!isSyncing && this.endFramesWidget && this.durationSecondsWidget && this.startFramesWidget) { + isSyncing = true; + + let newEndSeconds = this.endSecondsWidget.value; + let newEndFrames = Math.max(1, Math.round(newEndSeconds * this.getFrameRate())); + + const startFrame = this.startFramesWidget.value || 0; + let newDurationFrames = Math.max(1, newEndFrames - startFrame); + + if (newDurationFrames <= 1) { + newEndFrames = startFrame + 1; + newEndSeconds = newEndFrames / this.getFrameRate(); + this.endSecondsWidget.value = parseFloat(newEndSeconds.toFixed(3)); + newDurationFrames = 1; + } + + this.endFramesWidget.value = newEndFrames; + + this.durationFramesWidget.value = newDurationFrames; + if (this.durationSecondsWidget) { + this.durationSecondsWidget.value = parseFloat((newDurationFrames / this.getFrameRate()).toFixed(3)); + } + + isSyncing = false; + } + + this.commitChanges(); + }; + } + + // --- Duration Callbacks --- const origDurationFramesCallback = this.durationFramesWidget?.callback; if (this.durationFramesWidget) { this.durationFramesWidget.callback = (...args) => { if (origDurationFramesCallback) origDurationFramesCallback.apply(this.durationFramesWidget, args); - if (!isSyncing && this.durationSecondsWidget) { + if (!isSyncing && this.durationSecondsWidget && this.startFramesWidget && this.endFramesWidget) { isSyncing = true; this.durationSecondsWidget.value = parseFloat((this.getDurationFrames() / this.getFrameRate()).toFixed(3)); + + const newEndFrames = this.startFramesWidget.value + this.getDurationFrames(); + this.endFramesWidget.value = newEndFrames; + this.endSecondsWidget.value = parseFloat((newEndFrames / this.getFrameRate()).toFixed(3)); + isSyncing = false; } @@ -702,13 +1078,19 @@ class TimelineEditor { this.durationSecondsWidget.callback = (...args) => { if (origDurationSecondsCallback) origDurationSecondsCallback.apply(this.durationSecondsWidget, args); - if (!isSyncing && this.durationFramesWidget) { + if (!isSyncing && this.durationFramesWidget && this.startFramesWidget && this.endFramesWidget) { isSyncing = true; const newFrames = Math.max(1, Math.round(this.durationSecondsWidget.value * this.getFrameRate())); this.durationFramesWidget.value = newFrames; - if (this.durationFramesWidget.callback) this.durationFramesWidget.callback(newFrames); + + const newEndFrames = this.startFramesWidget.value + newFrames; + this.endFramesWidget.value = newEndFrames; + this.endSecondsWidget.value = parseFloat((newEndFrames / this.getFrameRate()).toFixed(3)); + isSyncing = false; } + + this.commitChanges(); }; } @@ -716,11 +1098,37 @@ class TimelineEditor { if (this.frameRateWidget) { this.frameRateWidget.callback = (...args) => { if (origFrameRateCallback) origFrameRateCallback.apply(this.frameRateWidget, args); - if (!isSyncing && this.durationSecondsWidget) { + + // Keep start_seconds and end_seconds constant; recompute frames to match the new rate. + if (!isSyncing && this.durationSecondsWidget && this.durationFramesWidget) { isSyncing = true; - this.durationSecondsWidget.value = parseFloat((this.getDurationFrames() / this.getFrameRate()).toFixed(3)); + const newFPS = this.getFrameRate(); + + // Recompute all segment frame values from their seconds snapshots. + // Using the snapshot avoids cumulative rounding errors when the user + // drags the slider rapidly through many intermediate FPS values. + this._rebaseSegmentsToFPS(newFPS); + + if (this.startSecondsWidget && this.startFramesWidget) { + const newStartFrames = Math.max(0, Math.round(this.startSecondsWidget.value * newFPS)); + this.startFramesWidget.value = newStartFrames; + this._prevStartFrames = newStartFrames; + } + + if (this.endSecondsWidget && this.endFramesWidget) { + const newEndFrames = Math.max(1, Math.round(this.endSecondsWidget.value * newFPS)); + this.endFramesWidget.value = newEndFrames; + } + + const newFrames = Math.max(1, Math.round(this.durationSecondsWidget.value * newFPS)); + this.durationFramesWidget.value = newFrames; + + // Update our tracked previous rate now that the change is complete. + this._prevFrameRate = newFPS; isSyncing = false; } + + this.commitChanges(); }; } @@ -739,6 +1147,125 @@ class TimelineEditor { this._renderLoop = requestAnimationFrame(() => this.checkResize()); } + isMultiSelectActive() { + if (!this.selectedSegmentIds || this.selectedSegmentIds.length <= 1) return false; + const baseIds = new Set(); + for (const id of this.selectedSegmentIds) { + const baseId = (id.endsWith("_v") || id.endsWith("_a")) ? id.slice(0, -2) : id; + baseIds.add(baseId); + } + return baseIds.size > 1; + } + + updateSelectionFromBox() { + if (!this._selectBoxStart || !this._selectBoxCurrent) return; + + const width = this.canvas.offsetWidth; + const totalFrames = this.getVisualDurationFrames(); + if (!width || totalFrames <= 0) return; + + const sx = this._selectBoxStart.x; + const sy = this._selectBoxStart.y; + const cx = this._selectBoxCurrent.x; + const cy = this._selectBoxCurrent.y; + + const left = Math.min(sx, cx); + const right = Math.max(sx, cx); + const top = Math.min(sy, cy); + const bottom = Math.max(sy, cy); + + const newSelectedIds = new Set(this._selectBoxInitialSelectedIds || []); + + for (const track of ["image", "motion", "audio"]) { + const arr = this.getSegmentArray(track); + if (!arr) continue; + + let trackTop = 0; + let trackBottom = 0; + + if (track === "image") { + trackTop = RULER_HEIGHT; + trackBottom = RULER_HEIGHT + this.blockHeight; + } else if (track === "audio") { + trackTop = RULER_HEIGHT + this.blockHeight; + trackBottom = RULER_HEIGHT + this.blockHeight + this.audioTrackHeight; + } else if (track === "motion") { + trackTop = RULER_HEIGHT + this.blockHeight + this.audioTrackHeight; + trackBottom = RULER_HEIGHT + this.blockHeight + this.audioTrackHeight + this.motionTrackHeight; + } + + for (const seg of arr) { + const startX = (seg.start / totalFrames) * width; + const pxWidth = (seg.length / totalFrames) * width; + const endX = startX + pxWidth; + + // Check rect intersection + const intersects = (left <= endX && right >= startX && top <= trackBottom && bottom >= trackTop); + + if (intersects) { + newSelectedIds.add(seg.id); + const sibId = seg.id.endsWith("_v") ? seg.id.slice(0, -2) + "_a" : (seg.id.endsWith("_a") ? seg.id.slice(0, -2) + "_v" : null); + if (sibId) { + newSelectedIds.add(sibId); + } + } + } + } + + this.selectedSegmentIds = Array.from(newSelectedIds); + this.syncSelectionTypeAndIndex(); + } + + syncSelectionTypeAndIndex() { + if (!this.selectedSegmentIds || this.selectedSegmentIds.length === 0) { + this._selectedIndex = -1; + return; + } + if (this.isMultiSelectActive()) { + this._selectedIndex = -1; + return; + } + // Sync single selection (which might be video + audio sibling) + const firstId = this.selectedSegmentIds[0]; + for (const track of ["image", "motion", "audio"]) { + const arr = this.getSegmentArray(track); + const idx = arr.findIndex(s => s.id === firstId); + if (idx !== -1) { + this.selectionType = track; + this._selectedIndex = idx; + break; + } + } + } + + get selectedIndex() { + return this._selectedIndex; + } + + set selectedIndex(val) { + this._selectedIndex = val; + if (this.selectedSegmentIds && !this.isMultiSelectActive()) { + if (val === -1) { + this.selectedSegmentIds = []; + } else { + const arr = this.getSegmentArray(this.selectionType); + const seg = arr ? arr[val] : null; + if (seg) { + this.selectedSegmentIds = [seg.id]; + if (seg.id.endsWith("_v")) { + const sibId = seg.id.slice(0, -2) + "_a"; + if (!this.selectedSegmentIds.includes(sibId)) this.selectedSegmentIds.push(sibId); + } else if (seg.id.endsWith("_a")) { + const sibId = seg.id.slice(0, -2) + "_v"; + if (!this.selectedSegmentIds.includes(sibId)) this.selectedSegmentIds.push(sibId); + } + } else { + this.selectedSegmentIds = []; + } + } + } + } + destroy() { cancelAnimationFrame(this._renderLoop); this.pauseAudio(); @@ -746,6 +1273,10 @@ class TimelineEditor { window.removeEventListener("paste", this.handlePaste, true); } + getStartFrames() { + return parseInt((this.startFramesWidget && this.startFramesWidget.value >= 0) ? this.startFramesWidget.value : 0, 10); + } + getDurationFrames() { return parseInt((this.durationFramesWidget && this.durationFramesWidget.value > 0) ? this.durationFramesWidget.value : 24, 10); } @@ -773,6 +1304,45 @@ class TimelineEditor { } } + // Force all start/end/duration widgets to match the retake video's duration exactly. + syncWidgetsToRetakeDuration(durationFrames) { + if (durationFrames <= 0) return; + const rate = this.getFrameRate(); + const durationSeconds = parseFloat((durationFrames / rate).toFixed(3)); + + const wasSuppressing = this._suppressCommit; + this._suppressCommit = true; + + if (this.startFramesWidget) { + this.startFramesWidget.value = 0; + if (this.startFramesWidget.callback) { + try { this.startFramesWidget.callback(0); } catch (_) {} + } + } + if (this.startSecondsWidget) { + this.startSecondsWidget.value = 0; + } + + if (this.durationFramesWidget) { + this.durationFramesWidget.value = durationFrames; + if (this.durationFramesWidget.callback) { + try { this.durationFramesWidget.callback(durationFrames); } catch (_) {} + } + } + if (this.durationSecondsWidget) { + this.durationSecondsWidget.value = durationSeconds; + } + + if (this.endFramesWidget) { + this.endFramesWidget.value = durationFrames; + } + if (this.endSecondsWidget) { + this.endSecondsWidget.value = durationSeconds; + } + + this._suppressCommit = wasSuppressing; + } + // Returns the maximum allowed zoom level, computed so that at max zoom // the viewport shows exactly 4 seconds of the visual timeline. getMaxZoom() { @@ -791,6 +1361,17 @@ class TimelineEditor { // the furthest segment end (across both tracks) × 1.30, with a floor of getDurationFrames(). // This is used for all rendering/positioning — the actual output duration is getDurationFrames(). getVisualDurationFrames() { + if (this.retakeMode) { + if (this.timeline.retakeVideo) { + const baseVideoDur = this.timeline.retakeVideo.videoDurationFrames || 0; + // Add 15% visual buffer duration on the right to prevent the video segment + // from being cut off by the DOM clipping (right ~9% of the viewport is clipped by ComfyUI). + return Math.max(24, Math.ceil(baseVideoDur * 1.15)); + } else { + return 24; + } + } + let furthest = 0; for (const seg of this.timeline.segments) { furthest = Math.max(furthest, seg.start + seg.length); @@ -798,6 +1379,9 @@ class TimelineEditor { for (const seg of this.timeline.audioSegments) { furthest = Math.max(furthest, seg.start + seg.length); } + for (const seg of this.timeline.motionSegments) { + furthest = Math.max(furthest, seg.start + seg.length); + } const outputDuration = this.getDurationFrames(); if (furthest <= 0) return outputDuration; return Math.max(outputDuration, Math.ceil(furthest * 1.30)); @@ -822,15 +1406,674 @@ class TimelineEditor { } } - loadImages() { - for (const seg of this.timeline.segments) { - const src = imagePreviewSrc(seg); - if (src && !seg.imgObj) { - seg._imagePreviewSrc = src; - seg.imgObj = new Image(); - seg.imgObj.onload = () => this.render(); - seg.imgObj.src = src; + _liveScrubVideo(seg, edge) { + if (!seg || (seg.type !== "video" && seg.type !== "motion_video")) return; + this._ensureVideoEl(seg); + if (!seg.videoEl) return; + const targetSec = edge === "end" + ? (seg.trimStart + seg.length) / this.getFrameRate() + : seg.trimStart / this.getFrameRate(); + + seg._scrubTargetSec = targetSec; + } + + _liveScrubPlayhead() { + const targetFrame = this.currentFrame; + if (this.retakeMode && this.timeline.retakeVideo) { + const retakeVid = this.timeline.retakeVideo; + this._ensureVideoEl(retakeVid); + if (retakeVid.videoEl) { + const targetSec = targetFrame / this.getFrameRate(); + retakeVid._scrubTargetSec = targetSec; } + return; + } + + const seg = this.timeline.segments.find(s => s.type === "video" && targetFrame >= s.start && targetFrame < s.start + s.length); + if (seg) { + this._ensureVideoEl(seg); + if (seg.videoEl) { + const targetSec = (seg.trimStart + (targetFrame - seg.start)) / this.getFrameRate(); + seg._scrubTargetSec = targetSec; + } + } + + const motionSeg = this.timeline.motionSegments.find(s => s.type === "motion_video" && targetFrame >= s.start && targetFrame < s.start + s.length); + if (motionSeg) { + this._ensureVideoEl(motionSeg); + if (motionSeg.videoEl) { + const targetSec = (motionSeg.trimStart + (targetFrame - motionSeg.start)) / this.getFrameRate(); + motionSeg._scrubTargetSec = targetSec; + } + } + } + + async _ensureThumbnails(seg) { + if (seg.thumbnails) return; + if (seg._extractingThumbs) return; + + const fileKey = seg.imageFile || seg.videoFile || seg._blobUrl; + if (!fileKey) return; + + this._thumbnailCache = this._thumbnailCache || new Map(); + this._thumbnailPromises = this._thumbnailPromises || new Map(); + + if (this._thumbnailCache.has(fileKey)) { + seg.thumbnails = this._thumbnailCache.get(fileKey); + this.render(); + return; + } + + if (this._thumbnailPromises.has(fileKey)) { + seg._extractingThumbs = true; + try { + const thumbs = await this._thumbnailPromises.get(fileKey); + seg.thumbnails = thumbs; + } catch (err) { + console.error("Failed to await thumbnails promise:", err); + } finally { + seg._extractingThumbs = false; + this.render(); + } + return; + } + + // Otherwise, we extract the thumbnails + seg._extractingThumbs = true; + seg.thumbnails = []; + + const extractPromise = (async () => { + const thumbs = []; + const parts = fileKey.split(/[/\\\\]/); + const filename = parts.pop() || ''; + const subfolder = parts.join('/'); + const vidUrl = seg._blobUrl || (seg.videoEl ? seg.videoEl.src : null) || api.apiURL(`/view?filename=${encodeURIComponent(filename)}&type=input&subfolder=${encodeURIComponent(subfolder)}`); + + const bgVid = document.createElement('video'); + bgVid.crossOrigin = "Anonymous"; + bgVid.muted = true; + bgVid.preload = 'auto'; + + try { + await new Promise(r => { + let resolved = false; + const done = () => { + if (!resolved) { + resolved = true; + r(); + } + }; + bgVid.onloadeddata = done; + bgVid.onerror = done; + bgVid.src = vidUrl; + if (bgVid.readyState >= 2) { + done(); + } + }); + + if (!bgVid.duration) { + return thumbs; + } + + const duration = bgVid.duration; + const isLargeFile = seg.fileSize > 500 * 1024 * 1024; + const numFrames = isLargeFile ? 10 : Math.max(5, Math.min(25, Math.ceil(duration * 1.0))); + const canvas = document.createElement('canvas'); + let w = bgVid.videoWidth, h = bgVid.videoHeight; + if (w === 0 || h === 0) return thumbs; + + if (h > this.blockHeight) { + w = Math.round(w * (this.blockHeight / h)); + h = this.blockHeight; + } + canvas.width = w; canvas.height = h; + const ctx = canvas.getContext('2d'); + + for (let i = 0; i < numFrames; i++) { + // Check if the file/segment is still active in the current timeline + const exists = this.timeline.segments.find(s => s.imageFile === fileKey || s.videoFile === fileKey || s._blobUrl === fileKey) || + this.timeline.motionSegments.find(s => s.imageFile === fileKey || s.videoFile === fileKey || s._blobUrl === fileKey) || + (this.timeline.retakeVideo && (this.timeline.retakeVideo.imageFile === fileKey || this.timeline.retakeVideo._blobUrl === fileKey)); + if (!exists) break; + + const time = (i / numFrames) * duration; + bgVid.currentTime = time; + + await new Promise(r => { + let resolved = false; + const onSeek = () => { if (!resolved) { resolved = true; r(); } }; + bgVid.onseeked = onSeek; + setTimeout(onSeek, 1000); + }); + + ctx.drawImage(bgVid, 0, 0, w, h); + const img = new Image(); + img.src = canvas.toDataURL('image/jpeg', 0.5); + await new Promise(r => { img.onload = r; }); + + thumbs.push({ time, img }); + + // Propagate the partial progress live to all active segments sharing this file + const matchingSegs = [ + ...this.timeline.segments.filter(s => s.imageFile === fileKey || s.videoFile === fileKey || s._blobUrl === fileKey), + ...(this.timeline.motionSegments || []).filter(s => s.imageFile === fileKey || s.videoFile === fileKey || s._blobUrl === fileKey) + ]; + if (this.timeline.retakeVideo && (this.timeline.retakeVideo.imageFile === fileKey || this.timeline.retakeVideo._blobUrl === fileKey)) { + matchingSegs.push(this.timeline.retakeVideo); + } + for (const ms of matchingSegs) { + ms.thumbnails = thumbs; + } + + this.render(); + } + } catch (err) { + console.error("Thumbnail extraction loop failed:", err); + } finally { + try { + bgVid.pause(); + bgVid.onloadeddata = null; + bgVid.onerror = null; + bgVid.onseeked = null; + bgVid.src = ""; + bgVid.load(); + } catch (_) { } + } + return thumbs; + })(); + + this._thumbnailPromises.set(fileKey, extractPromise); + + try { + const thumbs = await extractPromise; + this._thumbnailCache.set(fileKey, thumbs); + + const matchingSegs = [ + ...this.timeline.segments.filter(s => s.imageFile === fileKey || s.videoFile === fileKey || s._blobUrl === fileKey), + ...(this.timeline.motionSegments || []).filter(s => s.imageFile === fileKey || s.videoFile === fileKey || s._blobUrl === fileKey) + ]; + if (this.timeline.retakeVideo && (this.timeline.retakeVideo.imageFile === fileKey || this.timeline.retakeVideo._blobUrl === fileKey)) { + matchingSegs.push(this.timeline.retakeVideo); + } + for (const ms of matchingSegs) { + ms.thumbnails = thumbs; + ms._extractingThumbs = false; + + // If fileKey is a blob URL, and the segment now has a server file path, cache under that path too + if (fileKey.startsWith("blob:")) { + const serverKey = ms.imageFile || ms.videoFile; + if (serverKey) { + this._thumbnailCache.set(serverKey, thumbs); + } + } + } + } catch (err) { + console.error("Extraction error:", err); + const matchingSegs = [ + ...this.timeline.segments.filter(s => s.imageFile === fileKey || s.videoFile === fileKey || s._blobUrl === fileKey), + ...(this.timeline.motionSegments || []).filter(s => s.imageFile === fileKey || s.videoFile === fileKey || s._blobUrl === fileKey) + ]; + if (this.timeline.retakeVideo && (this.timeline.retakeVideo.imageFile === fileKey || this.timeline.retakeVideo._blobUrl === fileKey)) { + matchingSegs.push(this.timeline.retakeVideo); + } + for (const ms of matchingSegs) { + ms._extractingThumbs = false; + } + } finally { + this._thumbnailPromises.delete(fileKey); + this.render(); + } + } + + getSegmentArray(trackType) { + if (trackType === "motion") return this.timeline.motionSegments; + if (trackType === "audio") return this.timeline.audioSegments; + return this.timeline.segments; + } + + getSnappedPlayhead(mouseFrameX, logicalWidth) { + if (!this.isSnapping) return mouseFrameX; + + const totalFrames = this.getVisualDurationFrames(); + const thresholdFrames = (15 / logicalWidth) * totalFrames; + const snapCandidates = [0, this.getDurationFrames()]; + + // Add start and end frames of active generation range + snapCandidates.push(this.getStartFrames()); + if (this.endFramesWidget && this.endFramesWidget.value !== undefined) { + snapCandidates.push(parseInt(this.endFramesWidget.value, 10)); + } + + if (this.retakeMode) { + if (this.timeline.retakeVideo) { + const baseVideoDur = this.timeline.retakeVideo.videoDurationFrames || 0; + snapCandidates.push(baseVideoDur); + } + if (this.timeline.retakeStart !== undefined) { + snapCandidates.push(this.timeline.retakeStart); + if (this.timeline.retakeLength !== undefined) { + snapCandidates.push(this.timeline.retakeStart + this.timeline.retakeLength); + } + } + } + + const allTracks = [ + this.timeline.segments || [], + this.timeline.motionSegments || [], + this.timeline.audioSegments || [] + ]; + for (const track of allTracks) { + for (const seg of track) { + snapCandidates.push(seg.start); + snapCandidates.push(seg.start + seg.length); + } + } + + let bestFrame = mouseFrameX; + let minDiff = thresholdFrames; + for (const candidate of snapCandidates) { + const diff = Math.abs(mouseFrameX - candidate); + if (diff < minDiff) { + minDiff = diff; + bestFrame = candidate; + } + } + return bestFrame; + } + + getTrackFromY(y) { + if (y > RULER_HEIGHT + this.blockHeight + this.audioTrackHeight) return "motion"; + if (y > RULER_HEIGHT + this.blockHeight) return "audio"; + return "image"; + } + + _ensureVideoEl(seg) { + if (!seg) return; + + if (seg.videoEl) { + if (seg.videoEl.duration && !seg.videoDurationFrames) { + const frameRate = this.getFrameRate(); + seg.videoDurationFrames = Math.max(1, Math.ceil(seg.videoEl.duration * frameRate)); + } + if (this.retakeMode && seg === this.timeline.retakeVideo && seg.videoDurationFrames) { + this.syncWidgetsToRetakeDuration(seg.videoDurationFrames); + this.updateZoomSliderMax(); + this.commitChanges(true); + } + return; + } + + const cacheKey = seg.imageFile || seg.videoFile || seg._blobUrl; + if (!cacheKey) return; + + this._videoElementsCache = this._videoElementsCache || new Map(); + + if (this._videoElementsCache.has(cacheKey)) { + // Reuse the existing shared video element — do NOT re-seek it. + // Running initVideoSeek on an already-initialized element causes cascading seeks + // when multiple split segments share it (e.g. seg2 seeks to 5min, seg3 seeks to 10min), + // which breaks playback on long videos. Just grab the reference and ensure thumbnails. + seg.videoEl = this._videoElementsCache.get(cacheKey); + if (seg.videoEl.duration && !seg.videoDurationFrames) { + const frameRate = this.getFrameRate(); + seg.videoDurationFrames = Math.max(1, Math.ceil(seg.videoEl.duration * frameRate)); + } + if (this.retakeMode && seg === this.timeline.retakeVideo && seg.videoDurationFrames) { + this.syncWidgetsToRetakeDuration(seg.videoDurationFrames); + this.updateZoomSliderMax(); + this.commitChanges(true); + } + this._ensureThumbnails(seg); + return; + } + + const isRetake = seg === this.timeline?.retakeVideo; + const isVideo = (seg.type === "video" || isRetake) && (seg.imageFile || seg._blobUrl); + const isMotionVideo = seg.type === "motion_video" && seg.videoFile; + if (!isVideo && !isMotionVideo) return; + + const fileKey = (seg.type === "video" || isRetake) ? seg.imageFile : seg.videoFile; + let vidUrl = seg._blobUrl; + if (!vidUrl && fileKey) { + const fileParts = fileKey.split(/[/\\\\]/); + const justName = fileParts.pop() || ''; + const subfolder = fileParts.join('/'); + vidUrl = api.apiURL(`/view?filename=${encodeURIComponent(justName)}&type=input&subfolder=${encodeURIComponent(subfolder)}`); + } + if (!vidUrl) return; + + const vid = document.createElement('video'); + vid.crossOrigin = "Anonymous"; + vid.muted = true; + vid.preload = 'auto'; + + seg.videoEl = vid; + this._videoElementsCache.set(cacheKey, vid); + + vid.addEventListener('seeked', () => { + this.render(); + }); + + const onSeekedHandler = () => { + vid.removeEventListener('seeked', onSeekedHandler); + if (!seg.imageB64 || !seg.imgObj) { + const canvas = document.createElement('canvas'); + canvas.width = Math.min(vid.videoWidth, 512); + canvas.height = Math.round((vid.videoHeight / vid.videoWidth) * canvas.width); + canvas.getContext('2d').drawImage(vid, 0, 0, canvas.width, canvas.height); + seg.imageB64 = canvas.toDataURL('image/jpeg'); + const img = new Image(); + img.onload = () => { + seg.imgObj = img; + this.render(); + this.commitChanges(true); + }; + img.src = seg.imageB64; + } else { + this.render(); + } + }; + + let seekInitialized = false; + const initVideoSeek = () => { + if (seekInitialized) return; + seekInitialized = true; + + if (vid.duration) { + const frameRate = this.getFrameRate(); + const clipFrames = Math.max(1, Math.ceil(vid.duration * frameRate)); + seg.videoDurationFrames = clipFrames; + if (this.retakeMode && seg === this.timeline.retakeVideo) { + this.syncWidgetsToRetakeDuration(clipFrames); + this.updateZoomSliderMax(); + this.commitChanges(true); + } + } + + vid.addEventListener('seeked', onSeekedHandler); + vid.currentTime = (seg.trimStart || 0) / this.getFrameRate() + 0.01; + this._ensureThumbnails(seg); + }; + + vid.addEventListener('loadedmetadata', initVideoSeek, { once: true }); + vid.addEventListener('loadeddata', initVideoSeek, { once: true }); + + vid.src = vidUrl; + + if (vid.readyState >= 1) { + initVideoSeek(); + } + } + + async _getOrExtractAudio(seg) { + if (!seg.audioFile) return; + const isVideoFile = seg.audioFile.toLowerCase().match(/\.(mp4|webm|mkv|avi|mov|m4v|flv|wmv)$/); + if (!isVideoFile) return; + + this._audioExtractionPromises = this._audioExtractionPromises || new Map(); + const fileKey = seg.audioFile; + + if (this._audioExtractionPromises.has(fileKey)) { + try { + const res = await this._audioExtractionPromises.get(fileKey); + if (res && res.audio_file && res.peaks) { + seg.audioFile = res.audio_file; + seg.waveformPeaks = res.peaks; + } + } catch (err) { + console.warn("[LTXDirector] Awaiting shared server audio extract promise failed:", err); + } + return; + } + + const extractionPromise = (async () => { + const resp = await api.fetchApi(`/ltx_director_get_audio?filename=${encodeURIComponent(fileKey)}`); + if (resp.status === 200) { + return await resp.json(); + } + throw new Error(`Server returned status ${resp.status}`); + })(); + + this._audioExtractionPromises.set(fileKey, extractionPromise); + + try { + const res = await extractionPromise; + if (res && res.audio_file && res.peaks) { + seg.audioFile = res.audio_file; + seg.waveformPeaks = res.peaks; + + // Update all other segments matching this fileKey in the timeline + const allAudioSegs = this.timeline.audioSegments || []; + for (const s of allAudioSegs) { + if (s.audioFile === fileKey) { + s.audioFile = res.audio_file; + s.waveformPeaks = res.peaks; + } + } + } + } catch (err) { + console.warn("[LTXDirector] Server audio check/extract failed:", err); + } finally { + this._audioExtractionPromises.delete(fileKey); + } + } + + _extractAudioOnClient(file, audSegId, blobUrl) { + (async () => { + try { + const arrayBuffer = await file.arrayBuffer(); + const audioCtx = new (window.AudioContext || window.webkitAudioContext)(); + const audioBuffer = await audioCtx.decodeAudioData(arrayBuffer); + const channelData = audioBuffer.getChannelData(0); + const peaks = []; + const numPeaks = 200; + const step = Math.floor(channelData.length / numPeaks); + for (let i = 0; i < numPeaks; i++) { + let max = 0; + for (let j = 0; j < step; j++) { + const val = Math.abs(channelData[i * step + j]); + if (val > max) max = val; + } + peaks.push(max); + } + for (let s of this.timeline.audioSegments) { + if (s.id === audSegId || (blobUrl && s._blobUrl === blobUrl)) { + s.waveformPeaks = peaks; + s._decoding = false; + s._audioBuffer = audioBuffer; + } + } + this.render(); + } catch (e) { + console.warn("No audio in video or decode failed", e); + for (let s of this.timeline.audioSegments) { + if (s.id === audSegId || (blobUrl && s._blobUrl === blobUrl)) { + s._decoding = false; + } + } + this.render(); + } + })(); + } + + _isAudioDecodingAllowed(seg) { + if (seg.audioFile && seg.audioFile.toLowerCase().match(/\.(wav|mp3|ogg|flac|m4a)$/)) { + return true; + } + const isVideo = (seg.audioFile && seg.audioFile.toLowerCase().match(/\.(mp4|webm|mkv|avi|mov|m4v|flv|wmv)$/)) || + (!seg.audioFile && seg._blobUrl); + if (isVideo) { + const isSmall = seg.fileSize && seg.fileSize <= 100 * 1024 * 1024; + return !!isSmall; + } + return true; + } + + async _preloadAudioSegment(seg) { + if (seg._audioBuffer || seg._decoding) return; + if (!seg.audioFile && !seg._blobUrl) return; + + seg._decoding = true; + if (!this._isDragging) this.render(); + + try { + await this._getOrExtractAudio(seg); + + if (!this._isAudioDecodingAllowed(seg)) { + seg._decoding = false; + return; + } + + if (!this.audioContext) { + this.audioContext = new (window.AudioContext || window.webkitAudioContext)(); + } + + const parts = (seg.audioFile || "").split(/[/\\\\]/); + const filename = parts.pop() || ''; + const subfolder = parts.join('/'); + const audioUrl = seg._blobUrl || api.apiURL(`/view?filename=${encodeURIComponent(filename)}&type=input&subfolder=${encodeURIComponent(subfolder)}`); + + this._audioBufferCache = this._audioBufferCache || new Map(); + this._audioBufferPromises = this._audioBufferPromises || new Map(); + const cacheKey = seg.audioFile || audioUrl; + + let audioBuffer; + if (this._audioBufferCache.has(cacheKey)) { + audioBuffer = this._audioBufferCache.get(cacheKey); + } else if (this._audioBufferPromises.has(cacheKey)) { + audioBuffer = await this._audioBufferPromises.get(cacheKey); + } else { + const decodePromise = (async () => { + const resp = await fetch(audioUrl); + const arrayBuffer = await resp.arrayBuffer(); + return await this.audioContext.decodeAudioData(arrayBuffer); + })(); + this._audioBufferPromises.set(cacheKey, decodePromise); + try { + audioBuffer = await decodePromise; + this._audioBufferCache.set(cacheKey, audioBuffer); + } finally { + this._audioBufferPromises.delete(cacheKey); + } + } + + const matchingSegs = this.timeline.audioSegments.filter(s => s.audioFile === seg.audioFile || s._blobUrl === seg._blobUrl); + for (const s of matchingSegs) { + s._audioBuffer = audioBuffer; + s._decoding = false; + } + } catch (err) { + console.warn("Failed to preload audio segment:", err); + seg._decoding = false; + } finally { + if (!this._isDragging) this.render(); + } + } + + + async _preloadMotionAudioSegment(seg) { + if (seg._audioBuffer || seg._decodingAudio) return; + if (!seg.videoFile && !seg._blobUrl) return; + + seg._decodingAudio = true; + + try { + const mockSeg = { + audioFile: seg.videoFile || seg.fileName, + _blobUrl: seg._blobUrl, + fileSize: seg.fileSize + }; + + await this._getOrExtractAudio(mockSeg); + + if (!this._isAudioDecodingAllowed(mockSeg)) { + seg._decodingAudio = false; + return; + } + + if (!this.audioContext) { + this.audioContext = new (window.AudioContext || window.webkitAudioContext)(); + } + + const parts = (mockSeg.audioFile || "").split(/[/\\\\]/); + const filename = parts.pop() || ''; + const subfolder = parts.join('/'); + const audioUrl = mockSeg._blobUrl || api.apiURL(`/view?filename=${encodeURIComponent(filename)}&type=input&subfolder=${encodeURIComponent(subfolder)}`); + + this._audioBufferCache = this._audioBufferCache || new Map(); + this._audioBufferPromises = this._audioBufferPromises || new Map(); + const cacheKey = mockSeg.audioFile || audioUrl; + + let audioBuffer; + if (this._audioBufferCache.has(cacheKey)) { + audioBuffer = this._audioBufferCache.get(cacheKey); + } else if (this._audioBufferPromises.has(cacheKey)) { + audioBuffer = await this._audioBufferPromises.get(cacheKey); + } else { + const decodePromise = (async () => { + const resp = await fetch(audioUrl); + const arrayBuffer = await resp.arrayBuffer(); + return await this.audioContext.decodeAudioData(arrayBuffer); + })(); + this._audioBufferPromises.set(cacheKey, decodePromise); + try { + audioBuffer = await decodePromise; + this._audioBufferCache.set(cacheKey, audioBuffer); + } finally { + this._audioBufferPromises.delete(cacheKey); + } + } + seg._audioBuffer = audioBuffer; + } catch (e) { + console.warn("Failed to preload motion audio segment:", e); + } finally { + seg._decodingAudio = false; + } + } + + + loadMedia() { + for (const seg of this.timeline.segments) { + if (seg.imageB64 && !seg.imgObj) { + seg.imgObj = new Image(); + seg.imgObj.onload = () => { if (!this._isDragging) this.render(); }; + seg.imgObj.src = seg.imageB64; + } + if (seg.type === "video") { + this._ensureVideoEl(seg); + this._ensureThumbnails(seg); + } + } + + if (this.timeline.motionSegments) { + const isOverrideAudio = !!(this.node.properties.overrideAudio || this.timeline.overrideAudio); + for (const seg of this.timeline.motionSegments) { + if (seg.imageB64 && !seg.imgObj) { + seg.imgObj = new Image(); + seg.imgObj.onload = () => { if (!this._isDragging) this.render(); }; + seg.imgObj.src = seg.imageB64; + } + if (seg.type === "motion_video") { + this._ensureVideoEl(seg); + this._ensureThumbnails(seg); + if (isOverrideAudio) { + this._preloadMotionAudioSegment(seg); + } + } + } + } + + if (this.timeline.audioSegments) { + for (const seg of this.timeline.audioSegments) { + if (seg.type === "audio") { + this._preloadAudioSegment(seg); + } + } + } + + if (this.timeline.retakeVideo) { + this._ensureVideoEl(this.timeline.retakeVideo); + this._ensureThumbnails(this.timeline.retakeVideo); } } @@ -845,6 +2088,8 @@ class TimelineEditor { const activeTag = document.activeElement ? document.activeElement.tagName : ""; if (activeTag === "INPUT" || activeTag === "TEXTAREA") return; + const isCtrl = e.ctrlKey || e.metaKey; + if ((e.key === "Delete" || e.key === "Backspace") && this.selectedIndex !== -1 && this._isHovering) { this.deleteSelectedSegment(); e.stopPropagation(); @@ -855,6 +2100,87 @@ class TimelineEditor { e.stopPropagation(); e.stopImmediatePropagation(); e.preventDefault(); + } else if ((e.key === "b" || e.key === "B") && isCtrl && this._isHovering) { + if (this.selectedIndex !== -1) { + const arr = this.getSegmentArray(this.selectionType); + const seg = arr[this.selectedIndex]; + if (seg) this.splitSegmentAtPlayhead(seg, this.selectionType); + } + e.stopPropagation(); + e.stopImmediatePropagation(); + e.preventDefault(); + } else if ((e.key === "c" || e.key === "C") && isCtrl && this._isHovering) { + if (this.selectedIndex !== -1) { + const arr = this.getSegmentArray(this.selectionType); + const seg = arr[this.selectedIndex]; + if (seg) { + window._ltxCopiedSegment = { main: { ...seg }, sibling: null }; + window._ltxCopiedSegmentType = this.selectionType; + + // Keep image/video elements + if (seg.imgObj) window._ltxCopiedSegment.main.imgObj = seg.imgObj; + if (seg.videoEl) window._ltxCopiedSegment.main.videoEl = seg.videoEl; + + if (seg.id.endsWith("_v") || seg.id.endsWith("_a")) { + const isVid = seg.id.endsWith("_v"); + const sibId = isVid ? seg.id.slice(0, -2) + "_a" : seg.id.slice(0, -2) + "_v"; + const sibArr = isVid ? this.timeline.audioSegments : this.timeline.segments; + const sib = sibArr.find(s => s.id === sibId); + if (sib) { + window._ltxCopiedSegment.sibling = { ...sib }; + if (sib.imgObj) window._ltxCopiedSegment.sibling.imgObj = sib.imgObj; + if (sib.videoEl) window._ltxCopiedSegment.sibling.videoEl = sib.videoEl; + } + } + } + } + } else if ((e.key === "v" || e.key === "V") && isCtrl && this._isHovering) { + if (window._ltxCopiedSegment) { + this.pasteCopiedSegment(); + e.stopPropagation(); + e.stopImmediatePropagation(); + e.preventDefault(); + } + } else if ((e.key === "s" || e.key === "S") && !isCtrl && this._isHovering) { + this.isSnapping = !this.isSnapping; + this.node.properties.isSnapping = this.isSnapping; + if (typeof this.updateSnapStyle === "function") { + this.updateSnapStyle(); + } + this.commitChanges(); + this.render(); + e.stopPropagation(); + e.stopImmediatePropagation(); + e.preventDefault(); + } else if ((e.key === "i" || e.key === "I") && !isCtrl && this._isHovering) { + if (this.startFramesWidget) { + this.startFramesWidget.value = this.currentFrame; + if (this.startFramesWidget.callback) { + this.startFramesWidget.callback(this.currentFrame); + } + this.commitChanges(); + this.render(); + } + e.stopPropagation(); + e.stopImmediatePropagation(); + e.preventDefault(); + } else if ((e.key === "o" || e.key === "O") && !isCtrl && this._isHovering) { + if (this.endFramesWidget) { + this.endFramesWidget.value = this.currentFrame; + if (this.endFramesWidget.callback) { + this.endFramesWidget.callback(this.currentFrame); + } + this.commitChanges(); + this.render(); + } + e.stopPropagation(); + e.stopImmediatePropagation(); + e.preventDefault(); + } else if ((e.key === "x" || e.key === "X") && !isCtrl && this._isHovering) { + this.markCurrentSelection(); + e.stopPropagation(); + e.stopImmediatePropagation(); + e.preventDefault(); } }; window.addEventListener("keydown", this.handleKeyDown, true); @@ -897,32 +2223,79 @@ class TimelineEditor { this.audioFileInput.style.display = "none"; this.audioFileInput.addEventListener("change", (e) => this.handleAudioUpload(e.target.files)); + this.motionFileInput = document.createElement("input"); + this.motionFileInput.type = "file"; + this.motionFileInput.accept = "video/*"; + this.motionFileInput.multiple = true; + this.motionFileInput.style.display = "none"; + this.motionFileInput.addEventListener("change", (e) => this.handleMotionUpload(e.target.files)); + + this.videoFileInput = document.createElement("input"); + this.videoFileInput.type = "file"; + this.videoFileInput.accept = "video/*"; + this.videoFileInput.multiple = true; + this.videoFileInput.style.display = "none"; + this.videoFileInput.addEventListener("change", (e) => this.handleVideoUpload(e.target.files)); + const uploadBtn = document.createElement("button"); uploadBtn.className = "pr-btn"; uploadBtn.innerHTML = `${ICONS.upload} Add Image`; uploadBtn.addEventListener("click", () => this.fileInput.click()); + this.uploadBtn = uploadBtn; const uploadAudioBtn = document.createElement("button"); uploadAudioBtn.className = "pr-btn"; uploadAudioBtn.innerHTML = `${ICONS.audio} Add Audio`; uploadAudioBtn.addEventListener("click", () => this.audioFileInput.click()); + this.uploadAudioBtn = uploadAudioBtn; + + const uploadMotionBtn = document.createElement("button"); + uploadMotionBtn.className = "pr-btn"; + uploadMotionBtn.innerHTML = `${ICONS.motion} Add IC Video`; + uploadMotionBtn.addEventListener("click", () => this.motionFileInput.click()); + this.uploadMotionBtn = uploadMotionBtn; + + const uploadVideoBtn = document.createElement("button"); + uploadVideoBtn.className = "pr-btn"; + uploadVideoBtn.innerHTML = ` Add Video`; + uploadVideoBtn.addEventListener("click", () => this.videoFileInput.click()); + this.uploadVideoBtn = uploadVideoBtn; const addTextBtn = document.createElement("button"); addTextBtn.className = "pr-btn"; addTextBtn.innerHTML = `${ICONS.text} Add Text`; addTextBtn.addEventListener("click", () => this.addTextSegmentFreeSpace()); + this.addTextBtn = addTextBtn; const deleteBtn = document.createElement("button"); deleteBtn.className = "pr-btn pr-btn-danger"; deleteBtn.innerHTML = `${ICONS.trash} Delete`; deleteBtn.addEventListener("click", () => this.deleteSelectedSegment()); + this.deleteBtn = deleteBtn; actionGroup.appendChild(this.fileInput); actionGroup.appendChild(this.audioFileInput); + actionGroup.appendChild(this.motionFileInput); + actionGroup.appendChild(this.videoFileInput); actionGroup.appendChild(uploadBtn); actionGroup.appendChild(addTextBtn); actionGroup.appendChild(uploadAudioBtn); + actionGroup.appendChild(uploadVideoBtn); + actionGroup.appendChild(uploadMotionBtn); actionGroup.appendChild(deleteBtn); + + // Retake-mode-only delete button (shown next to Add Video when retakeMode is on) + const deleteRetakeBtn = document.createElement("button"); + deleteRetakeBtn.className = "pr-btn pr-btn-danger"; + deleteRetakeBtn.innerHTML = `${ICONS.trash} Delete`; + deleteRetakeBtn.title = "Remove retake video"; + deleteRetakeBtn.style.display = "none"; // hidden until retakeMode + video loaded + deleteRetakeBtn.addEventListener("click", () => { + this._deleteRetakeVideo(); + }); + this.deleteRetakeBtn = deleteRetakeBtn; + actionGroup.appendChild(deleteRetakeBtn); + toolbar.appendChild(actionGroup); const rightGroup = document.createElement("div"); @@ -930,7 +2303,7 @@ class TimelineEditor { this.segmentBoundsDisplay = document.createElement("div"); this.segmentBoundsDisplay.className = "pr-segment-bounds"; - this.segmentBoundsDisplay.textContent = "Start: - | End: -"; + this.segmentBoundsDisplay.textContent = "Start: - | End: - | Length: -"; this.timeCodeDisplay = document.createElement("div"); this.timeCodeDisplay.className = "pr-timecode"; @@ -939,6 +2312,8 @@ class TimelineEditor { const settingsBtn = document.createElement("button"); settingsBtn.className = "pr-btn"; settingsBtn.style.padding = "6px"; + settingsBtn.style.display = "flex"; + settingsBtn.style.alignItems = "center"; settingsBtn.style.justifyContent = "center"; settingsBtn.style.width = "28px"; settingsBtn.style.height = "28px"; @@ -954,64 +2329,357 @@ class TimelineEditor { } }); - const toggleBtn = document.createElement("button"); - toggleBtn.className = "pr-btn"; - toggleBtn.style.padding = "6px 8px"; - toggleBtn.style.fontSize = "11px"; - toggleBtn.style.marginRight = "0px"; - toggleBtn.textContent = "Custom Audio: OFF"; - toggleBtn.title = "Toggle Custom Audio Output"; + const inpaintToggleBtn = document.createElement("button"); + inpaintToggleBtn.className = "pr-btn"; + inpaintToggleBtn.style.padding = "4px 0px"; + inpaintToggleBtn.style.fontSize = "9px"; + inpaintToggleBtn.style.lineHeight = "1"; + inpaintToggleBtn.style.marginRight = "0px"; + inpaintToggleBtn.style.marginTop = "8px"; // Adjust this value to fine-tune spacing between the title and button + inpaintToggleBtn.style.width = "72px"; + inpaintToggleBtn.style.whiteSpace = "nowrap"; + inpaintToggleBtn.style.textAlign = "center"; + inpaintToggleBtn.style.justifyContent = "center"; + inpaintToggleBtn.style.alignItems = "center"; + inpaintToggleBtn.style.gap = "0px"; + inpaintToggleBtn.style.boxSizing = "border-box"; + inpaintToggleBtn.style.borderRadius = "2px"; + inpaintToggleBtn.textContent = "Inpaint: ON"; + inpaintToggleBtn.title = "Toggle Audio Inpainting in Gaps"; - const updateToggleStyle = (isOn) => { - toggleBtn.textContent = isOn ? "Custom Audio: ON" : "Custom Audio: OFF"; + this.updateInpaintToggleStyle = (isOn) => { + inpaintToggleBtn.textContent = isOn ? "Inpaint: ON" : "Inpaint: OFF"; if (isOn) { - toggleBtn.style.background = "#1c222d"; - toggleBtn.style.borderColor = "#283142"; - toggleBtn.style.color = "#e0e0e0"; + inpaintToggleBtn.classList.add("toggle-on"); } else { - toggleBtn.style.background = "#222"; - toggleBtn.style.borderColor = "#111"; - toggleBtn.style.color = "#e0e0e0"; + inpaintToggleBtn.classList.remove("toggle-on"); } }; - toggleBtn.addEventListener("click", (e) => { + this.syncInpaintState = () => { + const customAudioWidget = this.node.widgets?.find(w => w.name === "use_custom_audio"); + if (customAudioWidget && !customAudioWidget.value) { + inpaintToggleBtn.disabled = true; + inpaintToggleBtn.style.opacity = "0.4"; + inpaintToggleBtn.style.cursor = "default"; + inpaintToggleBtn.title = "Audio Inpainting requires Custom Audio to be ON"; + } else { + inpaintToggleBtn.disabled = false; + inpaintToggleBtn.style.opacity = "1.0"; + inpaintToggleBtn.style.cursor = "pointer"; + inpaintToggleBtn.title = "Toggle Audio Inpainting in Gaps"; + } + }; + + + + inpaintToggleBtn.addEventListener("click", (e) => { e.stopPropagation(); - const widget = this.node.widgets?.find(w => w.name === "use_custom_audio"); + if (inpaintToggleBtn.disabled) return; + const widget = this.node.widgets?.find(w => w.name === "inpaint_audio"); if (widget) { widget.value = !widget.value; - updateToggleStyle(widget.value); + if (this.node.properties) { + this.node.properties.inpaint_audio = widget.value; + } + this.updateInpaintToggleStyle(widget.value); + this.commitChanges(true); this.node.setDirtyCanvas(true, true); } }); // Initial state check (widgets might not be ready immediately) setTimeout(() => { - const widget = this.node.widgets?.find(w => w.name === "use_custom_audio"); + const inpaintWidget = this.node.widgets?.find(w => w.name === "inpaint_audio"); + if (inpaintWidget) { + this.updateInpaintToggleStyle(inpaintWidget.value); + } + }, 100); + + const overrideAudioToggleBtn = document.createElement("button"); + overrideAudioToggleBtn.className = "pr-btn"; + overrideAudioToggleBtn.style.padding = "4px 0px"; + overrideAudioToggleBtn.style.fontSize = "9px"; + overrideAudioToggleBtn.style.lineHeight = "1"; + overrideAudioToggleBtn.style.marginRight = "0px"; + overrideAudioToggleBtn.style.marginTop = "8px"; // Adjust this value to fine-tune spacing between the title and button + overrideAudioToggleBtn.style.width = "72px"; + overrideAudioToggleBtn.style.whiteSpace = "nowrap"; + overrideAudioToggleBtn.style.textAlign = "center"; + overrideAudioToggleBtn.style.justifyContent = "center"; + overrideAudioToggleBtn.style.alignItems = "center"; + overrideAudioToggleBtn.style.gap = "0px"; + overrideAudioToggleBtn.style.boxSizing = "border-box"; + overrideAudioToggleBtn.style.borderRadius = "2px"; + overrideAudioToggleBtn.textContent = "Audio: OFF"; + overrideAudioToggleBtn.title = "Override Audio: Use audio from IC-LoRA Video"; + + this.updateOverrideAudioToggleStyle = (isOn) => { + overrideAudioToggleBtn.textContent = isOn ? "Audio: ON" : "Audio: OFF"; + if (isOn) { + overrideAudioToggleBtn.classList.add("toggle-on"); + } else { + overrideAudioToggleBtn.classList.remove("toggle-on"); + } + }; + + overrideAudioToggleBtn.addEventListener("click", (e) => { + e.stopPropagation(); + if (overrideAudioToggleBtn.disabled) return; + const widget = this.node.widgets?.find(w => w.name === "override_audio"); if (widget) { - updateToggleStyle(widget.value); + widget.value = !widget.value; + this.node.properties.overrideAudio = widget.value; + this.updateOverrideAudioToggleStyle(widget.value); + + if (widget.value) { + // When this is toggled on, the audio track will automatically be disabled/muted. + this._audioTrackWasEnabledBeforeOverride = this.audioTrackEnabled; + this.audioTrackEnabled = false; + updateTrackIcon(this.audioTrackLabel._eyeBtn, "audio", false); + + const customAudioWidget = this.node.widgets?.find(w => w.name === "use_custom_audio"); + if (customAudioWidget) { + customAudioWidget.value = false; + if (this.updateToggleStyle) this.updateToggleStyle(false); + } + + inpaintToggleBtn.disabled = true; + inpaintToggleBtn.style.opacity = "0.3"; + + if (this.timeline.motionSegments) { + for (const seg of this.timeline.motionSegments) { + if (seg.type === "motion_video") { + this._preloadMotionAudioSegment(seg); + } + } + } + } else { + // When toggled off, restore the audio track status if it was previously enabled + if (this._audioTrackWasEnabledBeforeOverride) { + this.audioTrackEnabled = true; + updateTrackIcon(this.audioTrackLabel._eyeBtn, "audio", true); + + const customAudioWidget = this.node.widgets?.find(w => w.name === "use_custom_audio"); + if (customAudioWidget) { + customAudioWidget.value = true; + if (this.updateToggleStyle) this.updateToggleStyle(true); + } + + inpaintToggleBtn.disabled = false; + inpaintToggleBtn.style.opacity = "1.0"; + } + this._audioTrackWasEnabledBeforeOverride = false; + } + + this.commitChanges(true); + this.render(); + } + }); + + // Initial state check (widgets might not be ready immediately) + setTimeout(() => { + const overrideWidget = this.node.widgets?.find(w => w.name === "override_audio"); + if (overrideWidget) { + this.updateOverrideAudioToggleStyle(overrideWidget.value); } }, 100); const helpBtn = document.createElement("button"); helpBtn.className = "pr-btn"; helpBtn.style.padding = "6px"; + helpBtn.style.display = "flex"; + helpBtn.style.alignItems = "center"; helpBtn.style.justifyContent = "center"; helpBtn.style.width = "28px"; helpBtn.style.height = "28px"; helpBtn.style.boxSizing = "border-box"; - helpBtn.innerHTML = "?"; + helpBtn.innerHTML = ICONS.help; helpBtn.title = "Help / Documentation"; helpBtn.addEventListener("click", (e) => { e.stopPropagation(); window.open("https://github.com/WhatDreamsCost/WhatDreamsCost-ComfyUI", "_blank"); }); + this.isSnapping = this.node.properties.isSnapping !== false; + + const snapBtn = document.createElement("button"); + snapBtn.className = "pr-btn"; + snapBtn.style.padding = "6px"; + snapBtn.style.display = "flex"; + snapBtn.style.alignItems = "center"; + snapBtn.style.justifyContent = "center"; + snapBtn.style.width = "28px"; + snapBtn.style.height = "28px"; + snapBtn.style.boxSizing = "border-box"; + snapBtn.innerHTML = ICONS.magnet; + + const updateSnapStyle = () => { + snapBtn.title = this.isSnapping ? "Disable Snapping (Magnet)" : "Enable Snapping (Magnet)"; + if (this.isSnapping) { + snapBtn.classList.add("toggle-on"); + } else { + snapBtn.classList.remove("toggle-on"); + } + }; + this.updateSnapStyle = updateSnapStyle; + updateSnapStyle(); + + snapBtn.addEventListener("click", (e) => { + e.stopPropagation(); + this.isSnapping = !this.isSnapping; + this.node.properties.isSnapping = this.isSnapping; + updateSnapStyle(); + this.commitChanges(); + this.render(); + }); + + const startBtn = document.createElement("button"); + startBtn.className = "pr-btn"; + startBtn.style.padding = "6px"; + startBtn.style.display = "flex"; + startBtn.style.alignItems = "center"; + startBtn.style.justifyContent = "center"; + startBtn.style.width = "28px"; + startBtn.style.height = "28px"; + startBtn.style.boxSizing = "border-box"; + startBtn.innerHTML = ICONS.start; + startBtn.title = "Set Start Frame"; + startBtn.addEventListener("click", (e) => { + e.stopPropagation(); + if (this.retakeMode) return; + if (this.startFramesWidget) { + this.startFramesWidget.value = this.currentFrame; + if (this.startFramesWidget.callback) { + this.startFramesWidget.callback(this.currentFrame); + } + this.commitChanges(); + this.render(); + } + }); + + const endBtn = document.createElement("button"); + endBtn.className = "pr-btn"; + endBtn.style.padding = "6px"; + endBtn.style.display = "flex"; + endBtn.style.alignItems = "center"; + endBtn.style.justifyContent = "center"; + endBtn.style.width = "28px"; + endBtn.style.height = "28px"; + endBtn.style.boxSizing = "border-box"; + endBtn.innerHTML = ICONS.end; + endBtn.title = "Set End Frame"; + endBtn.addEventListener("click", (e) => { + e.stopPropagation(); + if (this.retakeMode) return; + if (this.endFramesWidget) { + this.endFramesWidget.value = this.currentFrame; + if (this.endFramesWidget.callback) { + this.endFramesWidget.callback(this.currentFrame); + } + this.commitChanges(); + this.render(); + } + }); + + const markBtn = document.createElement("button"); + markBtn.className = "pr-btn"; + markBtn.style.padding = "6px"; + markBtn.style.display = "flex"; + markBtn.style.alignItems = "center"; + markBtn.style.justifyContent = "center"; + markBtn.style.width = "28px"; + markBtn.style.height = "28px"; + markBtn.style.boxSizing = "border-box"; + markBtn.innerHTML = ICONS.mark; + markBtn.title = "Mark Selection (X)"; + markBtn.addEventListener("click", (e) => { + e.stopPropagation(); + if (this.retakeMode) return; + this.markCurrentSelection(); + }); + + const retakeToggleBtn = document.createElement("button"); + retakeToggleBtn.className = "pr-btn"; + retakeToggleBtn.style.padding = "4px 8px"; + retakeToggleBtn.style.display = "flex"; + retakeToggleBtn.style.alignItems = "center"; + retakeToggleBtn.style.justifyContent = "center"; + retakeToggleBtn.style.gap = "6px"; + retakeToggleBtn.style.height = "28px"; + retakeToggleBtn.style.boxSizing = "border-box"; + retakeToggleBtn.innerHTML = ` Retake Mode (BETA)`; + + const updateRetakeStyle = () => { + retakeToggleBtn.title = this.retakeMode ? "Switch to Multi-Clip Timeline" : "Switch to Retake Tab"; + if (this.retakeMode) { + retakeToggleBtn.classList.add("toggle-on"); + } else { + retakeToggleBtn.classList.remove("toggle-on"); + } + }; + this.updateRetakeStyle = updateRetakeStyle; + updateRetakeStyle(); + + retakeToggleBtn.addEventListener("click", (e) => { + e.stopPropagation(); + + // Stop and mute any active playback first + this.pauseAudio(); + + // Save current input value to the mode we are EXITING + if (this.retakeMode) { + this.timeline.retake_global_prompt = this.globalPromptInput ? this.globalPromptInput.value : ""; + } else { + this.timeline.global_prompt = this.globalPromptInput ? this.globalPromptInput.value : ""; + // Backup normal mode values before entering Retake Mode + this.timeline.normalStartFrame = this.getStartFrames(); + this.timeline.normalDurationFrames = this.getDurationFrames(); + } + + this.retakeMode = !this.retakeMode; + this.timeline.retakeMode = this.retakeMode; + if (this.node.properties) { + this.node.properties.retakeMode = this.retakeMode; + } + + // Adjust widgets for the new mode + if (this.retakeMode) { + if (this.timeline.retakeVideo && this.timeline.retakeVideo.videoDurationFrames) { + this.syncWidgetsToRetakeDuration(this.timeline.retakeVideo.videoDurationFrames); + } + } else { + // Restore normal mode backup + this._suppressCommit = true; + if (this.timeline.normalStartFrame !== undefined && this.startFramesWidget) { + this.startFramesWidget.value = this.timeline.normalStartFrame; + if (this.startFramesWidget.callback) { + try { this.startFramesWidget.callback(this.timeline.normalStartFrame); } catch (_) {} + } + } + if (this.timeline.normalDurationFrames !== undefined && this.durationFramesWidget) { + this.durationFramesWidget.value = this.timeline.normalDurationFrames; + if (this.durationFramesWidget.callback) { + try { this.durationFramesWidget.callback(this.timeline.normalDurationFrames); } catch (_) {} + } + } + this._suppressCommit = false; + } + + this.updateRetakeUIState(); + this.commitChanges(); + this.render(); + }); + const btnGroup = document.createElement("div"); btnGroup.style.display = "flex"; btnGroup.style.gap = "6px"; btnGroup.style.alignItems = "center"; - btnGroup.appendChild(toggleBtn); + btnGroup.appendChild(retakeToggleBtn); + btnGroup.appendChild(snapBtn); + btnGroup.appendChild(startBtn); + btnGroup.appendChild(endBtn); + btnGroup.appendChild(markBtn); btnGroup.appendChild(helpBtn); btnGroup.appendChild(settingsBtn); rightGroup.appendChild(btnGroup); @@ -1038,6 +2706,9 @@ class TimelineEditor { this.canvas.style.width = newWidth + "px"; this.viewport.scrollLeft = scrollRatio * newWidth - mouseX; + + if (this.node) this.node.setDirtyCanvas?.(true, true); + else if (window.app && window.app.graph) window.app.graph.setDirtyCanvas(true, true); } }, { passive: false, capture: true }); @@ -1053,44 +2724,280 @@ class TimelineEditor { this.canvas.style.height = `${CANVAS_HEIGHT}px`; // --- Content Area Container --- + if (!this.node.properties) this.node.properties = {}; + if (this.node.properties.showFilenames === undefined) { + this.node.properties.showFilenames = (this.timeline.showFilenames !== undefined) ? this.timeline.showFilenames : true; + } + if (this.node.properties.overrideAudio === undefined) { + this.node.properties.overrideAudio = (this.timeline.overrideAudio !== undefined) ? this.timeline.overrideAudio : false; + } + if (this.node.properties.propHeight === undefined && this.timeline.propHeight !== undefined) { + this.node.properties.propHeight = this.timeline.propHeight; + } + this.initialPropHeight = this.node.properties.propHeight || 90; + this.propHeight = this.initialPropHeight; + const propContainer = document.createElement("div"); propContainer.className = "pr-prop-container"; + propContainer.style.position = "relative"; + propContainer.style.flex = "none"; + propContainer.style.height = `${this.propHeight}px`; + propContainer.style.marginBottom = "5px"; // Add some spacing between the two prompt boxes + this.propContainer = propContainer; + + if (this.node.properties.globalPropHeight === undefined && this.timeline.globalPropHeight !== undefined) { + this.node.properties.globalPropHeight = this.timeline.globalPropHeight; + } + if (!this.node.properties.globalPropHeight) this.node.properties.globalPropHeight = 60; + this.globalPropHeight = this.node.properties.globalPropHeight; + + const globalPropContainer = document.createElement("div"); + globalPropContainer.className = "pr-prop-container"; + globalPropContainer.style.position = "relative"; + globalPropContainer.style.flex = "none"; + globalPropContainer.style.height = `${this.globalPropHeight}px`; + this.globalPropContainer = globalPropContainer; + + const globalPromptWrapper = document.createElement("div"); + globalPromptWrapper.className = "pr-prompt-wrapper"; + globalPromptWrapper.style.width = "100%"; + globalPromptWrapper.style.height = "100%"; + + this.globalPromptLabel = document.createElement("div"); + this.globalPromptLabel.className = "pr-prompt-label"; + this.globalPromptLabel.textContent = "Global Prompt"; + globalPromptWrapper.appendChild(this.globalPromptLabel); + + this.globalPromptInput = document.createElement("textarea"); + this.globalPromptInput.className = "pr-prompt-area"; + this.globalPromptInput.placeholder = "Enter global prompt here..."; + this.globalPromptInput.spellcheck = false; + globalPromptWrapper.appendChild(this.globalPromptInput); + + this.globalPromptInput.addEventListener("focus", () => { + globalPromptWrapper.classList.add("focus-active"); + this.wrapper.classList.add("has-focus"); + }); + this.globalPromptInput.addEventListener("blur", () => { + globalPromptWrapper.classList.remove("focus-active"); + this.wrapper.classList.remove("has-focus"); + }); + let saveTimeout = null; + const triggerAutoSave = () => { + try { + const canvasEl = app.canvasEl || app.canvas?.canvas; + if (canvasEl) { + canvasEl.dispatchEvent(new PointerEvent('pointerup', { bubbles: true, cancelable: true })); + } + if (app.canvas && app.canvas.checkState) app.canvas.checkState(); + if (app.canvas && app.canvas.captureCanvasState) app.canvas.captureCanvasState(); + } catch (_) { } + }; + + this.globalPromptInput.addEventListener("input", (e) => { + const val = e.target.value; + this.syncGlobalPrompt(val); + + if (this.selectionType === "motion") { + this.promptInput.value = val; + } + this.commitChanges(true); + this.render(); + + // Debounce ComfyUI auto-save by 300ms to avoid lag while typing + if (saveTimeout) clearTimeout(saveTimeout); + saveTimeout = setTimeout(triggerAutoSave, 300); + }); + + this.globalPromptInput.addEventListener("blur", () => { + if (saveTimeout) clearTimeout(saveTimeout); + triggerAutoSave(); + }); + + const globalPropResizer = document.createElement("div"); + globalPropResizer.style.position = "absolute"; + globalPropResizer.style.bottom = "0px"; + globalPropResizer.style.left = "0px"; + globalPropResizer.style.width = "100%"; + globalPropResizer.style.height = "12px"; // Hit area + globalPropResizer.style.cursor = "ns-resize"; + globalPropResizer.style.display = "flex"; + globalPropResizer.style.justifyContent = "center"; + globalPropResizer.style.alignItems = "flex-end"; + globalPropResizer.style.paddingBottom = "4px"; + globalPropResizer.style.zIndex = "10"; + globalPropResizer.innerHTML = `
`; + + let isGlobalResizing = false; + let startGlobalY = 0; + let startGlobalH = 0; + + globalPropResizer.addEventListener("mousedown", (ev) => { + isGlobalResizing = true; + startGlobalY = ev.clientY; + startGlobalH = this.globalPropHeight; + ev.stopPropagation(); + ev.preventDefault(); + }); + + document.addEventListener("mousemove", (ev) => { + if (isGlobalResizing) { + const newH = Math.max(60, startGlobalH + (ev.clientY - startGlobalY)); + this.globalPropHeight = newH; + this.node.properties.globalPropHeight = newH; + globalPropContainer.style.height = `${newH}px`; + + if (this.node && this.node.computeSize) { + const sz = this.node.computeSize(); + this.node.size[1] = sz[1]; + if (window.app && window.app.graph) { + window.app.graph.setDirtyCanvas(true, true); + } + } + } + }); + + document.addEventListener("mouseup", () => { + if (isGlobalResizing) { + isGlobalResizing = false; + } + }); + + globalPropContainer.appendChild(globalPromptWrapper); + globalPropContainer.appendChild(globalPropResizer); + + const propResizer = document.createElement("div"); + propResizer.style.position = "absolute"; + propResizer.style.bottom = "0px"; + propResizer.style.left = "0px"; + propResizer.style.width = "100%"; + propResizer.style.height = "12px"; // Hit area + propResizer.style.cursor = "ns-resize"; + propResizer.style.display = "flex"; + propResizer.style.justifyContent = "center"; + propResizer.style.alignItems = "flex-end"; + propResizer.style.paddingBottom = "4px"; + propResizer.innerHTML = `
`; + + propResizer.addEventListener("mousedown", (e) => { + e.preventDefault(); + const startY = e.clientY; + const startH = this.propHeight; + + const doDrag = (ev) => { + if (ev.buttons === 0) { + stopDrag(); + return; + } + const newH = Math.max(90, startH + (ev.clientY - startY)); + this.propHeight = newH; + this.node.properties.propHeight = newH; + propContainer.style.height = `${newH}px`; + + if (this.node && this.node.computeSize) { + const sz = this.node.computeSize(); + this.node.size[1] = sz[1]; + if (window.app && window.app.graph) { + window.app.graph.setDirtyCanvas(true, true); + } + } + }; + + const stopDrag = () => { + window.removeEventListener("mousemove", doDrag, true); + window.removeEventListener("mouseup", stopDrag, true); + document.body.style.cursor = "default"; + }; + + document.body.style.cursor = "ns-resize"; + window.addEventListener("mousemove", doDrag, true); + window.addEventListener("mouseup", stopDrag, true); + }); // --- Text Area (Image/Text) --- + this.promptWrapper = document.createElement("div"); + this.promptWrapper.className = "pr-prompt-wrapper"; + this.promptWrapper.style.width = "100%"; + this.promptWrapper.style.height = "100%"; + this.promptWrapper.style.display = "none"; + + this.segmentPromptLabel = document.createElement("div"); + this.segmentPromptLabel.className = "pr-prompt-label"; + this.segmentPromptLabel.textContent = "Segment Prompt"; + this.promptWrapper.appendChild(this.segmentPromptLabel); + this.promptInput = document.createElement("textarea"); this.promptInput.className = "pr-prompt-area"; - this.promptInput.placeholder = "Enter prompt for selected segment..."; + this.promptInput.placeholder = "No segment selected!"; + this.promptInput.style.opacity = "0.4"; + this.promptWrapper.appendChild(this.promptInput); + + this.promptInput.addEventListener("focus", () => { + this.promptWrapper.classList.add("focus-active"); + this.wrapper.classList.add("has-focus"); + }); + this.promptInput.addEventListener("blur", () => { + this.promptWrapper.classList.remove("focus-active"); + this.wrapper.classList.remove("has-focus"); + }); + this.promptInput.addEventListener("input", () => { + if (this.retakeMode) { + this.timeline.retakePrompt = this.promptInput.value; + this.commitChanges(); + return; + } if (this.selectionType === "image" && this.timeline.segments[this.selectedIndex]) { this.timeline.segments[this.selectedIndex].prompt = this.promptInput.value; this.commitChanges(); + } else if (this.selectionType === "motion") { + const val = this.promptInput.value; + if (this.globalPromptInput) { + this.globalPromptInput.value = val; + } + this.syncGlobalPrompt(val); + this.commitChanges(true); + this.render(); + + // Debounce ComfyUI auto-save by 300ms to avoid lag while typing + if (saveTimeout) clearTimeout(saveTimeout); + saveTimeout = setTimeout(triggerAutoSave, 300); } }); + // --- Motion Info Area --- + this.motionInfoArea = document.createElement("div"); + this.motionInfoArea.className = "pr-motion-info"; + // --- Audio Info Area --- this.audioInfoArea = document.createElement("div"); this.audioInfoArea.className = "pr-audio-info"; - propContainer.appendChild(this.promptInput); + propContainer.appendChild(this.promptWrapper); + propContainer.appendChild(this.motionInfoArea); propContainer.appendChild(this.audioInfoArea); + propContainer.appendChild(propResizer); this.wrapper.addEventListener("dragover", (e) => { e.preventDefault(); this.wrapper.classList.add("drag-active"); + if (this.retakeMode) { + return; // Skip ghost segments rendering when in retakeMode + } + const { x, y } = this.getMousePos(e); const logicalWidth = this.canvas.offsetWidth; const totalFrames = this.getVisualDurationFrames(); if (!logicalWidth || totalFrames <= 0) return; - const isAudioTrack = y > RULER_HEIGHT + this.blockHeight; - const trackType = isAudioTrack ? "audio" : "image"; - const arrToModify = isAudioTrack ? this.timeline.audioSegments : this.timeline.segments; + const trackType = this.getTrackFromY(y); + const arrToModify = this.getSegmentArray(trackType); if (!this._ghostSegmentId || this._ghostTrack !== trackType) { this._ghostSegmentId = "GHOST_" + Date.now(); this._ghostTrack = trackType; - this._ghostInitialTimeline = JSON.parse(JSON.stringify(arrToModify)); + this._ghostInitialTimeline = arrToModify.map(s => ({ ...s })); const frameRate = this.getFrameRate(); const newLength = Math.max(1, frameRate * 1); @@ -1119,6 +3026,16 @@ class TimelineEditor { totalFrames, logicalWidth ); + + for (let ps of this._previewSegments) { + const orig = arrToModify.find(s => s.id === ps.id); + if (orig) { + ps.videoEl = orig.videoEl; + ps.imgObj = orig.imgObj; + if (orig.thumbnails) ps.thumbnails = orig.thumbnails; + } + } + this.render(); }); @@ -1158,14 +3075,22 @@ class TimelineEditor { if (e.dataTransfer.files && e.dataTransfer.files.length > 0) { const imageFiles = []; const audioFiles = []; + const videoFiles = []; for (let file of e.dataTransfer.files) { - if (file.type.startsWith("audio/")) audioFiles.push(file); - if (file.type.startsWith("image/")) imageFiles.push(file); + if (file.type.startsWith("video/")) videoFiles.push(file); + else if (file.type.startsWith("audio/")) audioFiles.push(file); + else if (file.type.startsWith("image/")) imageFiles.push(file); } // Let implicit intent handle mixing drops: use the track we hovered over // for the first type we process, or fallback. - if (audioFiles.length > 0 && (targetTrack === "audio" || imageFiles.length === 0)) { + if (videoFiles.length > 0) { + if (targetTrack === "motion") { + this.handleMotionUpload(videoFiles, targetFrameStart); + } else { + this.handleVideoUpload(videoFiles, targetFrameStart); + } + } else if (audioFiles.length > 0 && (targetTrack === "audio" || imageFiles.length === 0)) { this.handleAudioUpload(audioFiles, targetFrameStart); } else if (imageFiles.length > 0) { this.handleImageUpload(imageFiles, targetFrameStart); @@ -1201,7 +3126,16 @@ class TimelineEditor { this.seekBar.value = "0"; this.seekBar.style.flex = "1"; // take up remaining space this.seekBar.addEventListener("input", (e) => { - this.currentFrame = parseInt(e.target.value, 10); + let val = parseInt(e.target.value, 10); + if (this.retakeMode && this.timeline.retakeVideo) { + const baseVideoDur = this.timeline.retakeVideo.videoDurationFrames || 0; + if (val > baseVideoDur) { + val = baseVideoDur; + this.seekBar.value = val; + } + } + this.currentFrame = val; + this.updateSeekBarBackground(); this.render(); if (this.isPlaying) { this.playAudio(); @@ -1247,6 +3181,9 @@ class TimelineEditor { const playheadRatio = this.currentFrame / totalFrames; const newPlayheadX = playheadRatio * newCanvasWidth; this.viewport.scrollLeft = newPlayheadX - (viewportWidth / 2); + + if (this.node) this.node.setDirtyCanvas?.(true, true); + else if (window.app && window.app.graph) window.app.graph.setDirtyCanvas(true, true); }); const zoomInBtn = document.createElement("button"); @@ -1275,6 +3212,9 @@ class TimelineEditor { this._lastWidth = viewportWidth; this._lastZoom = 1; this.viewport.scrollLeft = 0; + + if (this.node) this.node.setDirtyCanvas?.(true, true); + else if (window.app && window.app.graph) window.app.graph.setDirtyCanvas(true, true); }); zoomControls.appendChild(zoomOutBtn); @@ -1293,9 +3233,9 @@ class TimelineEditor { this.strengthRow = document.createElement("div"); this.strengthRow.className = "pr-strength-row"; - const strengthLabel = document.createElement("span"); - strengthLabel.className = "pr-strength-label"; - strengthLabel.textContent = "Guide Strength:"; + this.strengthLabel = document.createElement("span"); + this.strengthLabel.className = "pr-strength-label"; + this.strengthLabel.textContent = "Guide Strength:"; this.strengthValue = document.createElement("input"); this.strengthValue.type = "text"; @@ -1304,6 +3244,147 @@ class TimelineEditor { this.strengthValue.disabled = true; this.strengthValue.style.cursor = "ew-resize"; + this.vidStrLabel = document.createElement("span"); + this.vidStrLabel.className = "pr-strength-label"; + this.vidStrLabel.textContent = "Video Strength:"; + this.vidStrLabel.style.display = "none"; + + this.vidStrValue = document.createElement("input"); + this.vidStrValue.type = "text"; + this.vidStrValue.className = "pr-strength-input"; + this.vidStrValue.value = "1.00"; + this.vidStrValue.style.display = "none"; + this.vidStrValue.style.width = "40px"; + this.vidStrValue.style.cursor = "ew-resize"; + + this.vidAttnLabel = document.createElement("span"); + this.vidAttnLabel.className = "pr-strength-label"; + this.vidAttnLabel.textContent = "Video Attn:"; + this.vidAttnLabel.style.display = "none"; + this.vidAttnLabel.style.marginLeft = "10px"; + + this.vidAttnValue = document.createElement("input"); + this.vidAttnValue.type = "text"; + this.vidAttnValue.className = "pr-strength-input"; + this.vidAttnValue.value = "0.65"; + this.vidAttnValue.style.display = "none"; + this.vidAttnValue.style.width = "40px"; + this.vidAttnValue.style.cursor = "ew-resize"; + + this.vidStrValue.addEventListener("change", (e) => { + let val = parseFloat(e.target.value); + if (isNaN(val)) val = 1.0; + val = Math.max(0, Math.min(1, val)); + this.vidStrValue.value = val.toFixed(2); + if (this.selectionType === "motion" && this.timeline.motionSegments[this.selectedIndex]) { + this.timeline.motionSegments[this.selectedIndex].videoStrength = val; + this.commitChanges(); + } + }); + + this.vidAttnValue.addEventListener("change", (e) => { + let val = parseFloat(e.target.value); + if (isNaN(val)) val = 0.65; + val = Math.max(0, Math.min(1, val)); + this.vidAttnValue.value = val.toFixed(2); + if (this.selectionType === "motion" && this.timeline.motionSegments[this.selectedIndex]) { + this.timeline.motionSegments[this.selectedIndex].videoAttentionStrength = val; + this.commitChanges(); + } + }); + + // Dragging logic for video strength + this.vidStrValue.addEventListener("mousedown", (e) => { + if (this.vidStrValue.disabled) return; + const vStrStartX = e.clientX; + const vStrStartVal = parseFloat(this.vidStrValue.value) || 1.0; + let vStrHasMoved = false; + let vStrIsDragging = false; + + const onMouseMove = (moveEvent) => { + const deltaX = moveEvent.clientX - vStrStartX; + if (Math.abs(deltaX) > 3) { + vStrHasMoved = true; + vStrIsDragging = true; + } + + if (vStrIsDragging) { + moveEvent.preventDefault(); + const sensitivity = 0.002; + let newVal = vStrStartVal + deltaX * sensitivity; + + if (newVal < 0) newVal = 0; + if (newVal > 1) newVal = 1; + + this.vidStrValue.value = newVal.toFixed(2); + + if (this.selectionType === "motion" && this.timeline.motionSegments[this.selectedIndex]) { + this.timeline.motionSegments[this.selectedIndex].videoStrength = newVal; + this.commitChanges(); + } + } + }; + + const onMouseUp = () => { + document.removeEventListener("mousemove", onMouseMove); + document.removeEventListener("mouseup", onMouseUp); + + if (!vStrHasMoved) { + this.vidStrValue.focus(); + this.vidStrValue.select(); + } + }; + + document.addEventListener("mousemove", onMouseMove); + document.addEventListener("mouseup", onMouseUp); + }); + + // Dragging logic for video attention strength + this.vidAttnValue.addEventListener("mousedown", (e) => { + if (this.vidAttnValue.disabled) return; + const vAttnStartX = e.clientX; + const vAttnStartVal = parseFloat(this.vidAttnValue.value) || 0.65; + let vAttnHasMoved = false; + let vAttnIsDragging = false; + + const onMouseMove = (moveEvent) => { + const deltaX = moveEvent.clientX - vAttnStartX; + if (Math.abs(deltaX) > 3) { + vAttnHasMoved = true; + vAttnIsDragging = true; + } + + if (vAttnIsDragging) { + moveEvent.preventDefault(); + const sensitivity = 0.002; + let newVal = vAttnStartVal + deltaX * sensitivity; + + if (newVal < 0) newVal = 0; + if (newVal > 1) newVal = 1; + + this.vidAttnValue.value = newVal.toFixed(2); + + if (this.selectionType === "motion" && this.timeline.motionSegments[this.selectedIndex]) { + this.timeline.motionSegments[this.selectedIndex].videoAttentionStrength = newVal; + this.commitChanges(); + } + } + }; + + const onMouseUp = () => { + document.removeEventListener("mousemove", onMouseMove); + document.removeEventListener("mouseup", onMouseUp); + + if (!vAttnHasMoved) { + this.vidAttnValue.focus(); + this.vidAttnValue.select(); + } + }; + + document.addEventListener("mousemove", onMouseMove); + document.addEventListener("mouseup", onMouseUp); + }); + // Dragging logic for guide strength let isDragging = false; let startX = 0; @@ -1333,7 +3414,10 @@ class TimelineEditor { this.strengthValue.value = newVal.toFixed(2); - if (this.selectionType === "image" && this.timeline.segments[this.selectedIndex]) { + if (this.retakeMode) { + this.timeline.retakeStrength = newVal; + this.commitChanges(); + } else if (this.selectionType === "image" && this.timeline.segments[this.selectedIndex]) { const seg = this.timeline.segments[this.selectedIndex]; if (seg.type !== "text") { seg.guideStrength = newVal; @@ -1363,7 +3447,10 @@ class TimelineEditor { if (isNaN(val)) val = 1; val = Math.max(0, Math.min(1, val)); this.strengthValue.value = val.toFixed(2); - if (this.selectionType === "image" && this.timeline.segments[this.selectedIndex]) { + if (this.retakeMode) { + this.timeline.retakeStrength = val; + this.commitChanges(); + } else if (this.selectionType === "image" && this.timeline.segments[this.selectedIndex]) { const seg = this.timeline.segments[this.selectedIndex]; if (seg.type !== "text") { seg.guideStrength = val; @@ -1374,12 +3461,288 @@ class TimelineEditor { this.strengthRow.appendChild(this.timeCodeDisplay); this.strengthRow.appendChild(this.segmentBoundsDisplay); - this.strengthRow.appendChild(strengthLabel); + this.strengthRow.appendChild(this.strengthLabel); this.strengthRow.appendChild(this.strengthValue); + this.strengthRow.appendChild(this.vidStrLabel); + this.strengthRow.appendChild(this.vidStrValue); + this.strengthRow.appendChild(this.vidAttnLabel); + this.strengthRow.appendChild(this.vidAttnValue); + + // Layout container for sidebar + viewport + this.layoutContainer = document.createElement("div"); + this.layoutContainer.className = "pr-timeline-layout"; + this.layoutContainer.style.display = "flex"; + this.layoutContainer.style.flexDirection = "row"; + this.layoutContainer.style.width = "100%"; + this.layoutContainer.style.border = "1px solid #111"; + this.layoutContainer.style.borderRadius = "6px"; + this.layoutContainer.style.overflow = "hidden"; + + // Sidebar + this.sidebar = document.createElement("div"); + this.sidebar.className = "pr-timeline-sidebar"; + this.sidebar.style.width = "120px"; + this.sidebar.style.flexShrink = "0"; + this.sidebar.style.display = "flex"; + this.sidebar.style.flexDirection = "column"; + this.sidebar.style.borderRight = "1px solid #111"; + this.sidebar.style.boxSizing = "border-box"; + this.sidebar.style.backgroundColor = "#1e1e1e"; + this.sidebar.style.userSelect = "none"; + + // Spacer for Ruler + this.rulerSpacer = document.createElement("div"); + this.rulerSpacer.style.height = `${RULER_HEIGHT}px`; + this.rulerSpacer.style.width = "100%"; + this.rulerSpacer.style.borderBottom = "1px solid #111"; + this.rulerSpacer.style.backgroundColor = "#1e1e1e"; + this.rulerSpacer.style.boxSizing = "border-box"; + this.rulerSpacer.style.flexShrink = "0"; + this.sidebar.appendChild(this.rulerSpacer); + + const getTrackIconHtml = (trackId, isEnabled) => { + if (trackId === "audio") { + if (isEnabled) { + return ` + + + + `; + } else { + return ` + + + `; + } + } else { + return ` + + + ${!isEnabled ? '' : ''} + `; + } + }; + + const updateTrackIcon = (btn, trackId, isEnabled) => { + btn.style.color = isEnabled ? "#aaa" : "#444"; + btn.innerHTML = getTrackIconHtml(trackId, isEnabled); + }; + this.updateTrackIcon = updateTrackIcon; + + const createTrackLabel = (text, bgColor, trackId, isEnabled, toggleCallback) => { + const el = document.createElement("div"); + el.style.display = "flex"; + el.style.flexDirection = "column"; + el.style.alignItems = "center"; + el.style.justifyContent = "center"; + el.style.borderBottom = "1px solid #111"; + el.style.backgroundColor = bgColor; + el.style.boxSizing = "border-box"; + el.style.gap = "4px"; + el.style.overflow = "hidden"; + el.style.position = "relative"; + el.style.flexShrink = "0"; + + const headerRow = document.createElement("div"); + headerRow.style.display = "flex"; + headerRow.style.alignItems = "center"; + headerRow.style.justifyContent = "center"; + headerRow.style.gap = "6px"; + + const textSpan = document.createElement("span"); + textSpan.style.color = "#ccc"; + textSpan.style.fontSize = "12px"; + textSpan.style.fontWeight = "bold"; + textSpan.style.lineHeight = "1"; + textSpan.style.display = "inline-flex"; + textSpan.style.alignItems = "center"; + textSpan.textContent = text; + + const eyeBtn = document.createElement("div"); + eyeBtn.style.cursor = "pointer"; + eyeBtn.style.display = "inline-flex"; + eyeBtn.style.alignItems = "center"; + eyeBtn.style.justifyContent = "center"; + eyeBtn.style.width = "14px"; + eyeBtn.style.height = "14px"; + eyeBtn.style.color = isEnabled ? "#aaa" : "#444"; + eyeBtn.innerHTML = getTrackIconHtml(trackId, isEnabled); + + eyeBtn.addEventListener("click", (e) => { + e.stopPropagation(); + toggleCallback(); + }); + + // Store reference so we can update it later + el._eyeBtn = eyeBtn; + + headerRow.appendChild(textSpan); + headerRow.appendChild(eyeBtn); + el.appendChild(headerRow); + + return el; + }; + + this.mainTrackLabel = createTrackLabel("MAIN", "#1e1e1e", "main", this.mainTrackEnabled, () => { + this.mainTrackEnabled = !this.mainTrackEnabled; + updateTrackIcon(this.mainTrackLabel._eyeBtn, "main", this.mainTrackEnabled); + this.commitChanges(true); + this.render(); + }); + + this.audioTrackLabel = createTrackLabel("AUDIO", "#1e1e1e", "audio", this.audioTrackEnabled, () => { + this.audioTrackEnabled = !this.audioTrackEnabled; + updateTrackIcon(this.audioTrackLabel._eyeBtn, "audio", this.audioTrackEnabled); + + if (this.audioTrackEnabled) { + const overrideWidget = this.node.widgets?.find(w => w.name === "override_audio"); + if (overrideWidget && overrideWidget.value) { + overrideWidget.value = false; + this.node.properties.overrideAudio = false; + if (this.updateOverrideAudioToggleStyle) this.updateOverrideAudioToggleStyle(false); + } + this._audioTrackWasEnabledBeforeOverride = false; + } + + // Auto-disable custom audio if track disabled + const customAudioWidget = this.node.widgets?.find(w => w.name === "use_custom_audio"); + if (customAudioWidget) { + if (!this.audioTrackEnabled) { + // Store previous state just in case, though the user requested it auto-enables + this._prevCustomAudioState = customAudioWidget.value; + customAudioWidget.value = false; + } else { + // Auto-turn it back on as requested + customAudioWidget.value = true; + } + if (this.updateToggleStyle) this.updateToggleStyle(customAudioWidget.value); + } + + // Disable toggle buttons visually + inpaintToggleBtn.disabled = !this.audioTrackEnabled; + inpaintToggleBtn.style.opacity = this.audioTrackEnabled ? "1.0" : "0.3"; + + this.commitChanges(true); + this.render(); + }); + this.audioTrackLabel.appendChild(inpaintToggleBtn); + + // Initialize audio toggle states immediately + inpaintToggleBtn.disabled = !this.audioTrackEnabled; + inpaintToggleBtn.style.opacity = this.audioTrackEnabled ? "1.0" : "0.3"; + + this.motionTrackLabel = createTrackLabel("IC-LoRA Video", "#1e1e1e", "motion", this.motionTrackEnabled, () => { + this.motionTrackEnabled = !this.motionTrackEnabled; + updateTrackIcon(this.motionTrackLabel._eyeBtn, "motion", this.motionTrackEnabled); + + // Auto-disable custom motion if track disabled + const customMotionWidget = this.node.widgets?.find(w => w.name === "use_custom_motion"); + if (customMotionWidget) { + if (!this.motionTrackEnabled) { + customMotionWidget.value = false; + } else { + customMotionWidget.value = true; + } + } + + overrideAudioToggleBtn.disabled = !this.motionTrackEnabled; + overrideAudioToggleBtn.style.opacity = this.motionTrackEnabled ? "1.0" : "0.3"; + if (!this.motionTrackEnabled) { + const overrideWidget = this.node.widgets?.find(w => w.name === "override_audio"); + if (overrideWidget && overrideWidget.value) { + overrideWidget.value = false; + this.node.properties.overrideAudio = false; + if (this.updateOverrideAudioToggleStyle) this.updateOverrideAudioToggleStyle(false); + + // Restore audio track if it was previously enabled + if (this._audioTrackWasEnabledBeforeOverride) { + this.audioTrackEnabled = true; + updateTrackIcon(this.audioTrackLabel._eyeBtn, "audio", true); + + const customAudioWidget = this.node.widgets?.find(w => w.name === "use_custom_audio"); + if (customAudioWidget) { + customAudioWidget.value = true; + if (this.updateToggleStyle) this.updateToggleStyle(true); + } + + inpaintToggleBtn.disabled = false; + inpaintToggleBtn.style.opacity = "1.0"; + } + this._audioTrackWasEnabledBeforeOverride = false; + } + } + + this.commitChanges(true); + this.render(); + }); + this.motionTrackLabel.appendChild(overrideAudioToggleBtn); + + // Initialize motion override states immediately + overrideAudioToggleBtn.disabled = !this.motionTrackEnabled; + overrideAudioToggleBtn.style.opacity = this.motionTrackEnabled ? "1.0" : "0.3"; + + + this.sidebar.appendChild(this.mainTrackLabel); + this.sidebar.appendChild(this.audioTrackLabel); + this.sidebar.appendChild(this.motionTrackLabel); + + const setupSidebarLabelResizing = (labelEl, dragType) => { + labelEl.addEventListener("mousemove", (e) => { + if (this.retakeMode) { + labelEl.style.cursor = "default"; + return; + } + if (this._isDragging) return; + const rect = labelEl.getBoundingClientRect(); + const y = e.clientY - rect.top; + if (rect.height - y <= 8) { + labelEl.style.cursor = "ns-resize"; + } else { + labelEl.style.cursor = "default"; + } + }); + + labelEl.addEventListener("mousedown", (e) => { + if (this.retakeMode) return; + if (e.button !== 0) return; + if (e.target.closest("svg") || e.target.style.cursor === "pointer" || window.getComputedStyle(e.target).cursor === "pointer") { + return; + } + const rect = labelEl.getBoundingClientRect(); + const y = e.clientY - rect.top; + if (rect.height - y <= 8) { + this._isDragging = true; + this._dragType = dragType; + this._startBlockHeight = this.blockHeight; + this._startAudioTrackHeight = this.audioTrackHeight; + this._startMotionTrackHeight = this.motionTrackHeight; + this._startY = this.getMousePos(e).y; + document.body.style.userSelect = "none"; + document.body.style.cursor = "ns-resize"; + e.preventDefault(); + e.stopPropagation(); + } + }); + }; + + setupSidebarLabelResizing(this.mainTrackLabel, "divider"); + setupSidebarLabelResizing(this.audioTrackLabel, "audio_divider"); + setupSidebarLabelResizing(this.motionTrackLabel, "height_resize"); + + this.updateSidebarHeights(); + + this.layoutContainer.appendChild(this.sidebar); + + // Viewport takes remaining space + this.viewport.style.flexGrow = "1"; + this.viewport.style.minWidth = "0"; + this.layoutContainer.appendChild(this.viewport); + this.wrapper.appendChild(toolbar); - this.wrapper.appendChild(this.viewport); + this.wrapper.appendChild(this.layoutContainer); + const controlsGroup = document.createElement("div"); controlsGroup.className = "pr-controls-group"; @@ -1387,11 +3750,78 @@ class TimelineEditor { controlsGroup.appendChild(playerControls); this.wrapper.appendChild(controlsGroup); this.wrapper.appendChild(propContainer); + this.wrapper.appendChild(this.globalPropContainer); this.container.appendChild(this.wrapper); } + syncWidgetsAndUI() { + console.log("[LTXDirector debug] syncWidgetsAndUI() called."); + console.log(` - mainTrackEnabled: ${this.mainTrackEnabled}`); + console.log(` - audioTrackEnabled: ${this.audioTrackEnabled}`); + console.log(` - motionTrackEnabled: ${this.motionTrackEnabled}`); + + // 1. Sync the widgets with the loaded track enablement states + const customAudioWidget = this.node.widgets?.find(w => w.name === "use_custom_audio"); + if (customAudioWidget) { + customAudioWidget.value = this.audioTrackEnabled; + console.log(` - Set use_custom_audio widget value to ${this.audioTrackEnabled}`); + } + const customMotionWidget = this.node.widgets?.find(w => w.name === "use_custom_motion"); + if (customMotionWidget) { + customMotionWidget.value = this.motionTrackEnabled; + console.log(` - Set use_custom_motion widget value to ${this.motionTrackEnabled}`); + } + + // 2. Sync the track icon buttons + if (this.mainTrackLabel?._eyeBtn && this.updateTrackIcon) { + this.updateTrackIcon(this.mainTrackLabel._eyeBtn, "main", this.mainTrackEnabled); + console.log(" - Updated main track eye icon"); + } + if (this.audioTrackLabel?._eyeBtn && this.updateTrackIcon) { + this.updateTrackIcon(this.audioTrackLabel._eyeBtn, "audio", this.audioTrackEnabled); + console.log(" - Updated audio track eye icon"); + } + if (this.motionTrackLabel?._eyeBtn && this.updateTrackIcon) { + this.updateTrackIcon(this.motionTrackLabel._eyeBtn, "motion", this.motionTrackEnabled); + console.log(" - Updated motion track eye icon"); + } + + // 3. Sync the inpaint button disabled/opacity state + const inpaintToggleBtn = this.audioTrackLabel?.querySelector(".pr-btn"); + if (inpaintToggleBtn) { + inpaintToggleBtn.disabled = !this.audioTrackEnabled; + inpaintToggleBtn.style.opacity = this.audioTrackEnabled ? "1.0" : "0.3"; + console.log(` - Updated inpaint toggle button disabled: ${inpaintToggleBtn.disabled}`); + } + + if (this.updateInpaintToggleStyle) { + const inpaintWidget = this.node.widgets?.find(w => w.name === "inpaint_audio"); + if (inpaintWidget) { + console.log(` - calling updateInpaintToggleStyle with ${inpaintWidget.value}`); + this.updateInpaintToggleStyle(inpaintWidget.value); + } + } + + // 4. Sync the override audio button disabled/opacity state + const overrideAudioToggleBtn = this.motionTrackLabel?.querySelector(".pr-btn"); + if (overrideAudioToggleBtn) { + overrideAudioToggleBtn.disabled = !this.motionTrackEnabled; + overrideAudioToggleBtn.style.opacity = this.motionTrackEnabled ? "1.0" : "0.3"; + console.log(` - Updated override audio toggle button disabled: ${overrideAudioToggleBtn.disabled}`); + } + + if (this.updateOverrideAudioToggleStyle) { + const overrideWidget = this.node.widgets?.find(w => w.name === "override_audio"); + if (overrideWidget) { + console.log(` - calling updateOverrideAudioToggleStyle with ${overrideWidget.value}`); + this.updateOverrideAudioToggleStyle(overrideWidget.value); + } + } + } + checkResize() { + this.syncLayoutToNode(false); const viewportWidth = this.viewport.clientWidth; const currentScale = this.getRenderScale(); @@ -1403,10 +3833,51 @@ class TimelineEditor { const newCanvasWidth = Math.max(viewportWidth, viewportWidth * this.zoomLevel); this.canvas.style.width = newCanvasWidth + "px"; this.resizeCanvas(newCanvasWidth); + + if (this.node) this.node.setDirtyCanvas?.(true, true); + else if (window.app && window.app.graph) window.app.graph.setDirtyCanvas(true, true); } this._renderLoop = requestAnimationFrame(() => this.checkResize()); } + syncLayoutToNode(forceRender = true) { + const nodeWidth = this.node?.size?.[0] || 1375; + const targetWidth = Math.max(10, nodeWidth - 30); + + if (this.container) { + this.container.style.width = `${targetWidth}px`; + this.container.style.maxWidth = `${targetWidth}px`; + this.container.style.setProperty("height", "auto", "important"); + this.container.style.boxSizing = "border-box"; + } + if (this.wrapper) { + this.wrapper.style.width = "100%"; + this.wrapper.style.maxWidth = "100%"; + this.wrapper.style.setProperty("height", "auto", "important"); + this.wrapper.style.boxSizing = "border-box"; + } + if (this.viewport) { + this.viewport.style.boxSizing = "content-box"; + this.viewport.style.height = `${this.canvasHeight}px`; + this.viewport.style.minHeight = `${this.canvasHeight}px`; + this.viewport.style.flexShrink = "0"; + } + if (this.layoutContainer) { + this.layoutContainer.style.flexShrink = "0"; + } + + const viewportWidth = this.viewport?.clientWidth || targetWidth; + const canvasWidth = Math.max(viewportWidth, viewportWidth * this.zoomLevel); + const currentWidth = parseFloat(this.canvas?.style?.width) || 0; + if (viewportWidth > 0 && Math.abs(currentWidth - canvasWidth) > 1) { + this.canvas.style.width = `${canvasWidth}px`; + this.resizeCanvas(canvasWidth); + this._lastWidth = viewportWidth; + this._lastZoom = this.zoomLevel; + if (forceRender) this.render(); + } + } + getRenderScale() { const dpr = window.devicePixelRatio || 1; let graphScale = 1; @@ -1455,6 +3926,7 @@ class TimelineEditor { try { const body = new FormData(); body.append("image", file); + body.append("subfolder", "whatdreamscost"); const resp = await api.fetchApi("/upload/image", { method: "POST", body }); if (resp.status !== 200) { resolve(); return; } @@ -1487,7 +3959,12 @@ class TimelineEditor { // Resolve physics to push existing segments let tempId = "TEMP_" + Date.now(); this.timeline.segments.push({ id: tempId, start: newStart, length: newLength, type: "temp" }); - let result = this._applyCenterDragPhysics(this.timeline.segments, tempId, newStart, newStart + newLength / 2, currentDuration, currentDuration, 1); + let physicsCenter = newStart + this.getFrameRate() / 2; + let result = this._applyCenterDragPhysics(this.timeline.segments, tempId, newStart, physicsCenter, currentDuration, currentDuration, 1); + + let siblingPhysics = (this.timeline.audioSegments || []).map(s => ({ ...s })); + + this._resolveGlobalPhysics(result, siblingPhysics, currentDuration, this.timeline.segments, this.timeline.audioSegments); // Update original segments with resolved physics to preserve imgObj for (let shiftedSeg of result) { @@ -1497,6 +3974,13 @@ class TimelineEditor { } } + for (let shiftedSib of siblingPhysics) { + let originalSib = this.timeline.audioSegments.find(s => s.id === shiftedSib.id); + if (originalSib) { + originalSib.start = shiftedSib.start; + } + } + let tempSeg = this.timeline.segments.find(s => s.id === tempId); newStart = tempSeg.start; this.timeline.segments = this.timeline.segments.filter(s => s.id !== tempId); @@ -1529,6 +4013,10 @@ class TimelineEditor { this.selectionType = "image"; this.selectedIndex = this.timeline.segments.findIndex(s => s.id === seg.id); + if (!this.retakeMode) { + this.growTimelineIfNeeded(seg.start + seg.length); + } + this.updateUIFromSelection(); this.commitChanges(true); }; @@ -1542,6 +4030,565 @@ class TimelineEditor { this.fileInput.value = ""; } + // Shared chunked upload helper for all video types in the LTX Director. + // Files <= 50 MB go through ComfyUI's standard /upload/image endpoint; + // larger files are split into 50 MB chunks and sent to the LTX Director's + // own /ltx_director_upload_chunk endpoint to bypass the 413 size limit. + async _uploadVideoFile(file) { + const CHUNK_SIZE = 50 * 1024 * 1024; // 50 MB + const safeFileName = file.name.replace(/[^a-zA-Z0-9.\-_]/g, '_'); + + // First check if the file already exists on the server to de-duplicate + try { + const checkResp = await api.fetchApi(`/ltx_director_check_file?filename=${encodeURIComponent(safeFileName)}&size=${file.size}`); + if (checkResp.status === 200) { + const checkResult = await checkResp.json(); + if (checkResult.exists) { + console.log(`[LTXDirector] File already exists: ${checkResult.name}. Reusing existing file.`); + return checkResult.name; + } + } + } catch (e) { + console.warn("[LTXDirector] Failed to check for existing file, proceeding with upload", e); + } + + if (file.size > CHUNK_SIZE) { + // --- Chunked path --- + const safeName = Date.now() + "_" + safeFileName; + const totalChunks = Math.ceil(file.size / CHUNK_SIZE); + for (let i = 0; i < totalChunks; i++) { + const chunk = file.slice(i * CHUNK_SIZE, (i + 1) * CHUNK_SIZE); + const formData = new FormData(); + formData.append("file", chunk); + formData.append("filename", safeName); + formData.append("chunk_index", i); + formData.append("total_chunks", totalChunks); + const resp = await api.fetchApi("/ltx_director_upload_chunk", { method: "POST", body: formData }); + if (resp.status !== 200) throw new Error("LTX Director video chunk upload failed"); + } + return safeName; // filename (no subfolder) in the input dir + } else { + // --- Single-shot path (small file) --- + const body = new FormData(); + body.append("image", file); + body.append("subfolder", "whatdreamscost"); + const resp = await api.fetchApi("/upload/image", { method: "POST", body }); + if (resp.status !== 200) throw new Error(`LTX Director video upload failed: ${resp.statusText}`); + const data = await resp.json(); + const subfolder = data.subfolder || ""; + return subfolder ? subfolder + "/" + data.name : data.name; + } + } + + async handleVideoUpload(files, targetFrameStart = null) { + const frameRate = this.getFrameRate(); + + if (this.retakeMode) { + const file = files[0]; + if (!file || !file.type.startsWith("video/")) return; + + // Clean up previous retake video if one exists + if (this.timeline.retakeVideo) { + const oldVid = this.timeline.retakeVideo; + if (oldVid.videoEl) { + oldVid.videoEl.pause(); + oldVid.videoEl.src = ""; + oldVid.videoEl.load(); + } + if (oldVid._blobUrl) { + URL.revokeObjectURL(oldVid._blobUrl); + } + } + + const blobUrl = URL.createObjectURL(file); + const vid = document.createElement('video'); + vid.crossOrigin = "Anonymous"; + vid.preload = 'auto'; + vid.muted = true; + + await new Promise((resolve) => { + vid.onloadeddata = async () => { + vid.onloadeddata = null; + const clipDurationSecs = vid.duration || 1; + const clipFrames = Math.max(1, Math.ceil(clipDurationSecs * frameRate)); + + this.timeline.retakeVideo = { + fileName: file.name, + imageFile: "", + videoDurationFrames: clipFrames, + _blobUrl: blobUrl, + fileSize: file.size, + videoEl: vid, + _uploading: true + }; + + // Initialize retake region to the middle 50% of the clip (25%–75%) + const retakeLen = Math.max(1, Math.round(clipFrames * 0.5)); + const retakeStartFrame = Math.round((clipFrames - retakeLen) / 2); + this.timeline.retakeStart = retakeStartFrame; + this.timeline.retakeLength = retakeLen; + if (this.timeline.retakePrompt === undefined) this.timeline.retakePrompt = ""; + if (this.timeline.retakeStrength === undefined) this.timeline.retakeStrength = 1.0; + + // Start background upload + this._uploadVideoFile(file).then(filePath => { + if (this.timeline.retakeVideo) { + this.timeline.retakeVideo.imageFile = filePath; + this.timeline.retakeVideo._uploading = false; + } + this.commitChanges(true); + this.render(); + }).catch(e => { + console.error(e); + if (this.timeline.retakeVideo) { + this.timeline.retakeVideo._uploading = false; + } + this.commitChanges(true); + this.render(); + }); + + this._ensureThumbnails(this.timeline.retakeVideo); + + this.syncWidgetsToRetakeDuration(clipFrames); + this.commitChanges(true); + this.render(); + resolve(); + }; + vid.src = blobUrl; + }); + return; + } + + for (let file of files) { + if (!file.type.startsWith("video/")) continue; + + await new Promise(async (resolve) => { + try { + // Use a local blob URL so the video element loads instantly from disk — + // no waiting for the server upload before the segment appears. + const blobUrl = URL.createObjectURL(file); + + const vid = document.createElement('video'); + vid.crossOrigin = "Anonymous"; + vid.preload = 'auto'; + vid.muted = true; + + vid.onloadeddata = async () => { + vid.onloadeddata = null; // prevent re-firing if src changes or browser buffers more data + const clipDurationSecs = vid.duration || 1; + const clipFrames = Math.max(1, Math.ceil(clipDurationSecs * frameRate)); + let newLength = clipFrames; + let newStart = targetFrameStart; + + if (newStart === null) { + newStart = 0; + this.timeline.segments.sort((a, b) => a.start - b.start); + for (let i = 0; i < this.timeline.segments.length; i++) { + let seg = this.timeline.segments[i]; + if (newStart + newLength <= seg.start) break; + newStart = Math.max(newStart, seg.start + seg.length); + } + } + + const currentDuration = this.getVisualDurationFrames(); + + if (targetFrameStart !== null) { + let tempId = "TEMP_" + Date.now(); + let tempVidId = tempId + "_v"; + let tempAudId = tempId + "_a"; + + this.timeline.segments.push({ id: tempVidId, start: newStart, length: newLength, type: "temp" }); + this.timeline.audioSegments.push({ id: tempAudId, start: newStart, length: newLength, type: "temp" }); + + let physicsCenter = newStart + this.getFrameRate() / 2; + + let resultSegments = this._applyCenterDragPhysics(this.timeline.segments, tempVidId, newStart, physicsCenter, currentDuration, currentDuration, 1); + let resultAudioSegments = this._applyCenterDragPhysics(this.timeline.audioSegments, tempAudId, newStart, physicsCenter, currentDuration, currentDuration, 1); + + this._resolveGlobalPhysics(resultSegments, resultAudioSegments, currentDuration, this.timeline.segments, this.timeline.audioSegments); + + for (let shiftedSeg of resultSegments) { + let original = this.timeline.segments.find(s => s.id === shiftedSeg.id); + if (original) original.start = shiftedSeg.resolvedStart !== undefined ? shiftedSeg.resolvedStart : shiftedSeg.start; + } + for (let shiftedSib of resultAudioSegments) { + let originalSib = this.timeline.audioSegments.find(s => s.id === shiftedSib.id); + if (originalSib) originalSib.start = shiftedSib.resolvedStart !== undefined ? shiftedSib.resolvedStart : shiftedSib.start; + } + + let tempVidSeg = resultSegments.find(s => s.id === tempVidId); + newStart = tempVidSeg.start; + this.timeline.segments = this.timeline.segments.filter(s => s.id !== tempVidId); + this.timeline.audioSegments = this.timeline.audioSegments.filter(s => s.id !== tempAudId); + targetFrameStart = newStart + newLength; + } + + const sharedId = Date.now().toString() + Math.random().toString(36).substr(2, 5); + + const vidSeg = { + id: sharedId + "_v", + type: "video", + start: newStart, + length: newLength, + trimStart: 0, + videoDurationFrames: clipFrames, + imageFile: "", // filled in once background upload completes + fileName: file.name, + prompt: "", + videoEl: vid, + _uploading: true, + _blobUrl: blobUrl, + fileSize: file.size + }; + + const audSeg = { + id: sharedId + "_a", + type: "audio", + start: newStart, + length: newLength, + trimStart: 0, + audioDurationFrames: clipFrames, + audioFile: "", // filled in once background upload completes + fileName: file.name, + waveformPeaks: [], + _uploading: true, + _decoding: true, + _blobUrl: blobUrl, + fileSize: file.size + }; + + // Extract first-frame thumbnail from local blob — instant + vid.currentTime = 0.01; + vid.onseeked = () => { + vid.onseeked = null; + const canvas = document.createElement('canvas'); + canvas.width = Math.min(vid.videoWidth, 512); + canvas.height = Math.round((vid.videoHeight / vid.videoWidth) * canvas.width); + const ctx = canvas.getContext('2d'); + ctx.drawImage(vid, 0, 0, canvas.width, canvas.height); + vidSeg.imageB64 = canvas.toDataURL('image/jpeg'); + + const imgObj = new Image(); + imgObj.onload = () => { vidSeg.imgObj = imgObj; this.render(); }; + imgObj.src = vidSeg.imageB64; + + // Add to timeline immediately + this.timeline.segments.push(vidSeg); + this.timeline.audioSegments.push(audSeg); + this.timeline.segments.sort((a, b) => a.start - b.start); + this.timeline.audioSegments.sort((a, b) => a.start - b.start); + + if (!this.retakeMode) { + this.growTimelineIfNeeded(vidSeg.start + vidSeg.length); + } + + this.selectionType = "image"; + this.selectedIndex = this.timeline.segments.findIndex(s => s.id === vidSeg.id); + this.updateUIFromSelection(); + this.commitChanges(true); + resolve(); // resolve immediately — don't block on upload + this._ensureThumbnails(vidSeg); + + // Background audio extraction (waveform peaks) — runs while user can already work + const IS_LARGE_FILE = file.size > 100 * 1024 * 1024; + if (IS_LARGE_FILE) { + console.log(`[LTXDirector] Large file detected (${(file.size / 1024 / 1024).toFixed(1)} MB). Offloading audio extraction to server.`); + } else { + this._extractAudioOnClient(file, audSeg.id, blobUrl); + } + + // Background upload — runs while the user can already work. + // We intentionally do NOT change vid.src after upload — the blob URL + // works perfectly for local playback. Only imageFile/audioFile + // need updating so Python can find the file at generation time. + this._uploadVideoFile(file).then(filePath => { + for (let s of this.timeline.segments) { + if (s._blobUrl === blobUrl || s.id === vidSeg.id) { + s.imageFile = filePath; + s._uploading = false; + } + } + for (let s of this.timeline.audioSegments) { + if (s._blobUrl === blobUrl || s.id === audSeg.id) { + s.audioFile = filePath; + s._uploading = false; + } + } + if (blobUrl && filePath) { + this._thumbnailCache = this._thumbnailCache || new Map(); + this._thumbnailPromises = this._thumbnailPromises || new Map(); + if (this._thumbnailCache.has(blobUrl)) { + this._thumbnailCache.set(filePath, this._thumbnailCache.get(blobUrl)); + } + if (this._thumbnailPromises.has(blobUrl)) { + this._thumbnailPromises.set(filePath, this._thumbnailPromises.get(blobUrl)); + } + } + + // Query server for extracted WAV audio file and waveform peaks + if (filePath) { + api.fetchApi(`/ltx_director_get_audio?filename=${encodeURIComponent(filePath)}`) + .then(r => r.json()) + .then(res => { + if (res.audio_file && res.peaks) { + for (let s of this.timeline.audioSegments) { + if (s.audioFile === filePath || s._blobUrl === blobUrl) { + s.audioFile = res.audio_file; + s.waveformPeaks = res.peaks; + s._decoding = false; + this._preloadAudioSegment(s); + } + } + } else { + // Fallback + if (IS_LARGE_FILE) { + console.warn("[LTXDirector] Server audio extraction failed for large file, skipping."); + for (let s of this.timeline.audioSegments) { + if (s.audioFile === filePath || s._blobUrl === blobUrl) { + s._decoding = false; + } + } + } else { + this._extractAudioOnClient(file, audSeg.id, blobUrl); + } + } + this.commitChanges(true); + this.render(); + }) + .catch(err => { + console.error("[LTXDirector] Server audio extraction query failed:", err); + for (let s of this.timeline.audioSegments) { + if (s.audioFile === filePath || s._blobUrl === blobUrl) { + s._decoding = false; + } + } + this.render(); + }); + } else { + this.commitChanges(true); + this.render(); + } + }).catch(err => { + console.error("[LTXDirector] Background video upload failed", err); + const currentVidSeg = this.timeline.segments.find(s => s.id === vidSeg.id); + if (currentVidSeg) currentVidSeg._uploading = false; + const currentAudSeg = this.timeline.audioSegments.find(s => s.id === audSeg.id); + if (currentAudSeg) currentAudSeg._uploading = false; + this.render(); + }); + }; + }; + + vid.onerror = (e) => { + console.error("Video load error", e); + URL.revokeObjectURL(blobUrl); + resolve(); + }; + + vid.src = blobUrl; + + } catch (err) { + console.error("Video upload failed", err); + resolve(); + } + }); + } + + if (this.videoFileInput) { + this.videoFileInput.value = ""; + } + } + + async generateVideoPreviewThumbs(file, count = 18) { + const url = URL.createObjectURL(file); + const video = document.createElement("video"); + video.preload = "metadata"; + video.muted = true; + video.src = url; + await new Promise((resolve, reject) => { + video.onloadedmetadata = () => resolve(); + video.onerror = () => reject(new Error("preview metadata failed")); + }); + const duration = Math.max(0.001, video.duration || 0.001); + const canvas = document.createElement("canvas"); + const maxW = 160, maxH = 90; + const scale = Math.min(maxW / Math.max(1, video.videoWidth || maxW), maxH / Math.max(1, video.videoHeight || maxH)); + canvas.width = Math.max(1, Math.round((video.videoWidth || maxW) * scale)); + canvas.height = Math.max(1, Math.round((video.videoHeight || maxH) * scale)); + const ctx = canvas.getContext("2d"); + const thumbs = []; + const seekTo = (t) => new Promise((resolve) => { + let finished = false; + const done = () => { + if (finished) return; + finished = true; + try { + ctx.drawImage(video, 0, 0, canvas.width, canvas.height); + thumbs.push(canvas.toDataURL("image/jpeg", 0.78)); + } catch (_) { } + resolve(); + }; + video.onseeked = done; + video.currentTime = Math.min(duration - 0.001, Math.max(0, t)); + setTimeout(done, 700); + }); + for (let i = 0; i < count; i++) { + const t = (duration * (i + 0.5)) / count; + await seekTo(t); + } + URL.revokeObjectURL(url); + return thumbs.filter(Boolean); + } + + // --- Async Motion Video Upload Logic --- + async handleMotionUpload(files, targetFrameStart = null) { + const frameRate = this.getFrameRate(); + + for (let file of files) { + if (!(file.type.startsWith("video/") || file.name.toLowerCase().match(/\.(mp4|webm|mkv|avi|mov|m4v|flv|wmv)$/))) continue; + + await new Promise(async (resolve) => { + try { + // Load from local blob immediately — no waiting for server upload + const blobUrl = URL.createObjectURL(file); + + const vid = document.createElement('video'); + vid.crossOrigin = "Anonymous"; + vid.preload = 'auto'; + vid.muted = true; + vid.onerror = (e) => { console.error("Motion video load error", e); URL.revokeObjectURL(blobUrl); resolve(); }; + + vid.onloadeddata = () => { + vid.onloadeddata = null; // prevent re-firing if src changes or browser buffers more data + const clipDurationSecs = vid.duration || 1; + const clipFrames = Math.max(1, Math.ceil(clipDurationSecs * frameRate)); + let newLength = clipFrames; + let newStart = targetFrameStart; + + if (newStart === null) { + newStart = 0; + this.timeline.motionSegments.sort((a, b) => a.start - b.start); + for (let i = 0; i < this.timeline.motionSegments.length; i++) { + let s = this.timeline.motionSegments[i]; + if (newStart + newLength <= s.start) break; + newStart = Math.max(newStart, s.start + s.length); + } + } + + const currentDuration = this.getVisualDurationFrames(); + if (targetFrameStart !== null) { + let tempId = "TEMP_" + Date.now(); + this.timeline.motionSegments.push({ id: tempId, start: newStart, length: newLength, type: "temp" }); + let result = this._applyCenterDragPhysics(this.timeline.motionSegments, tempId, newStart, newStart + newLength / 2, currentDuration, currentDuration, 1); + for (let shiftedSeg of result) { + let original = this.timeline.motionSegments.find(s => s.id === shiftedSeg.id); + if (original) original.start = shiftedSeg.resolvedStart !== undefined ? shiftedSeg.resolvedStart : shiftedSeg.start; + } + let tempSeg = this.timeline.motionSegments.find(s => s.id === tempId); + newStart = tempSeg.start; + this.timeline.motionSegments = this.timeline.motionSegments.filter(s => s.id !== tempId); + targetFrameStart = newStart + newLength; + } + + const seg = { + id: Date.now().toString() + Math.random().toString(36).substr(2, 5), + type: "motion_video", + start: newStart, + length: newLength, + trimStart: 0, + videoDurationFrames: clipFrames, + videoFile: "", // filled in once background upload completes + fileName: file.name, + videoStrength: 1.0, + videoAttentionStrength: 0.65, + resampleMode: "nearest", + previewThumbs: [], + previewThumbSourceFrames: clipFrames, + videoEl: vid, + _uploading: true, + _blobUrl: blobUrl, + fileSize: file.size + }; + + vid.currentTime = 0.01; + vid.onseeked = () => { + vid.onseeked = null; + const canvas = document.createElement('canvas'); + canvas.width = Math.min(vid.videoWidth, 512); + canvas.height = Math.round((vid.videoHeight / vid.videoWidth) * canvas.width); + const ctx = canvas.getContext('2d'); + ctx.drawImage(vid, 0, 0, canvas.width, canvas.height); + seg.imageB64 = canvas.toDataURL('image/jpeg'); + + const imgObj = new Image(); + imgObj.onload = () => { seg.imgObj = imgObj; this.render(); }; + imgObj.src = seg.imageB64; + + // Add to timeline immediately + this.timeline.motionSegments.push(seg); + this.timeline.motionSegments.sort((a, b) => a.start - b.start); + + if (!this.retakeMode) { + this.growTimelineIfNeeded(seg.start + seg.length); + } + + this.selectionType = "motion"; + this.selectedIndex = this.timeline.motionSegments.findIndex(s => s.id === seg.id); + this.updateUIFromSelection(); + this.commitChanges(true); + resolve(); // resolve immediately — don't block on upload + this._ensureThumbnails(seg); + + // Background upload — runs while the user can already work. + // We intentionally do NOT change vid.src after upload — the blob URL + // works perfectly for local playback. Only videoFile needs updating + // so Python can find the file at generation time. + this._uploadVideoFile(file).then(filePath => { + for (let s of this.timeline.motionSegments) { + if (s._blobUrl === blobUrl || s.id === seg.id) { + s.videoFile = filePath; + s._uploading = false; + } + } + if (blobUrl && filePath) { + this._thumbnailCache = this._thumbnailCache || new Map(); + this._thumbnailPromises = this._thumbnailPromises || new Map(); + if (this._thumbnailCache.has(blobUrl)) { + this._thumbnailCache.set(filePath, this._thumbnailCache.get(blobUrl)); + } + if (this._thumbnailPromises.has(blobUrl)) { + this._thumbnailPromises.set(filePath, this._thumbnailPromises.get(blobUrl)); + } + } + const isOverrideAudio = !!(this.node.properties.overrideAudio || this.timeline.overrideAudio); + if (isOverrideAudio) { + const s = this.timeline.motionSegments.find(s => s.id === seg.id); + if (s) { + this._preloadMotionAudioSegment(s); + } + } + this.commitChanges(true); + this.render(); + }).catch(err => { + console.error("[LTXDirector] Background motion video upload failed", err); + const currentSeg = this.timeline.motionSegments.find(s => s.id === seg.id); + if (currentSeg) currentSeg._uploading = false; + this.render(); + }); + }; + }; + + vid.src = blobUrl; + + } catch (err) { + console.error("[LTXDirector] Motion video processing failed", err); + resolve(); + } + }); + } + } + + // --- Async Audio Upload Logic --- async handleAudioUpload(files, targetFrameStart = null) { const frameRate = this.getFrameRate(); @@ -1554,6 +4601,7 @@ class TimelineEditor { try { const body = new FormData(); body.append("image", file); + body.append("subfolder", "whatdreamscost"); const resp = await api.fetchApi("/upload/image", { method: "POST", body }); if (resp.status !== 200) { resolve(); return; } @@ -1602,13 +4650,25 @@ class TimelineEditor { if (targetFrameStart !== null) { let tempId = "TEMP_" + Date.now(); this.timeline.audioSegments.push({ id: tempId, start: newStart, length: newLength, type: "temp" }); - let result = this._applyCenterDragPhysics(this.timeline.audioSegments, tempId, newStart, newStart + newLength / 2, currentDuration, currentDuration, 1); + let physicsCenter = newStart + this.getFrameRate() / 2; + let result = this._applyCenterDragPhysics(this.timeline.audioSegments, tempId, newStart, physicsCenter, currentDuration, currentDuration, 1); + + let siblingPhysics = (this.timeline.segments || []).map(s => ({ ...s })); + + this._resolveGlobalPhysics(siblingPhysics, result, currentDuration, this.timeline.segments, this.timeline.audioSegments); for (let shiftedSeg of result) { let original = this.timeline.audioSegments.find(s => s.id === shiftedSeg.id); if (original) original.start = shiftedSeg.resolvedStart !== undefined ? shiftedSeg.resolvedStart : shiftedSeg.start; } + for (let shiftedSib of siblingPhysics) { + let originalSib = this.timeline.segments.find(s => s.id === shiftedSib.id); + if (originalSib) { + originalSib.start = shiftedSib.start; + } + } + let tempSeg = this.timeline.audioSegments.find(s => s.id === tempId); newStart = tempSeg.start; this.timeline.audioSegments = this.timeline.audioSegments.filter(s => s.id !== tempId); @@ -1627,11 +4687,17 @@ class TimelineEditor { audioDurationFrames: clipFrames, audioFile: audioFile, fileName: file.name, - waveformPeaks: peaks + waveformPeaks: peaks, + _audioBuffer: audioBuffer }; this.timeline.audioSegments.push(seg); this.timeline.audioSegments.sort((a, b) => a.start - b.start); + + if (!this.retakeMode) { + this.growTimelineIfNeeded(seg.start + seg.length); + } + this.selectionType = "audio"; this.selectedIndex = this.timeline.audioSegments.findIndex(s => s.id === seg.id); @@ -1648,16 +4714,458 @@ class TimelineEditor { this.audioFileInput.value = ""; } - deleteSelectedSegment() { - if (this.selectionType === "audio") { - if (this.timeline.audioSegments.length === 0 || this.selectedIndex === -1) return; - this.timeline.audioSegments.splice(this.selectedIndex, 1); - this.selectedIndex = Math.max(-1, this.selectedIndex - 1); - } else { - if (this.timeline.segments.length === 0 || this.selectedIndex === -1) return; - this.timeline.segments.splice(this.selectedIndex, 1); - this.selectedIndex = Math.max(-1, this.selectedIndex - 1); + markSegment(seg) { + if (!seg) return; + const newStart = Math.round(seg.start); + const newEnd = Math.max(newStart + 1, Math.round(seg.start + seg.length)); + + const currentStart = this.getStartFrames(); + const currentEnd = this.endFramesWidget ? parseInt(this.endFramesWidget.value, 10) : (currentStart + this.getDurationFrames()); + + let targetStart = newStart; + let targetEnd = newEnd; + + if (currentStart === newStart && currentEnd === newEnd) { + const allSegs = [ + ...(this.timeline.segments || []), + ...(this.timeline.motionSegments || []), + ...(this.timeline.audioSegments || []) + ]; + let lastSegmentEnd = 0; + for (const s of allSegs) { + if (s.start + s.length > lastSegmentEnd) { + lastSegmentEnd = s.start + s.length; + } + } + if (lastSegmentEnd <= 0) { + lastSegmentEnd = this.getDurationFrames(); + } + targetStart = 0; + targetEnd = Math.max(1, Math.round(lastSegmentEnd)); } + + if (this.startFramesWidget && this.endFramesWidget) { + this.startFramesWidget.value = targetStart; + this.endFramesWidget.value = targetEnd; + if (this.startFramesWidget.callback) { + this.startFramesWidget.callback(targetStart); + } + if (this.endFramesWidget.callback) { + this.endFramesWidget.callback(targetEnd); + } + this.commitChanges(); + this.render(); + } + } + + markCurrentSelection() { + if (this.retakeMode) { + if (this.timeline.retakeVideo) { + const baseVideoDur = this.timeline.retakeVideo.videoDurationFrames || 24; + const targetStart = 0; + const targetEnd = baseVideoDur; + + if (this.startFramesWidget && this.endFramesWidget) { + this.startFramesWidget.value = targetStart; + this.endFramesWidget.value = targetEnd; + if (this.startFramesWidget.callback) { + this.startFramesWidget.callback(targetStart); + } + if (this.endFramesWidget.callback) { + this.endFramesWidget.callback(targetEnd); + } + this.commitChanges(); + this.render(); + } + } + return; + } + + const allSegs = [ + ...(this.timeline.segments || []), + ...(this.timeline.motionSegments || []), + ...(this.timeline.audioSegments || []) + ]; + let targetSegs = []; + + if (this.selectedSegmentIds && this.selectedSegmentIds.length > 0) { + targetSegs = allSegs.filter(s => this.selectedSegmentIds.includes(s.id)); + } + + if (targetSegs.length === 0 && this.selectedIndex >= 0 && this.selectionType) { + const arr = this.getSegmentArray(this.selectionType); + if (arr && arr[this.selectedIndex]) { + targetSegs = [arr[this.selectedIndex]]; + } + } + + if (targetSegs.length === 0) return; + + let minStart = Infinity; + let maxEnd = -Infinity; + for (const s of targetSegs) { + if (s.start < minStart) { + minStart = s.start; + } + if (s.start + s.length > maxEnd) { + maxEnd = s.start + s.length; + } + } + + if (minStart !== Infinity && maxEnd !== -Infinity) { + const newStart = Math.round(minStart); + const newEnd = Math.max(newStart + 1, Math.round(maxEnd)); + + const currentStart = this.getStartFrames(); + const currentEnd = this.endFramesWidget ? parseInt(this.endFramesWidget.value, 10) : (currentStart + this.getDurationFrames()); + + let targetStart = newStart; + let targetEnd = newEnd; + + if (currentStart === newStart && currentEnd === newEnd) { + let lastSegmentEnd = 0; + for (const s of allSegs) { + if (s.start + s.length > lastSegmentEnd) { + lastSegmentEnd = s.start + s.length; + } + } + if (lastSegmentEnd <= 0) { + lastSegmentEnd = this.getDurationFrames(); + } + targetStart = 0; + targetEnd = Math.max(1, Math.round(lastSegmentEnd)); + } + + if (this.startFramesWidget && this.endFramesWidget) { + this.startFramesWidget.value = targetStart; + this.endFramesWidget.value = targetEnd; + if (this.startFramesWidget.callback) { + this.startFramesWidget.callback(targetStart); + } + if (this.endFramesWidget.callback) { + this.endFramesWidget.callback(targetEnd); + } + this.commitChanges(); + this.render(); + } + } + } + + deleteSelectedSegment() { + if (this.selectedSegmentIds && this.isMultiSelectActive()) { + const idsToDelete = new Set(this.selectedSegmentIds); + for (const id of this.selectedSegmentIds) { + if (id.endsWith("_v")) idsToDelete.add(id.slice(0, -2) + "_a"); + else if (id.endsWith("_a")) idsToDelete.add(id.slice(0, -2) + "_v"); + } + + this.timeline.segments = this.timeline.segments.filter(s => !idsToDelete.has(s.id)); + this.timeline.motionSegments = this.timeline.motionSegments.filter(s => !idsToDelete.has(s.id)); + this.timeline.audioSegments = this.timeline.audioSegments.filter(s => !idsToDelete.has(s.id)); + + this.selectedSegmentIds = []; + this.selectedIndex = -1; + } else { + const delSibling = (seg) => { + if (!seg || !seg.id) return; + const isVid = seg.id.endsWith("_v"); + const isAud = seg.id.endsWith("_a"); + if (!isVid && !isAud) return; + + const siblingId = isVid ? seg.id.slice(0, -2) + "_a" : seg.id.slice(0, -2) + "_v"; + const siblingArray = isVid ? this.timeline.audioSegments : this.timeline.segments; + const sIdx = siblingArray.findIndex(s => s.id === siblingId); + if (sIdx !== -1) siblingArray.splice(sIdx, 1); + }; + + if (this.selectionType === "audio") { + if (this.timeline.audioSegments.length === 0 || this.selectedIndex === -1) return; + delSibling(this.timeline.audioSegments[this.selectedIndex]); + this.timeline.audioSegments.splice(this.selectedIndex, 1); + this.selectedIndex = Math.max(-1, this.selectedIndex - 1); + } else if (this.selectionType === "motion") { + if (this.timeline.motionSegments.length === 0 || this.selectedIndex === -1) return; + delSibling(this.timeline.motionSegments[this.selectedIndex]); + this.timeline.motionSegments.splice(this.selectedIndex, 1); + this.selectedIndex = Math.max(-1, this.selectedIndex - 1); + } else { + if (this.timeline.segments.length === 0 || this.selectedIndex === -1) return; + delSibling(this.timeline.segments[this.selectedIndex]); + this.timeline.segments.splice(this.selectedIndex, 1); + this.selectedIndex = Math.max(-1, this.selectedIndex - 1); + } + this.selectedSegmentIds = []; + } + this.updateUIFromSelection(); + this.commitChanges(); + this.render(); + } + + getCanonicalTrack(track) { + if (track === "image" || track === "video" || track === "text") return "image"; + if (track === "audio") return "audio"; + if (track === "motion" || track === "motion_video") return "motion"; + return track; + } + + pasteCopiedSegment() { + if (!window._ltxCopiedSegment || !window._ltxCopiedSegmentType) return; + const trackType = window._ltxCopiedSegmentType; + const startFrame = Math.round(this.currentFrame); + this.pasteSegmentAtFrame(window._ltxCopiedSegment.main, trackType, window._ltxCopiedSegment.sibling, startFrame); + } + + pasteSegmentAtFrame(copiedSegData, copiedTrack, siblingSegData, startFrame) { + const isAudio = copiedTrack === "audio"; + + const randId = () => Date.now().toString() + Math.random().toString(36).substr(2, 5); + const baseId = randId(); + + let mainSeg = { ...copiedSegData }; + let sibSeg = siblingSegData ? { ...siblingSegData } : null; + + if (sibSeg) { + mainSeg.id = baseId + (isAudio ? "_a" : "_v"); + sibSeg.id = baseId + (isAudio ? "_v" : "_a"); + } else { + if (mainSeg.id && (mainSeg.id.endsWith("_v") || mainSeg.id.endsWith("_a"))) { + mainSeg.id = mainSeg.id.slice(0, -2); + } else { + mainSeg.id = baseId; + } + } + + if (mainSeg.thumbnails) mainSeg.thumbnails = [...mainSeg.thumbnails]; + if (sibSeg && sibSeg.thumbnails) sibSeg.thumbnails = [...sibSeg.thumbnails]; + + mainSeg.start = startFrame; + if (sibSeg) sibSeg.start = startFrame; + + const mainArr = isAudio ? [...this.timeline.audioSegments] : (copiedTrack === "motion" ? [...this.timeline.motionSegments] : [...this.timeline.segments]); + mainArr.push(mainSeg); + mainArr.sort((a, b) => a.start - b.start); + + const sibArr = isAudio ? [...this.timeline.segments] : [...this.timeline.audioSegments]; + if (sibSeg) { + sibArr.push(sibSeg); + sibArr.sort((a, b) => a.start - b.start); + } + + const durationFrames = this.getDurationFrames(); + const totalFrames = this.getVisualDurationFrames(); + const width = this.canvas.offsetWidth || this._lastWidth; + + const mainInit = mainArr.map(s => ({ ...s })); + const sibInit = sibSeg ? sibArr.map(s => ({ ...s })) : null; + + let finalMain, finalSib; + finalMain = this._applyCenterDragPhysics(mainInit, mainSeg.id, startFrame, startFrame + mainSeg.length / 2, durationFrames, totalFrames, width, true); + if (sibSeg) { + finalSib = this._applyCenterDragPhysics(sibInit, sibSeg.id, startFrame, startFrame + sibSeg.length / 2, durationFrames, totalFrames, width, true); + } + + if (sibSeg) { + const activeTimeline = isAudio ? finalMain : finalSib; + const siblingTimeline = isAudio ? finalSib : finalMain; + this._resolveGlobalPhysics(activeTimeline, siblingTimeline, durationFrames, mainInit, sibInit); + } + + const restoreDOM = (outArr, refArr) => { + for (let ps of outArr) { + const orig = refArr.find(s => s.id === ps.id); + if (orig) { + ps.videoEl = orig.videoEl; + ps.imgObj = orig.imgObj; + if (orig.thumbnails) ps.thumbnails = orig.thumbnails; + } + } + }; + + restoreDOM(finalMain, mainArr); + if (sibSeg) restoreDOM(finalSib, sibArr); + + if (copiedTrack === "audio") { + this.timeline.audioSegments = finalMain; + if (sibSeg) this.timeline.segments = finalSib; + } else if (copiedTrack === "motion") { + this.timeline.motionSegments = finalMain; + } else { + this.timeline.segments = finalMain; + if (sibSeg) this.timeline.audioSegments = finalSib; + } + + this.selectionType = copiedTrack; + this.selectedIndex = this.getSegmentArray(copiedTrack).findIndex(s => s.id === mainSeg.id); + + if (!this.retakeMode) { + this.growTimelineIfNeeded(mainSeg.start + mainSeg.length); + } + + this.updateUIFromSelection(); + this.commitChanges(); + this.render(); + } + + splitSegmentAtPlayhead(seg, trackType) { + if (this.isPlaying) { + this.pauseAudio(); + } + + const splitFrame = Math.round(this.currentFrame); + if (splitFrame <= seg.start || splitFrame >= seg.start + seg.length) { + return; + } + + const isVidLink = (trackType === "image" || trackType === "video") && seg.id.endsWith("_v"); + const isAudLink = trackType === "audio" && seg.id.endsWith("_a"); + let sibling = null; + if (isVidLink) { + sibling = this.timeline.audioSegments.find(s => s.id === seg.id.slice(0, -2) + "_a"); + } else if (isAudLink) { + sibling = this.timeline.segments.find(s => s.id === seg.id.slice(0, -2) + "_v"); + } + + const randId = () => Date.now().toString() + Math.random().toString(36).substr(2, 5); + const leftBase = randId(); + const rightBase = randId(); + + const leftLen = splitFrame - seg.start; + const rightLen = seg.start + seg.length - splitFrame; + + if (sibling) { + const videoSeg = isVidLink ? seg : sibling; + const audioSeg = isVidLink ? sibling : seg; + + const leftVid = { + ...videoSeg, + id: leftBase + "_v", + length: leftLen, + videoEl: null, + _blobUrl: videoSeg._blobUrl || (videoSeg.videoEl ? videoSeg.videoEl.src : null), + thumbnails: videoSeg.thumbnails ? [...videoSeg.thumbnails] : null + }; + const leftAud = { + ...audioSeg, + id: leftBase + "_a", + length: leftLen, + _blobUrl: videoSeg._blobUrl || (videoSeg.videoEl ? videoSeg.videoEl.src : null) + }; + + let rightImageB64 = videoSeg.imageB64; + let rightImgObj = videoSeg.imgObj; + if (videoSeg.thumbnails && videoSeg.thumbnails.length > 0) { + const targetTime = ((videoSeg.trimStart || 0) + leftLen) / this.getFrameRate(); + let nearest = videoSeg.thumbnails[0]; + let minDiff = Infinity; + for (const t of videoSeg.thumbnails) { + const diff = Math.abs(t.time - targetTime); + if (diff < minDiff) { + minDiff = diff; + nearest = t; + } + } + if (nearest && nearest.img) { + rightImageB64 = nearest.img.src; + rightImgObj = nearest.img; + } + } + + const rightVid = { + ...videoSeg, + id: rightBase + "_v", + start: splitFrame, + length: rightLen, + trimStart: (videoSeg.trimStart || 0) + leftLen, + videoEl: null, + imageB64: rightImageB64, + imgObj: rightImgObj, + _blobUrl: videoSeg._blobUrl || (videoSeg.videoEl ? videoSeg.videoEl.src : null), + thumbnails: videoSeg.thumbnails ? [...videoSeg.thumbnails] : null + }; + const rightAud = { + ...audioSeg, + id: rightBase + "_a", + start: splitFrame, + length: rightLen, + trimStart: (audioSeg.trimStart || 0) + leftLen, + _blobUrl: videoSeg._blobUrl || (videoSeg.videoEl ? videoSeg.videoEl.src : null) + }; + + this.timeline.segments = this.timeline.segments.filter(s => s.id !== videoSeg.id); + this.timeline.audioSegments = this.timeline.audioSegments.filter(s => s.id !== audioSeg.id); + + this.timeline.segments.push(leftVid, rightVid); + this.timeline.audioSegments.push(leftAud, rightAud); + + this.timeline.segments.sort((a, b) => a.start - b.start); + this.timeline.audioSegments.sort((a, b) => a.start - b.start); + + this.selectionType = trackType; + const targetId = trackType === "audio" ? leftAud.id : leftVid.id; + const targetArray = this.getSegmentArray(trackType); + this.selectedIndex = targetArray.findIndex(s => s.id === targetId); + + } else { + const targetArray = this.getSegmentArray(trackType); + + const leftSeg = { + ...seg, + id: leftBase, + length: leftLen + }; + if (seg.type === "video" || seg.type === "motion_video") { + leftSeg.videoEl = null; + leftSeg._blobUrl = seg._blobUrl || (seg.videoEl ? seg.videoEl.src : null); + leftSeg.thumbnails = seg.thumbnails ? [...seg.thumbnails] : null; + } + + let rightImageB64 = seg.imageB64; + let rightImgObj = seg.imgObj; + if (seg.thumbnails && seg.thumbnails.length > 0) { + const targetTime = ((seg.trimStart || 0) + leftLen) / this.getFrameRate(); + let nearest = seg.thumbnails[0]; + let minDiff = Infinity; + for (const t of seg.thumbnails) { + const diff = Math.abs(t.time - targetTime); + if (diff < minDiff) { + minDiff = diff; + nearest = t; + } + } + if (nearest && nearest.img) { + rightImageB64 = nearest.img.src; + rightImgObj = nearest.img; + } + } + + const rightSeg = { + ...seg, + id: rightBase, + start: splitFrame, + length: rightLen, + trimStart: (seg.trimStart || 0) + leftLen + }; + if (seg.type === "video" || seg.type === "motion_video") { + rightSeg.videoEl = null; + rightSeg.imageB64 = rightImageB64; + rightSeg.imgObj = rightImgObj; + rightSeg._blobUrl = seg._blobUrl || (seg.videoEl ? seg.videoEl.src : null); + rightSeg.thumbnails = seg.thumbnails ? [...seg.thumbnails] : null; + } + + const idx = targetArray.findIndex(s => s.id === seg.id); + if (idx !== -1) { + targetArray.splice(idx, 1); + } + + targetArray.push(leftSeg, rightSeg); + targetArray.sort((a, b) => a.start - b.start); + + this.selectionType = trackType; + this.selectedIndex = targetArray.findIndex(s => s.id === leftSeg.id); + } + + this.loadMedia(); this.updateUIFromSelection(); this.commitChanges(); this.render(); @@ -1666,7 +5174,7 @@ class TimelineEditor { formatTime(frames, dropSuffix = false) { const mode = this.displayModeWidget ? this.displayModeWidget.value : "seconds"; if (mode === "seconds") { - const secs = frames / this.getFrameRate(); + const secs = Math.round(frames) / this.getFrameRate(); return dropSuffix ? secs.toFixed(2) : secs.toFixed(2) + "s"; } return dropSuffix ? Math.round(frames).toString() : Math.round(frames) + " frames"; @@ -1674,22 +5182,64 @@ class TimelineEditor { updateWidgetVisibility() { const mode = this.displayModeWidget ? this.displayModeWidget.value : "seconds"; + const isSeconds = mode === "seconds"; - if (this.durationFramesWidget) { - // Always visible regardless of display mode - this.durationFramesWidget.type = "INT"; - if (!this.durationFramesWidget.options) this.durationFramesWidget.options = {}; - this.durationFramesWidget.options.hidden = false; - this.durationFramesWidget.hidden = false; - delete this.durationFramesWidget.computeSize; - } - if (this.durationSecondsWidget) { - // Always visible regardless of display mode - this.durationSecondsWidget.type = "FLOAT"; - if (!this.durationSecondsWidget.options) this.durationSecondsWidget.options = {}; - this.durationSecondsWidget.options.hidden = false; - this.durationSecondsWidget.hidden = false; - delete this.durationSecondsWidget.computeSize; + const isLiteGraph = !window.LiteGraph || !window.LiteGraph.vueNodesMode; + + if (isSeconds) { + if (this.startFramesWidget) hideWidget(this.startFramesWidget); + if (this.endFramesWidget) hideWidget(this.endFramesWidget); + if (this.durationFramesWidget) hideWidget(this.durationFramesWidget); + if (this.startSecondsWidget) showWidget(this.startSecondsWidget); + if (this.endSecondsWidget) showWidget(this.endSecondsWidget); + if (this.durationSecondsWidget) showWidget(this.durationSecondsWidget); + + // LiteGraph: remove frame input slots, restore second input slots + if (isLiteGraph && this.node.inputs) { + for (const name of ["start_frame", "end_frame", "duration_frames"]) { + const idx = this.node.inputs.findIndex(i => i.name === name); + if (idx !== -1 && this.node.inputs[idx].link == null) { + this.node.removeInput(idx); + } + } + for (const [name, type] of [["start_second", "FLOAT"], ["end_second", "FLOAT"], ["duration_seconds", "FLOAT"]]) { + if (!this.node.inputs.find(i => i.name === name)) { + const w = this.node.widgets?.find(w => w.name === name); + const slot = this.node.addInput(name, type); + // keep the slot linked to its widget + if (w && slot != null) { + const inp = this.node.inputs[this.node.inputs.length - 1]; + if (inp) inp.widget = { name }; + } + } + } + } + } else { + if (this.startSecondsWidget) hideWidget(this.startSecondsWidget); + if (this.endSecondsWidget) hideWidget(this.endSecondsWidget); + if (this.durationSecondsWidget) hideWidget(this.durationSecondsWidget); + if (this.startFramesWidget) showWidget(this.startFramesWidget); + if (this.endFramesWidget) showWidget(this.endFramesWidget); + if (this.durationFramesWidget) showWidget(this.durationFramesWidget); + + // LiteGraph: remove second input slots, restore frame input slots + if (isLiteGraph && this.node.inputs) { + for (const name of ["start_second", "end_second", "duration_seconds"]) { + const idx = this.node.inputs.findIndex(i => i.name === name); + if (idx !== -1 && this.node.inputs[idx].link == null) { + this.node.removeInput(idx); + } + } + for (const [name, type] of [["start_frame", "INT"], ["end_frame", "INT"], ["duration_frames", "INT"]]) { + if (!this.node.inputs.find(i => i.name === name)) { + const slot = this.node.addInput(name, type); + if (slot != null) { + const inp = this.node.inputs[this.node.inputs.length - 1]; + if (inp) inp.widget = { name }; + } + } + } + } } // Force node resize and redraw deferred to next tick @@ -1704,7 +5254,143 @@ class TimelineEditor { }, 0); } + getGlobalPrompt() { + if (this.globalPromptInput) { + return this.globalPromptInput.value || ""; + } + let val = ""; + if (this.node) { + const globalInput = this.node.inputs?.find(i => i.name === "global_prompt"); + if (globalInput && globalInput.link !== null && globalInput.link !== undefined) { + const link = window.app.graph?.links?.[globalInput.link]; + if (link) { + const originNode = window.app.graph.getNodeById(link.origin_id); + if (originNode && originNode.widgets && originNode.widgets.length > 0) { + val = originNode.widgets[0].value || ""; + } + } + } else { + const w = this.node.widgets?.find(x => x.name === "global_prompt"); + if (w) { + val = w.value || ""; + } else { + val = this.node.properties?.global_prompt || ""; + } + } + } + return val; + } + + syncGlobalPrompt(val) { + if (this.node.properties) { + this.node.properties.global_prompt = val; + } + if (this.retakeMode) { + this.timeline.retake_global_prompt = val; + } else { + this.timeline.global_prompt = val; + } + const globalInput = this.node.inputs?.find(i => i.name === "global_prompt"); + let synced = false; + if (globalInput && globalInput.link !== null && globalInput.link !== undefined) { + const link = window.app.graph?.links?.[globalInput.link]; + if (link) { + const originNode = window.app.graph.getNodeById(link.origin_id); + if (originNode && originNode.widgets && originNode.widgets.length > 0) { + const w = originNode.widgets[0]; + const oldVal = w.value; + w.value = val; + if (originNode.onWidgetChanged) { + originNode.onWidgetChanged(w.name, val, oldVal, w); + } + if (w.callback) { + try { + originNode.widgets[0].callback(val); + } catch (err) { } + } + synced = true; + } + } + } + if (!synced) { + const w = this.node.widgets?.find(x => x.name === "global_prompt"); + if (w) { + const oldVal = w.value; + w.value = val; + if (this.node.onWidgetChanged) { + this.node.onWidgetChanged(w.name, val, oldVal, w); + } + if (w.callback) { + try { + w.callback(val); + } catch (err) { } + } + } + } + if (this.globalPromptInput && this.globalPromptInput.value !== val) { + this.globalPromptInput.value = val; + } + if (this.node) { + this.node.setDirtyCanvas(true, false); + } + if (window.app?.graph) { + if (window.app.graph.change) window.app.graph.change(); + if (window.app.graph.onNodeChanged) window.app.graph.onNodeChanged(this.node); + if (window.app.graph.onStateChanged) window.app.graph.onStateChanged(); + } + } + updateUIFromSelection() { + if (this.selectedSegmentIds && this.isMultiSelectActive()) { + if (this.globalPromptInput) { + this.globalPromptInput.disabled = true; + this.globalPromptInput.style.opacity = "0.35"; + } + if (this.promptWrapper) this.promptWrapper.style.display = "block"; + if (this.promptInput) { + this.promptInput.value = ""; + this.promptInput.placeholder = "(Multiple Segments Selected)"; + this.promptInput.disabled = true; + this.promptInput.style.opacity = "0.35"; + } + + if (this.segmentPromptLabel) { + this.segmentPromptLabel.style.display = "block"; + this.segmentPromptLabel.textContent = "Segment Prompt"; + } + + if (this.strengthRow) this.strengthRow.style.display = "flex"; + if (this.strengthLabel) this.strengthLabel.style.display = "inline"; + if (this.strengthValue) { + this.strengthValue.style.display = "inline-block"; + this.strengthValue.value = ""; + this.strengthValue.placeholder = "(Multiple)"; + this.strengthValue.disabled = true; + this.strengthValue.style.opacity = "0.35"; + } + + if (this.vidStrLabel) this.vidStrLabel.style.display = "none"; + if (this.vidStrValue) { + this.vidStrValue.style.display = "none"; + this.vidStrValue.disabled = true; + this.vidStrValue.style.opacity = "0.35"; + } + if (this.vidAttnLabel) this.vidAttnLabel.style.display = "none"; + if (this.vidAttnValue) { + this.vidAttnValue.style.display = "none"; + this.vidAttnValue.disabled = true; + this.vidAttnValue.style.opacity = "0.35"; + } + + if (this.audioInfoArea) this.audioInfoArea.style.display = "none"; + if (this.motionInfoArea) this.motionInfoArea.style.display = "none"; + + if (this.segmentBoundsDisplay) { + this.segmentBoundsDisplay.textContent = "Multiple Segments Selected"; + } + return; + } + let seg = null; if (this.selectedIndex >= 0) { if (this.selectionType === "audio") { @@ -1714,6 +5400,13 @@ class TimelineEditor { const arr = (this._previewSegments && previewIsAudio) ? this._previewSegments : this.timeline.audioSegments; seg = arr.find(s => s.id === origSeg.id) || origSeg; } + } else if (this.selectionType === "motion") { + const origSeg = this.timeline.motionSegments[this.selectedIndex]; + if (origSeg) { + const previewIsMotion = this._ghostTrack === 'motion' || (this._previewSegments && this._ghostTrack === null && this.selectionType === 'motion'); + const arr = (this._previewSegments && previewIsMotion) ? this._previewSegments : this.timeline.motionSegments; + seg = arr.find(s => s.id === origSeg.id) || origSeg; + } } else { const origSeg = this.timeline.segments[this.selectedIndex]; if (origSeg) { @@ -1724,10 +5417,69 @@ class TimelineEditor { } } - if (this.selectionType === "audio" && seg) { - this.promptInput.style.display = "none"; + // Reset default disabled/opacity values + if (this.vidStrValue) { + this.vidStrValue.disabled = false; + this.vidStrValue.style.opacity = ""; + } + if (this.vidAttnValue) { + this.vidAttnValue.disabled = false; + this.vidAttnValue.style.opacity = ""; + } + if (this.strengthValue) { + this.strengthValue.style.opacity = ""; + this.strengthValue.placeholder = ""; + } + if (this.promptInput) { + this.promptInput.placeholder = ""; + this.promptInput.style.opacity = ""; + } + + if (this.retakeMode) { + if (this.promptWrapper) this.promptWrapper.style.display = "block"; + this.promptInput.disabled = false; + this.promptInput.style.opacity = "1.0"; + this.promptInput.placeholder = "Enter prompt for retake region..."; + this.promptInput.value = this.timeline.retakePrompt || ""; + this.strengthRow.style.display = "flex"; + this.strengthLabel.style.display = "inline"; + this.strengthLabel.textContent = "Guide Strength:"; + this.strengthValue.style.display = "inline-block"; + this.strengthValue.disabled = true; + this.strengthValue.style.opacity = "0.35"; + this.strengthValue.value = (this.timeline.retakeStrength ?? 1.0).toFixed(2); + + this.vidStrLabel.style.display = "none"; + this.vidStrValue.style.display = "none"; + this.vidAttnLabel.style.display = "none"; + this.vidAttnValue.style.display = "none"; + + this.audioInfoArea.style.display = "none"; + this.motionInfoArea.style.display = "none"; + + if (this.segmentBoundsDisplay) { + const startStr = this.formatTime(this.timeline.retakeStart, true); + const endStr = this.formatTime(this.timeline.retakeStart + this.timeline.retakeLength, true); + const lengthStr = this.formatTime(this.timeline.retakeLength, true); + this.segmentBoundsDisplay.textContent = `Start: ${startStr} | End: ${endStr} | Length: ${lengthStr}`; + } + } else if (this.selectionType === "audio" && seg) { + if (this.globalPromptInput) { + this.globalPromptInput.disabled = false; + this.globalPromptInput.style.opacity = "1.0"; + } + if (this.promptWrapper) this.promptWrapper.style.display = "none"; + this.strengthRow.style.display = "flex"; + this.strengthLabel.style.display = "inline"; + this.strengthLabel.textContent = "Guide Strength:"; + this.strengthValue.style.display = "inline-block"; + this.vidStrLabel.style.display = "none"; + this.vidStrValue.style.display = "none"; + this.vidAttnLabel.style.display = "none"; + this.vidAttnValue.style.display = "none"; this.audioInfoArea.style.display = "block"; + this.motionInfoArea.style.display = "none"; this.audioInfoArea.innerHTML = ` File: ${seg.fileName || "Unknown"}
Length: ${this.formatTime(seg.audioDurationFrames)} Output Length: ${this.formatTime(seg.length)}
@@ -1735,40 +5487,193 @@ class TimelineEditor { `; this.strengthValue.value = "1.00"; this.strengthValue.disabled = true; - } else { - this.audioInfoArea.style.display = "none"; - this.promptInput.style.display = "block"; + } else if (this.selectionType === "motion" && seg) { + if (this.globalPromptInput) { + this.globalPromptInput.disabled = true; + this.globalPromptInput.style.opacity = "0.4"; + } + if (this.promptWrapper) this.promptWrapper.style.display = "block"; + this.promptInput.disabled = false; + this.promptInput.style.opacity = "1.0"; + this.promptInput.placeholder = "Global prompt (syncs across all IC LoRA segments)..."; + this.promptInput.value = this.getGlobalPrompt(); + if (this.segmentPromptLabel) { + this.segmentPromptLabel.style.display = "block"; + this.segmentPromptLabel.textContent = "Global Prompt (IC-LoRA)"; + } + this.strengthRow.style.display = "flex"; + this.strengthLabel.style.display = "none"; + this.strengthValue.style.display = "none"; + this.vidStrLabel.style.display = "inline"; + this.vidStrValue.style.display = "inline-block"; + this.vidAttnLabel.style.display = "inline"; + this.vidAttnValue.style.display = "inline-block"; + + this.vidStrValue.value = (seg.videoStrength ?? 1.0).toFixed(2); + this.vidAttnValue.value = (seg.videoAttentionStrength ?? 0.65).toFixed(2); + + this.audioInfoArea.style.display = "none"; + this.motionInfoArea.style.display = "none"; + } else { + if (this.segmentPromptLabel) { + this.segmentPromptLabel.style.display = "block"; + this.segmentPromptLabel.textContent = "Segment Prompt"; + } + if (this.globalPromptInput) { + this.globalPromptInput.disabled = false; + this.globalPromptInput.style.opacity = "1.0"; + } + this.audioInfoArea.style.display = "none"; + this.motionInfoArea.style.display = "none"; + if (this.promptWrapper) this.promptWrapper.style.display = "block"; + this.strengthRow.style.display = "flex"; + this.strengthLabel.style.display = "inline"; + this.strengthLabel.textContent = "Guide Strength:"; + this.strengthValue.style.display = "inline-block"; + this.vidStrLabel.style.display = "none"; + this.vidStrValue.style.display = "none"; + this.vidAttnLabel.style.display = "none"; + this.vidAttnValue.style.display = "none"; if (seg) { - this.promptInput.value = seg.prompt || ""; + if (this.selectionType !== "motion") { + this.promptInput.value = seg.prompt || ""; + this.promptInput.placeholder = "Enter prompt for selected segment..."; + } this.promptInput.disabled = false; + this.promptInput.style.opacity = "1.0"; - const isImage = seg.type !== "text"; + const isImage = (this.selectionType === "image") && (seg.type === "image" || seg.type === "video"); const strength = isImage ? (seg.guideStrength ?? 1.0) : 1.0; this.strengthValue.value = strength.toFixed(2); this.strengthValue.disabled = !isImage; + this.strengthValue.style.opacity = isImage ? "1.0" : "0.35"; } else { this.promptInput.value = ""; + this.promptInput.placeholder = "No segment selected!"; this.promptInput.disabled = true; + this.promptInput.style.opacity = "0.4"; this.strengthValue.value = "1.00"; this.strengthValue.disabled = true; + this.strengthValue.style.opacity = "0.35"; } } - if (this.segmentBoundsDisplay) { + if (this.segmentBoundsDisplay && !this.retakeMode) { if (seg) { const startStr = this.formatTime(seg.start, true); const endStr = this.formatTime(seg.start + seg.length, true); - this.segmentBoundsDisplay.textContent = `Start: ${startStr} | End: ${endStr}`; + const lengthStr = this.formatTime(seg.length, true); + this.segmentBoundsDisplay.textContent = `Start: ${startStr} | End: ${endStr} | Length: ${lengthStr}`; } else { - this.segmentBoundsDisplay.textContent = "Start: - | End: -"; + this.segmentBoundsDisplay.textContent = "Start: - | End: - | Length: -"; } } } + + updateRetakeUIState() { + const isRetake = this.retakeMode; + + if (this.globalPromptInput) { + const p = isRetake ? (this.timeline.retake_global_prompt || "") : (this.timeline.global_prompt || ""); + if (this.globalPromptInput.value !== p) { + this.globalPromptInput.value = p; + this.syncGlobalPrompt(p); + } + } + + // 1. Set track heights + if (isRetake) { + if (this.blockHeight > 0 && this.audioTrackHeight > 0) { + this._oldBlockHeight = this.blockHeight; + this._oldAudioTrackHeight = this.audioTrackHeight; + this._oldMotionTrackHeight = this.motionTrackHeight; + } + this.blockHeight = this.canvasHeight - this.rulerHeight; + this.audioTrackHeight = 0; + this.motionTrackHeight = 0; + // In retake mode, uploadVideoBtn stays as "Add Video" (same as normal mode) + if (this.mainTrackLabel) { + const textSpan = this.mainTrackLabel.querySelector("span"); + if (textSpan) textSpan.textContent = "VIDEO"; + if (this.mainTrackLabel._eyeBtn) this.mainTrackLabel._eyeBtn.style.display = "none"; + this.mainTrackLabel.style.backgroundColor = "#1e1e1e"; + this.audioTrackLabel.style.display = "none"; + this.motionTrackLabel.style.display = "none"; + } + if (this.sidebar) this.sidebar.style.backgroundColor = "#1e1e1e"; + if (this.rulerSpacer) this.rulerSpacer.style.backgroundColor = "#1e1e1e"; + } else { + this.blockHeight = this._oldBlockHeight ?? BLOCK_HEIGHT; + this.audioTrackHeight = this._oldAudioTrackHeight ?? AUDIO_TRACK_HEIGHT; + this.motionTrackHeight = this._oldMotionTrackHeight ?? MOTION_TRACK_HEIGHT; + if (this.uploadVideoBtn) { + this.uploadVideoBtn.innerHTML = ` Add Video`; + } + if (this.mainTrackLabel) { + const textSpan = this.mainTrackLabel.querySelector("span"); + if (textSpan) textSpan.textContent = "MAIN"; + if (this.mainTrackLabel._eyeBtn) this.mainTrackLabel._eyeBtn.style.display = "inline-flex"; + this.mainTrackLabel.style.backgroundColor = "#1e1e1e"; + this.audioTrackLabel.style.display = "flex"; + this.motionTrackLabel.style.display = "flex"; + } + if (this.sidebar) this.sidebar.style.backgroundColor = "#1e1e1e"; + if (this.rulerSpacer) this.rulerSpacer.style.backgroundColor = "#1e1e1e"; + } + + this.updateSidebarHeights(); + + // Reset zoom to fit viewport when entering retake mode so full video is visible + if (isRetake) { + this.zoomLevel = 1; + if (this.zoomSlider) this.zoomSlider.value = 1; + this.updateZoomSliderMax(); + const vw = this.viewport ? this.viewport.clientWidth : 0; + if (vw > 0) { + this.resizeCanvas(vw); + this._lastWidth = vw; + this._lastZoom = 1; + if (this.viewport) this.viewport.scrollLeft = 0; + } + } + + // 2. Hide/show toolbar action buttons + if (this.uploadBtn) this.uploadBtn.style.display = isRetake ? "none" : ""; + if (this.addTextBtn) this.addTextBtn.style.display = isRetake ? "none" : ""; + if (this.uploadAudioBtn) this.uploadAudioBtn.style.display = isRetake ? "none" : ""; + if (this.uploadMotionBtn) this.uploadMotionBtn.style.display = isRetake ? "none" : ""; + if (this.deleteBtn) this.deleteBtn.style.display = isRetake ? "none" : ""; + // deleteRetakeBtn is visible whenever Retake Mode is active + if (this.deleteRetakeBtn) { + this.deleteRetakeBtn.style.display = isRetake ? "" : "none"; + } + + // 3. Update the toggle button class/title + if (this.updateRetakeStyle) this.updateRetakeStyle(); + + // 4. Update the prompt labels + if (this.segmentPromptLabel) { + this.segmentPromptLabel.textContent = isRetake ? "Retake Prompt" : "Local Prompt"; + } + + // 5. Update UI selection inputs + this.updateUIFromSelection(); + } + + updateSidebarHeights() { + if (this.mainTrackLabel) { + this.mainTrackLabel.style.height = `${this.blockHeight}px`; + this.audioTrackLabel.style.height = `${this.audioTrackHeight}px`; + this.motionTrackLabel.style.height = `${this.motionTrackHeight}px`; + } + } + // --- Rendering logic --- render() { + if (!this.canvas) return; const width = this.canvas.offsetWidth || this._lastWidth; const height = this.canvasHeight; const totalFrames = this.getVisualDurationFrames(); @@ -1777,14 +5682,45 @@ class TimelineEditor { this.ctx.clearRect(0, 0, width, height); + // Lazy load active video/motion segments + const targetFrame = this.currentFrame; + if (this.retakeMode && this.timeline.retakeVideo) { + this._ensureVideoEl(this.timeline.retakeVideo); + } else { + const activeSeg = this.timeline.segments.find(s => s.type === "video" && targetFrame >= s.start && targetFrame < s.start + s.length); + if (activeSeg) this._ensureVideoEl(activeSeg); + if (this.timeline.motionSegments) { + const activeMotionSeg = this.timeline.motionSegments.find(s => s.type === "motion_video" && targetFrame >= s.start && targetFrame < s.start + s.length); + if (activeMotionSeg) this._ensureVideoEl(activeMotionSeg); + } + } + + if (this.selectedIndex !== -1) { + const selSeg = this.getSegmentArray(this.selectionType)[this.selectedIndex]; + if (selSeg && (selSeg.type === "video" || selSeg.type === "motion_video")) { + this._ensureVideoEl(selSeg); + } + } + + if (this._isDragging && this._dragTargetId) { + const dragSeg = this.timeline.segments.find(s => s.id === this._dragTargetId) || + (this.timeline.motionSegments && this.timeline.motionSegments.find(s => s.id === this._dragTargetId)); + if (dragSeg && (dragSeg.type === "video" || dragSeg.type === "motion_video")) { + this._ensureVideoEl(dragSeg); + } + } // Render Track Backgrounds - this.ctx.fillStyle = "#111"; // Image track bg + this.ctx.fillStyle = "#121212"; // Image track bg this.ctx.fillRect(0, RULER_HEIGHT, width, this.blockHeight); - this.ctx.fillStyle = "#111"; // Audio track bg + + this.ctx.fillStyle = "#141414"; // Audio track bg this.ctx.fillRect(0, RULER_HEIGHT + this.blockHeight, width, this.audioTrackHeight); + this.ctx.fillStyle = "#121212"; // Motion track bg + this.ctx.fillRect(0, RULER_HEIGHT + this.blockHeight + this.audioTrackHeight, width, this.motionTrackHeight); + // Determine which track the preview belongs to. @@ -1792,239 +5728,999 @@ class TimelineEditor { // During canvas mouse drags, _ghostTrack is null, so fall back to selectionType. const previewIsAudio = this._ghostTrack === 'audio' || (this._previewSegments && this._ghostTrack === null && this.selectionType === 'audio'); + const previewIsMotion = this._ghostTrack === 'motion' || + (this._previewSegments && this._ghostTrack === null && this.selectionType === 'motion'); + const previewIsImage = !previewIsAudio && !previewIsMotion; - let renderSegments = (this._previewSegments && !previewIsAudio) - ? this._previewSegments : this.timeline.segments; + let renderSegments = this.timeline.segments; + let renderAudioSegments = this.timeline.audioSegments; + let renderMotionSegments = this.timeline.motionSegments; - let renderAudioSegments = (this._previewSegments && previewIsAudio) - ? this._previewSegments : this.timeline.audioSegments; + if (this._isDragging && this._multiDragPreviewTimelines) { + if (this._multiDragPreviewTimelines.image) renderSegments = this._multiDragPreviewTimelines.image; + if (this._multiDragPreviewTimelines.motion) renderMotionSegments = this._multiDragPreviewTimelines.motion; + if (this._multiDragPreviewTimelines.audio) renderAudioSegments = this._multiDragPreviewTimelines.audio; + } else { + const previewIsAudio = this._ghostTrack === 'audio' || + (this._previewSegments && this._ghostTrack === null && this.selectionType === 'audio'); + const previewIsMotion = this._ghostTrack === 'motion' || + (this._previewSegments && this._ghostTrack === null && this.selectionType === 'motion'); + const previewIsImage = !previewIsAudio && !previewIsMotion; + if (this._previewSegments && previewIsImage) renderSegments = this._previewSegments; + else if (this._previewSiblingSegments && previewIsAudio) renderSegments = this._previewSiblingSegments; + if (this._previewSegments && previewIsAudio) renderAudioSegments = this._previewSegments; + else if (this._previewSiblingSegments && previewIsImage) renderAudioSegments = this._previewSiblingSegments; - const activeSegId = this.timeline.segments[this.selectedIndex]?.id; - const activeAudioSegId = this.timeline.audioSegments[this.selectedIndex]?.id; - - // Sort segments so that the selected one is drawn last (on top) - const isImageSelection = this.selectionType === "image"; - const sortedSegments = [...renderSegments].sort((a, b) => { - const aSel = isImageSelection && a.id === activeSegId; - const bSel = isImageSelection && b.id === activeSegId; - return aSel - bSel; - }); - - const isAudioSelection = this.selectionType === "audio"; - const sortedAudioSegments = [...renderAudioSegments].sort((a, b) => { - const aSel = isAudioSelection && a.id === activeAudioSegId; - const bSel = isAudioSelection && b.id === activeAudioSegId; - return aSel - bSel; - }); - - // --- Draw Image/Text Segments --- - for (let i = 0; i < sortedSegments.length; i++) { - const seg = sortedSegments[i]; - const startX = (seg.start / totalFrames) * width; - const pxWidth = (seg.length / totalFrames) * width; - const isSelected = (this.selectionType === "image" && seg.id === activeSegId); - - const originalSeg = this.timeline.segments.find(s => s.id === seg.id); - const imgObj = originalSeg ? originalSeg.imgObj : seg.imgObj; - - if ((this._isDragging && this.selectionType === "image" && seg.id === this._dragTargetId) || (this._ghostSegmentId && seg.id === this._ghostSegmentId)) { - this.ctx.globalAlpha = 0.65; - } else { - this.ctx.globalAlpha = 1.0; - } - - if (seg.type === "ghost") { - this.ctx.fillStyle = "#2a2a2a"; - this.ctx.fillRect(startX, RULER_HEIGHT, pxWidth, this.blockHeight); - - this.ctx.strokeStyle = "#777"; - this.ctx.lineWidth = 2; - this.ctx.setLineDash([5, 5]); - this.ctx.strokeRect(startX, RULER_HEIGHT + 1, pxWidth, this.blockHeight - 2); - this.ctx.setLineDash([]); - - this.ctx.fillStyle = "#aaa"; - this.ctx.textAlign = "center"; - this.ctx.textBaseline = "middle"; - this.ctx.font = "bold 12px sans-serif"; - this.ctx.fillText("Drop to Place", startX + pxWidth / 2, RULER_HEIGHT + this.blockHeight / 2); - } else { - this.ctx.fillStyle = seg.type === "text" ? "#000b12" : "#000"; - this.ctx.fillRect(startX, RULER_HEIGHT + 1, pxWidth, this.blockHeight - 2); - } - - if (imgObj && imgObj.complete && imgObj.naturalWidth > 0 && seg.type !== "ghost") { - const imgRatio = imgObj.naturalWidth / imgObj.naturalHeight; - const boxRatio = pxWidth / this.blockHeight; - let drawW, drawH, drawX, drawY; - if (imgRatio > boxRatio) { - drawW = pxWidth; drawH = pxWidth / imgRatio; - drawX = startX; drawY = RULER_HEIGHT + (this.blockHeight - drawH) / 2; - } else { - drawH = this.blockHeight; drawW = this.blockHeight * imgRatio; - drawY = RULER_HEIGHT; drawX = startX + (pxWidth - drawW) / 2; - } - - // Clip to segment bounds so tiled images don't bleed into adjacent segments - this.ctx.save(); - this.ctx.beginPath(); - this.ctx.rect(startX, RULER_HEIGHT + 1, pxWidth, this.blockHeight - 2); - this.ctx.clip(); - - if (imgRatio > boxRatio) { - // Fits width, vertical letterboxing (black bars top/bottom) — keep as is - this.ctx.drawImage(imgObj, drawX, drawY, drawW, drawH); - } else { - // Fits height, horizontal letterboxing (black bars left/right) — tile horizontally - this.ctx.drawImage(imgObj, drawX, drawY, drawW, drawH); - - // Tile left - let leftX = drawX - drawW; - while (leftX + drawW > startX) { - this.ctx.drawImage(imgObj, leftX, drawY, drawW, drawH); - leftX -= drawW; - } - - // Tile right - let rightX = drawX + drawW; - while (rightX < startX + pxWidth) { - this.ctx.drawImage(imgObj, rightX, drawY, drawW, drawH); - rightX += drawW; - } - } - this.ctx.restore(); - - // --- Prompt subtitle overlay --- - if (seg.prompt && seg.type !== "ghost" && pxWidth > 24) { - const overlayH = Math.round(this.blockHeight * 0.20); - const overlayY = RULER_HEIGHT + this.blockHeight - overlayH; - - this.ctx.save(); - this.ctx.beginPath(); - this.ctx.rect(startX, overlayY, pxWidth, overlayH); - this.ctx.clip(); - - // Translucent background - this.ctx.fillStyle = "rgba(0, 0, 0, 0.60)"; - this.ctx.fillRect(startX, overlayY, pxWidth, overlayH); - - // Text - const fontSize = Math.min(11, overlayH * 0.58); - this.ctx.font = `${fontSize}px sans-serif`; - this.ctx.fillStyle = "#e0e3ed"; - this.ctx.textAlign = "center"; - this.ctx.textBaseline = "middle"; - - // Measure and truncate to single line - const maxTextW = pxWidth - 10; - let label = seg.prompt; - if (this.ctx.measureText(label).width > maxTextW) { - while (label.length > 0 && this.ctx.measureText(label + "…").width > maxTextW) { - label = label.slice(0, -1); - } - label += "…"; - } - - this.ctx.fillText(label, startX + pxWidth / 2, overlayY + overlayH / 2); - this.ctx.restore(); - } - } else if (seg.type === "text") { - const pad = 8; - const boxW = pxWidth - pad * 2; - if (boxW > 12) { - this.ctx.save(); - this.ctx.beginPath(); - this.ctx.rect(startX + pad, RULER_HEIGHT + pad, boxW, this.blockHeight - pad * 2); - this.ctx.clip(); - this.ctx.fillStyle = "#e0e3ed"; - this.ctx.font = "11px sans-serif"; - this.ctx.textAlign = "center"; - this.ctx.textBaseline = "top"; - const label = seg.prompt || "(no prompt)"; - const words = label.split(" "); - const lineH = 15; - let line = ""; - let lines = []; - for (const word of words) { - const test = line ? line + " " + word : word; - if (this.ctx.measureText(test).width > boxW && line) { - lines.push(line); - line = word; - } else { - line = test; - } - } - if (line) lines.push(line); - - const maxLines = Math.max(1, Math.floor((this.blockHeight - pad * 2) / lineH)); - if (lines.length > maxLines) { - lines = lines.slice(0, maxLines); - lines[lines.length - 1] += "…"; - } - - const totalTextHeight = lines.length * lineH; - let ty = RULER_HEIGHT + (this.blockHeight - totalTextHeight) / 2 + 2; - - for (const l of lines) { - this.ctx.fillText(l, startX + pxWidth / 2, ty); - ty += lineH; - } - this.ctx.restore(); - } - } - - if (isSelected) { - this.ctx.strokeStyle = "#fff"; - this.ctx.lineWidth = 2; - this.ctx.strokeRect(startX, RULER_HEIGHT + 1, pxWidth, this.blockHeight - 2); - this.ctx.fillStyle = "#fff"; - this.ctx.beginPath(); - this.ctx.roundRect(startX, RULER_HEIGHT + this.blockHeight / 2 - 12, 4, 24, 2); - this.ctx.fill(); - this.ctx.beginPath(); - this.ctx.roundRect(startX + pxWidth - 4, RULER_HEIGHT + this.blockHeight / 2 - 12, 4, 24, 2); - this.ctx.fill(); - } else { - this.ctx.strokeStyle = "#000"; - this.ctx.lineWidth = 1.5; - this.ctx.strokeRect(startX, RULER_HEIGHT + 1, pxWidth, this.blockHeight - 2); - } - this.ctx.globalAlpha = 1.0; + if (this._previewSegments && previewIsMotion) renderMotionSegments = this._previewSegments; } - // --- Draw Audio Segments --- - for (let i = 0; i < sortedAudioSegments.length; i++) { - const seg = sortedAudioSegments[i]; - const startX = (seg.start / totalFrames) * width; - const pxWidth = (seg.length / totalFrames) * width; - const isSelected = (this.selectionType === "audio" && seg.id === activeAudioSegId); - const trackY = RULER_HEIGHT + this.blockHeight; + const sortedSegments = [...renderSegments].sort((a, b) => { + const aSel = this.selectedSegmentIds.includes(a.id) ? 1 : 0; + const bSel = this.selectedSegmentIds.includes(b.id) ? 1 : 0; + return aSel - bSel; + }); + + const sortedMotionSegments = [...renderMotionSegments].sort((a, b) => { + const aSel = this.selectedSegmentIds.includes(a.id) ? 1 : 0; + const bSel = this.selectedSegmentIds.includes(b.id) ? 1 : 0; + return aSel - bSel; + }); + + const sortedAudioSegments = [...renderAudioSegments].sort((a, b) => { + const aSel = this.selectedSegmentIds.includes(a.id) ? 1 : 0; + const bSel = this.selectedSegmentIds.includes(b.id) ? 1 : 0; + return aSel - bSel; + }); + + if (this.retakeMode) { + // Draw Retake Mode Filmstrip and Overlay + const retakeVid = this.timeline.retakeVideo; + const frameRate = this.getFrameRate(); + if (retakeVid) { + const showLivePreview = this.isPlaying || (this._isDragging && (this._dragType === "playhead" || this._dragType === "retake_left" || this._dragType === "retake_right" || this._dragType === "retake_center")); + + // Calculate the actual visual width of the base video block + const baseVideoDur = retakeVid.videoDurationFrames || 0; + const videoWidthPx = totalFrames > 0 ? (baseVideoDur / totalFrames) * width : width; + + if (showLivePreview) { + let targetTime = this.currentFrame / frameRate; + if (this._isDragging) { + if (this._dragType === "retake_left") { + targetTime = (this.timeline.retakeStart ?? 0) / frameRate; + } else if (this._dragType === "retake_right") { + targetTime = ((this.timeline.retakeStart ?? 0) + (this.timeline.retakeLength ?? baseVideoDur)) / frameRate; + } else if (this._dragType === "retake_center") { + targetTime = (this.timeline.retakeStart ?? 0) / frameRate; + } + } + + let drawSource = null; + const useLiveVideo = this.isPlaying || (this._isDragging ? this._dragType !== "playhead" : true); + if (useLiveVideo && retakeVid.videoEl && retakeVid.videoEl.readyState >= 2 && !retakeVid.videoEl.seeking) { + drawSource = retakeVid.videoEl; + } else if (retakeVid.thumbnails && retakeVid.thumbnails.length > 0) { + let nearestImg = retakeVid.thumbnails[0].img; + let minDiff = Infinity; + for (const t of retakeVid.thumbnails) { + const diff = Math.abs(t.time - targetTime); + if (diff < minDiff) { + minDiff = diff; + nearestImg = t.img; + } + } + drawSource = nearestImg; + } else { + drawSource = retakeVid.videoEl || (retakeVid.imgObj && retakeVid.imgObj.complete ? retakeVid.imgObj : null); + } + + this.ctx.fillStyle = "#000"; + this.ctx.fillRect(0, RULER_HEIGHT + 1, videoWidthPx, this.blockHeight - 2); + + if (drawSource) { + const isVid = !!drawSource.videoWidth; + const natW = isVid ? drawSource.videoWidth : drawSource.naturalWidth; + const natH = isVid ? drawSource.videoHeight : drawSource.naturalHeight; + + if (natW > 0) { + const imgRatio = natW / natH; + const trackRatio = videoWidthPx / this.blockHeight; + let drawW, drawH, drawX, drawY; + + if (imgRatio > trackRatio) { + drawW = videoWidthPx; + drawH = videoWidthPx / imgRatio; + drawX = 0; + drawY = RULER_HEIGHT + (this.blockHeight - drawH) / 2; + + this.ctx.save(); + this.ctx.beginPath(); + this.ctx.rect(0, RULER_HEIGHT + 1, videoWidthPx, this.blockHeight - 2); + this.ctx.clip(); + this.ctx.drawImage(drawSource, drawX, drawY, drawW, drawH); + this.ctx.restore(); + } else { + drawH = this.blockHeight; + drawW = this.blockHeight * imgRatio; + drawY = RULER_HEIGHT; + drawX = (videoWidthPx - drawW) / 2; + + this.ctx.save(); + this.ctx.beginPath(); + this.ctx.rect(0, RULER_HEIGHT + 1, videoWidthPx, this.blockHeight - 2); + this.ctx.clip(); + + // Draw centered preview frame + this.ctx.drawImage(drawSource, drawX, drawY, drawW, drawH); + + // Tile to the left + let leftX = drawX - drawW; + while (leftX + drawW > 0) { + this.ctx.drawImage(drawSource, leftX, drawY, drawW, drawH); + leftX -= drawW; + } + + // Tile to the right + let rightX = drawX + drawW; + while (rightX < videoWidthPx) { + this.ctx.drawImage(drawSource, rightX, drawY, drawW, drawH); + rightX += drawW; + } + + this.ctx.restore(); + } + } + } + } else { + // Static state: pick the midpoint thumbnail and tile it at its natural aspect ratio, + // matching the visual appearance of the live-preview path. + const durationSecs = baseVideoDur / frameRate; + const midTime = durationSecs / 2; + + let drawSource = null; + if (retakeVid.thumbnails && retakeVid.thumbnails.length > 0) { + let nearestImg = retakeVid.thumbnails[0].img; + let minDiff = Infinity; + for (const t of retakeVid.thumbnails) { + const diff = Math.abs(t.time - midTime); + if (diff < minDiff) { + minDiff = diff; + nearestImg = t.img; + } + } + drawSource = nearestImg; + } else { + drawSource = retakeVid.imgObj && retakeVid.imgObj.complete ? retakeVid.imgObj : null; + } + + this.ctx.fillStyle = "#000"; + this.ctx.fillRect(0, RULER_HEIGHT + 1, videoWidthPx, this.blockHeight - 2); + + if (drawSource) { + const isVid = !!drawSource.videoWidth; + const natW = isVid ? drawSource.videoWidth : drawSource.naturalWidth; + const natH = isVid ? drawSource.videoHeight : drawSource.naturalHeight; + + if (natW > 0) { + const imgRatio = natW / natH; + const trackRatio = videoWidthPx / this.blockHeight; + + this.ctx.save(); + this.ctx.beginPath(); + this.ctx.rect(0, RULER_HEIGHT + 1, videoWidthPx, this.blockHeight - 2); + this.ctx.clip(); + + if (imgRatio > trackRatio) { + // Video is wider than the track: fill width, letterbox top/bottom + const drawW = videoWidthPx; + const drawH = videoWidthPx / imgRatio; + const drawY = RULER_HEIGHT + (this.blockHeight - drawH) / 2; + this.ctx.drawImage(drawSource, 0, drawY, drawW, drawH); + } else { + // Video is taller/square: fill height and tile left+right at natural AR + const drawH = this.blockHeight; + const drawW = drawH * imgRatio; + const drawX = (videoWidthPx - drawW) / 2; + const drawY = RULER_HEIGHT; + // Draw centered tile + this.ctx.drawImage(drawSource, drawX, drawY, drawW, drawH); + // Tile to the left + let leftX = drawX - drawW; + while (leftX + drawW > 0) { + this.ctx.drawImage(drawSource, leftX, drawY, drawW, drawH); + leftX -= drawW; + } + // Tile to the right + let rightX = drawX + drawW; + while (rightX < videoWidthPx) { + this.ctx.drawImage(drawSource, rightX, drawY, drawW, drawH); + rightX += drawW; + } + } + + this.ctx.restore(); + } + } + } + + + if (retakeVid._uploading || retakeVid._extractingThumbs) { + this.ctx.save(); + this.ctx.fillStyle = "rgba(0, 14, 37, 0.8)"; + const upText = retakeVid._extractingThumbs ? "Extracting frames..." : "Uploading base video..."; + this.ctx.font = "bold 11px sans-serif"; + const upW = this.ctx.measureText(upText).width + 20; + this.ctx.fillRect(10, RULER_HEIGHT + 35, upW, 24); + this.ctx.fillStyle = "#fff"; + this.ctx.textBaseline = "middle"; + this.ctx.textAlign = "center"; + this.ctx.fillText(upText, 10 + upW / 2, RULER_HEIGHT + 47); + this.ctx.restore(); + } - if ((this._isDragging && this.selectionType === "audio" && seg.id === this._dragTargetId) || (this._ghostSegmentId && seg.id === this._ghostSegmentId)) { - this.ctx.globalAlpha = 0.65; } else { - this.ctx.globalAlpha = 1.0; - } + // No video loaded: Render a placeholder box with upload instructions centered on active timeline + this.ctx.fillStyle = "#121212"; + this.ctx.fillRect(0, RULER_HEIGHT + 1, width, this.blockHeight - 2); + + // In retake mode, center the placeholder across the visible viewport + const activeStart = this.viewport ? this.viewport.scrollLeft : 0; + let activeWidth = this.viewport ? this.viewport.clientWidth : width; + // The right ~9% of the DOM is clipped, so squish the box to visually center it in the unclipped area + activeWidth = activeWidth * 0.91; - if (seg.type === "ghost") { - this.ctx.fillStyle = "#1a1a1a"; - this.ctx.fillRect(startX, trackY, pxWidth, this.audioTrackHeight); this.ctx.strokeStyle = "#555"; this.ctx.lineWidth = 2; - this.ctx.setLineDash([5, 5]); - this.ctx.strokeRect(startX, trackY, pxWidth, this.audioTrackHeight); + this.ctx.setLineDash([6, 6]); + this.ctx.strokeRect(activeStart + 12, RULER_HEIGHT + 12, Math.max(10, activeWidth - 24), this.blockHeight - 24); this.ctx.setLineDash([]); + this.ctx.fillStyle = "#888"; this.ctx.textAlign = "center"; this.ctx.textBaseline = "middle"; - this.ctx.font = "bold 12px sans-serif"; - this.ctx.fillText("Drop Audio", startX + pxWidth / 2, trackY + this.audioTrackHeight / 2); - } else { - this.drawAudioSegmentVisuals(this.ctx, seg, isSelected, trackY, this.audioTrackHeight, startX, pxWidth); + this.ctx.font = "14px sans-serif"; + this.ctx.fillText("Drag & Drop or Click to Add a Video", activeStart + activeWidth / 2, RULER_HEIGHT + this.blockHeight / 2); + } + + // Only draw the retake region overlay, borders, handles, and label if a video is loaded + if (this.timeline.retakeVideo) { + // Draw the white outline retake region overlay box bounded by retakeStart and retakeLength. + // Tint outside the box (locked/preserved regions) with a dark blue-grey tint overlay (rgba(3, 5, 12, 0.75)). + const retakeStart = this.timeline.retakeStart ?? 0; + const retakeLength = this.timeline.retakeLength ?? totalFrames; + + const baseVideoDur = this.timeline.retakeVideo.videoDurationFrames || 0; + const videoWidthPx = totalFrames > 0 ? (baseVideoDur / totalFrames) * width : width; + + const rX1 = (retakeStart / totalFrames) * width; + const rX2 = ((retakeStart + retakeLength) / totalFrames) * width; + + // Tint preserved left region + if (rX1 > 0) { + this.ctx.fillStyle = "rgba(0, 0, 0, 0.70)"; + this.ctx.fillRect(0, RULER_HEIGHT + 1, rX1, this.blockHeight - 2); + } + + // Tint preserved right region (only up to videoWidthPx, not the padding zone) + if (rX2 < videoWidthPx) { + this.ctx.fillStyle = "rgba(0, 0, 0, 0.70)"; + this.ctx.fillRect(rX2, RULER_HEIGHT + 1, videoWidthPx - rX2, this.blockHeight - 2); + } + + // Draw the Retake Overlay Box + const boxW = rX2 - rX1; + + // White border + this.ctx.strokeStyle = "#ffffff"; + this.ctx.lineWidth = 2.5; + this.ctx.strokeRect(rX1, RULER_HEIGHT + 1, boxW, this.blockHeight - 2); + + // Draw handles on the left and right edges + this.ctx.fillStyle = "#ffffff"; + this.ctx.beginPath(); + this.ctx.roundRect(rX1 - 3, RULER_HEIGHT + this.blockHeight / 2 - 20, 6, 40, 3); + this.ctx.fill(); + + this.ctx.beginPath(); + this.ctx.roundRect(rX2 - 3, RULER_HEIGHT + this.blockHeight / 2 - 20, 6, 40, 3); + this.ctx.fill(); + + // Draw "RETAKE REGION" centered label inside the retake box + { + const labelPadX = 14; + const labelPadY = 7; + const labelFontSize = 15; + const labelText = "RETAKE REGION"; + const labelY = RULER_HEIGHT + this.blockHeight - labelFontSize - labelPadY * 2 - 4; + const labelCenterX = rX1 + boxW / 2; + + this.ctx.save(); + // Clip to retake region so text/bg never bleeds outside + this.ctx.beginPath(); + this.ctx.rect(rX1, RULER_HEIGHT, boxW, this.blockHeight); + this.ctx.clip(); + + this.ctx.font = `bold ${labelFontSize}px sans-serif`; + this.ctx.textAlign = "center"; + this.ctx.textBaseline = "middle"; + + // Truncate if too narrow + let displayText = labelText; + const maxTextW = Math.max(0, boxW - labelPadX * 2 - 8); + if (this.ctx.measureText(displayText).width > maxTextW) { + while (displayText.length > 0 && this.ctx.measureText(displayText + "…").width > maxTextW) { + displayText = displayText.slice(0, -1); + } + displayText = displayText.length > 0 ? displayText + "…" : ""; + } + + if (displayText.length > 0) { + const textW = this.ctx.measureText(displayText).width; + const bgW = textW + labelPadX * 2; + const bgH = labelFontSize + labelPadY * 2; + const bgX = labelCenterX - bgW / 2; + const bgY = labelY - bgH / 2; + + // Background pill + this.ctx.fillStyle = "rgba(0, 0, 0, 0.62)"; + this.ctx.beginPath(); + this.ctx.roundRect(bgX, bgY, bgW, bgH, 3); + this.ctx.fill(); + + // Label text + this.ctx.fillStyle = "#ffffff"; + this.ctx.fillText(displayText, labelCenterX, labelY); + } + this.ctx.restore(); + } + + // Show video info badge / filename (styled exactly like a regular video segment, drawn on top of overlays) + this.ctx.save(); + this.ctx.beginPath(); + this.ctx.rect(0, RULER_HEIGHT, videoWidthPx, this.blockHeight); + this.ctx.clip(); + + // 1. Draw the "VIDEO" label badge + this.ctx.fillStyle = "rgba(0, 0, 0, 0.60)"; + this.ctx.fillRect(0, RULER_HEIGHT + 1, 42, 16); + this.ctx.fillStyle = "#fff"; + this.ctx.font = "bold 10px sans-serif"; + this.ctx.textAlign = "center"; + this.ctx.textBaseline = "middle"; + this.ctx.fillText("VIDEO", 21, RULER_HEIGHT + 9); + + // 2. Draw the filename badge + if (this.node.properties.showFilenames && videoWidthPx > 46) { + let rawPath = retakeVid.imageFile || retakeVid.fileName || ""; + let fname = rawPath.split(/[/\\]/).pop() || ""; + this.ctx.font = "9px sans-serif"; + this.ctx.textAlign = "left"; + this.ctx.textBaseline = "middle"; + const maxFileTextW = videoWidthPx - 42 - 10; + if (this.ctx.measureText(fname).width > maxFileTextW) { + while (fname.length > 0 && this.ctx.measureText(fname + "…").width > maxFileTextW) { + fname = fname.slice(0, -1); + } + fname += "…"; + } + const textW = this.ctx.measureText(fname).width; + this.ctx.fillStyle = "rgba(0, 0, 0, 0.50)"; + this.ctx.fillRect(43, RULER_HEIGHT + 1, textW + 8, 16); + this.ctx.fillStyle = "#fff"; + this.ctx.fillText(fname, 47, RULER_HEIGHT + 9); + } + this.ctx.restore(); + + } + } else { + // --- Draw Image/Text Segments --- + for (let i = 0; i < sortedSegments.length; i++) { + const seg = sortedSegments[i]; + const rawStartX = (seg.start / totalFrames) * width; + const rawEndX = ((seg.start + seg.length) / totalFrames) * width; + const startX = Math.floor(rawStartX); + const pxWidth = Math.max(1, Math.floor(rawEndX) - startX); + const isSelected = this.selectedSegmentIds.includes(seg.id); + + const originalSeg = this.timeline.segments.find(s => s.id === seg.id); + const imgObj = originalSeg ? originalSeg.imgObj : seg.imgObj; + const videoEl = originalSeg ? originalSeg.videoEl : seg.videoEl; + + const isPlayheadOverSeg = (this.currentFrame >= seg.start && this.currentFrame < seg.start + seg.length); + const isScrubbingThis = this._isDragging && (this._dragTargetId === seg.id || this._dragTargetIdRight === seg.id); + const isLiveActive = this.isPlaying && isPlayheadOverSeg; + + if ((this._isDragging && this.selectionType === "image" && seg.id === this._dragTargetId) || (this._ghostSegmentId && seg.id === this._ghostSegmentId)) { + this.ctx.globalAlpha = 0.65; + } else { + this.ctx.globalAlpha = 1.0; + } + + if (seg.type === "ghost") { + this.ctx.fillStyle = "#2a2a2a"; + this.ctx.fillRect(startX, RULER_HEIGHT, pxWidth, this.blockHeight); + + this.ctx.strokeStyle = "#777"; + this.ctx.lineWidth = 2; + this.ctx.setLineDash([5, 5]); + this.ctx.strokeRect(startX, RULER_HEIGHT + 1, pxWidth, this.blockHeight - 2); + this.ctx.setLineDash([]); + + this.ctx.fillStyle = "#aaa"; + this.ctx.textAlign = "center"; + this.ctx.textBaseline = "middle"; + this.ctx.font = "bold 12px sans-serif"; + this.ctx.fillText("Drop to Place", startX + pxWidth / 2, RULER_HEIGHT + this.blockHeight / 2); + } else { + this.ctx.fillStyle = seg.type === "text" ? "#000b12" : "#000"; + this.ctx.fillRect(startX, RULER_HEIGHT + 1, pxWidth, this.blockHeight - 2); + } + + let drawSource = null; + if (isLiveActive && videoEl && videoEl.readyState >= 2) { + drawSource = videoEl; + } else { + if (seg.type === "video" && seg.thumbnails && seg.thumbnails.length > 0) { + const targetTime = seg._scrubTargetSec !== undefined + ? seg._scrubTargetSec + : (isPlayheadOverSeg ? (this.currentFrame - seg.start + seg.trimStart) / this.getFrameRate() : seg.trimStart / this.getFrameRate()); + let nearestImg = seg.thumbnails[0].img; + let minDiff = Infinity; + for (const t of seg.thumbnails) { + const diff = Math.abs(t.time - targetTime); + if (diff < minDiff) { + minDiff = diff; + nearestImg = t.img; + } + } + drawSource = nearestImg; + } else { + drawSource = imgObj && imgObj.complete ? imgObj : null; + } + } + + if (drawSource && seg.type !== "ghost") { + const isVid = !!drawSource.videoWidth; + const natW = isVid ? drawSource.videoWidth : drawSource.naturalWidth; + const natH = isVid ? drawSource.videoHeight : drawSource.naturalHeight; + + if (natW > 0) { + const imgRatio = natW / natH; + const boxRatio = pxWidth / this.blockHeight; + let drawW, drawH, drawX, drawY; + if (imgRatio > boxRatio) { + drawW = pxWidth; drawH = pxWidth / imgRatio; + drawX = startX; drawY = RULER_HEIGHT + (this.blockHeight - drawH) / 2; + } else { + drawH = this.blockHeight; drawW = this.blockHeight * imgRatio; + drawY = RULER_HEIGHT; drawX = startX + (pxWidth - drawW) / 2; + } + + // Clip to segment bounds so tiled images don't bleed into adjacent segments + this.ctx.save(); + this.ctx.beginPath(); + this.ctx.rect(startX, RULER_HEIGHT + 1, pxWidth, this.blockHeight - 2); + this.ctx.clip(); + + if (imgRatio > boxRatio) { + // Fits width, vertical letterboxing (black bars top/bottom) — keep as is + this.ctx.drawImage(drawSource, drawX, drawY, drawW, drawH); + } else { + // Fits height, horizontal letterboxing (black bars left/right) + this.ctx.drawImage(drawSource, drawX, drawY, drawW, drawH); + + // Tile left + let leftX = drawX - drawW; + while (leftX + drawW > startX) { + this.ctx.drawImage(drawSource, leftX, drawY, drawW, drawH); + leftX -= drawW; + } + // Tile right + let rightX = drawX + drawW; + while (rightX < startX + pxWidth) { + this.ctx.drawImage(drawSource, rightX, drawY, drawW, drawH); + rightX += drawW; + } + } + this.ctx.restore(); + } + } + + if ((seg.type === "video" || drawSource) && seg.type !== "ghost") { + if (seg.type === "video" && pxWidth > 0) { + this.ctx.save(); + this.ctx.beginPath(); + this.ctx.rect(startX, RULER_HEIGHT, pxWidth, this.blockHeight); + this.ctx.clip(); + this.ctx.fillStyle = "rgba(0, 0, 0, 0.60)"; + this.ctx.fillRect(startX, RULER_HEIGHT + 1, 42, 16); + this.ctx.fillStyle = "#fff"; + this.ctx.font = "bold 10px sans-serif"; + this.ctx.textAlign = "center"; + this.ctx.textBaseline = "middle"; + this.ctx.fillText("VIDEO", startX + 21, RULER_HEIGHT + 9); + this.ctx.restore(); + + // Uploading / Loading indicator badge (bottom-left corner) + if ((seg._uploading || seg._extractingThumbs) && pxWidth > 60) { + this.ctx.save(); + this.ctx.beginPath(); + this.ctx.rect(startX, RULER_HEIGHT, pxWidth, this.blockHeight); + this.ctx.clip(); + this.ctx.font = "bold 9px sans-serif"; + const upText = seg._extractingThumbs ? "Loading..." : "Uploading..."; + const upW = this.ctx.measureText(upText).width + 10; + this.ctx.fillStyle = "rgba(0, 14, 37, 0.7)"; + this.ctx.fillRect(startX + 1, RULER_HEIGHT + this.blockHeight - 17, upW, 16); + this.ctx.fillStyle = "#fff"; + this.ctx.textAlign = "center"; + this.ctx.textBaseline = "middle"; + this.ctx.fillText(upText, startX + 1 + upW / 2, RULER_HEIGHT + this.blockHeight - 9); + this.ctx.restore(); + } + + // Filename next to VIDEO tag + if (this.node.properties.showFilenames && pxWidth > 46) { + this.ctx.save(); + this.ctx.beginPath(); + this.ctx.rect(startX, RULER_HEIGHT, pxWidth, this.blockHeight); + this.ctx.clip(); + let rawPath = seg.imageFile || ""; + let fname = rawPath.split(/[/\\]/).pop() || ""; + this.ctx.font = "9px sans-serif"; + this.ctx.textAlign = "left"; + this.ctx.textBaseline = "middle"; + const maxFileTextW = pxWidth - 42 - 10; + if (this.ctx.measureText(fname).width > maxFileTextW) { + while (fname.length > 0 && this.ctx.measureText(fname + "…").width > maxFileTextW) { + fname = fname.slice(0, -1); + } + fname += "…"; + } + const textW = this.ctx.measureText(fname).width; + this.ctx.fillStyle = "rgba(0, 0, 0, 0.50)"; + this.ctx.fillRect(startX + 43, RULER_HEIGHT + 1, textW + 8, 16); + this.ctx.fillStyle = "#fff"; + this.ctx.fillText(fname, startX + 47, RULER_HEIGHT + 9); + this.ctx.restore(); + } + } else if (seg.type === "image" && pxWidth > 0) { + this.ctx.save(); + this.ctx.beginPath(); + this.ctx.rect(startX, RULER_HEIGHT, pxWidth, this.blockHeight); + this.ctx.clip(); + this.ctx.fillStyle = "rgba(0, 0, 0, 0.60)"; + this.ctx.fillRect(startX, RULER_HEIGHT + 1, 42, 16); + this.ctx.fillStyle = "#fff"; + this.ctx.font = "bold 10px sans-serif"; + this.ctx.textAlign = "center"; + this.ctx.textBaseline = "middle"; + this.ctx.fillText("IMAGE", startX + 21, RULER_HEIGHT + 9); + this.ctx.restore(); + + // Filename next to IMAGE tag + if (this.node.properties.showFilenames && pxWidth > 46) { + this.ctx.save(); + this.ctx.beginPath(); + this.ctx.rect(startX, RULER_HEIGHT, pxWidth, this.blockHeight); + this.ctx.clip(); + let rawPath = seg.imageFile || ""; + let fname = rawPath.split(/[/\\]/).pop() || ""; + this.ctx.font = "9px sans-serif"; + this.ctx.textAlign = "left"; + this.ctx.textBaseline = "middle"; + const maxFileTextW = pxWidth - 42 - 10; + if (this.ctx.measureText(fname).width > maxFileTextW) { + while (fname.length > 0 && this.ctx.measureText(fname + "…").width > maxFileTextW) { + fname = fname.slice(0, -1); + } + fname += "…"; + } + const textW = this.ctx.measureText(fname).width; + this.ctx.fillStyle = "rgba(0, 0, 0, 0.50)"; + this.ctx.fillRect(startX + 43, RULER_HEIGHT + 1, textW + 8, 16); + this.ctx.fillStyle = "#fff"; + this.ctx.fillText(fname, startX + 47, RULER_HEIGHT + 9); + this.ctx.restore(); + } + } + + if (seg.type === "image" && seg.isEndFrame && pxWidth > 0) { + this.ctx.save(); + this.ctx.beginPath(); + this.ctx.rect(startX, RULER_HEIGHT, pxWidth, this.blockHeight); + this.ctx.clip(); + + this.ctx.font = "bold 9px sans-serif"; + const badgeText = "END FRAME"; + const badgeTextW = this.ctx.measureText(badgeText).width; + const badgeW = badgeTextW + 10; + const badgeH = 16; + const badgeX = startX + pxWidth - badgeW; + const badgeY = RULER_HEIGHT + 1; + + this.ctx.fillStyle = "rgba(0, 0, 0, 0.60)"; + this.ctx.fillRect(badgeX, badgeY, badgeW, badgeH); + + this.ctx.fillStyle = "#fff"; + this.ctx.textAlign = "center"; + this.ctx.textBaseline = "middle"; + this.ctx.fillText(badgeText, badgeX + badgeW / 2, badgeY + badgeH / 2); + this.ctx.restore(); + } + + // --- Prompt subtitle overlay --- + if (seg.prompt && seg.type !== "ghost" && pxWidth > 24) { + const overlayH = Math.round(this.blockHeight * 0.20); + const overlayY = RULER_HEIGHT + this.blockHeight - overlayH; + + this.ctx.save(); + this.ctx.beginPath(); + this.ctx.rect(startX, overlayY, pxWidth, overlayH); + this.ctx.clip(); + + // Translucent background + this.ctx.fillStyle = "rgba(0, 0, 0, 0.60)"; + this.ctx.fillRect(startX, overlayY, pxWidth, overlayH); + + // Text + const fontSize = Math.min(11, overlayH * 0.58); + this.ctx.font = `${fontSize}px sans-serif`; + this.ctx.fillStyle = "#e0e3ed"; + this.ctx.textAlign = "center"; + this.ctx.textBaseline = "middle"; + + // Measure and truncate to single line + const maxTextW = pxWidth - 10; + let label = seg.prompt; + if (this.ctx.measureText(label).width > maxTextW) { + while (label.length > 0 && this.ctx.measureText(label + "…").width > maxTextW) { + label = label.slice(0, -1); + } + label += "…"; + } + + this.ctx.fillText(label, startX + pxWidth / 2, overlayY + overlayH / 2); + this.ctx.restore(); + } + } else if (seg.type === "text") { + const pad = 8; + const boxW = pxWidth - pad * 2; + if (boxW > 12) { + this.ctx.save(); + this.ctx.beginPath(); + this.ctx.rect(startX + pad, RULER_HEIGHT + pad, boxW, this.blockHeight - pad * 2); + this.ctx.clip(); + this.ctx.fillStyle = "#e0e3ed"; + this.ctx.font = "11px sans-serif"; + this.ctx.textAlign = "center"; + this.ctx.textBaseline = "top"; + const label = seg.prompt || "(no prompt)"; + const words = label.split(" "); + const lineH = 15; + let line = ""; + let lines = []; + for (const word of words) { + const test = line ? line + " " + word : word; + if (this.ctx.measureText(test).width > boxW && line) { + lines.push(line); + line = word; + } else { + line = test; + } + } + if (line) lines.push(line); + + const maxLines = Math.max(1, Math.floor((this.blockHeight - pad * 2) / lineH)); + if (lines.length > maxLines) { + lines = lines.slice(0, maxLines); + lines[lines.length - 1] += "…"; + } + + const totalTextHeight = lines.length * lineH; + let ty = RULER_HEIGHT + (this.blockHeight - totalTextHeight) / 2 + 2; + + for (const l of lines) { + this.ctx.fillText(l, startX + pxWidth / 2, ty); + ty += lineH; + } + this.ctx.restore(); + } + } + + if (isSelected) { + const outlineColor = "#fff"; + this.ctx.strokeStyle = outlineColor; + this.ctx.lineWidth = 2; + this.ctx.strokeRect(startX, RULER_HEIGHT + 1, pxWidth, this.blockHeight - 2); + if (!this.isMultiSelectActive()) { + this.ctx.fillStyle = outlineColor; + this.ctx.beginPath(); + this.ctx.roundRect(startX, RULER_HEIGHT + this.blockHeight / 2 - 12, 4, 24, 2); + this.ctx.fill(); + this.ctx.beginPath(); + this.ctx.roundRect(startX + pxWidth - 4, RULER_HEIGHT + this.blockHeight / 2 - 12, 4, 24, 2); + this.ctx.fill(); + } + } else { + this.ctx.strokeStyle = "#000"; + this.ctx.lineWidth = 1.5; + this.ctx.strokeRect(startX, RULER_HEIGHT + 1, pxWidth, this.blockHeight - 2); + } + this.ctx.globalAlpha = 1.0; + } + + // --- Draw Motion Segments --- + for (let i = 0; i < sortedMotionSegments.length; i++) { + const seg = sortedMotionSegments[i]; + const startX = Math.floor((seg.start / totalFrames) * width); + const rawEndX = ((seg.start + seg.length) / totalFrames) * width; + const pxWidth = Math.max(1, Math.floor(rawEndX) - startX); + const isSelected = this.selectedSegmentIds.includes(seg.id); + const trackY = RULER_HEIGHT + this.blockHeight + this.audioTrackHeight; + + if ((this._isDragging && this.selectionType === "motion" && seg.id === this._dragTargetId) || (this._ghostSegmentId && seg.id === this._ghostSegmentId)) { + this.ctx.globalAlpha = 0.65; + } else { + this.ctx.globalAlpha = 1.0; + } + + if (seg.type === "ghost") { + this.ctx.fillStyle = "#1a1a1a"; + this.ctx.fillRect(startX, trackY, pxWidth, this.motionTrackHeight); + this.ctx.strokeStyle = "#555"; + this.ctx.lineWidth = 2; + this.ctx.setLineDash([5, 5]); + this.ctx.strokeRect(startX, trackY, pxWidth, this.motionTrackHeight); + this.ctx.setLineDash([]); + this.ctx.fillStyle = "#888"; + this.ctx.textAlign = "center"; + this.ctx.textBaseline = "middle"; + this.ctx.font = "bold 12px sans-serif"; + this.ctx.fillText("Drop Motion", startX + pxWidth / 2, trackY + this.motionTrackHeight / 2); + } else { + this.ctx.fillStyle = "#000"; + this.ctx.fillRect(startX, trackY + 1, pxWidth, this.motionTrackHeight - 2); + + const originalSeg = this.timeline.motionSegments.find(s => s.id === seg.id); + const imgObj = originalSeg ? originalSeg.imgObj : seg.imgObj; + const videoEl = originalSeg ? originalSeg.videoEl : seg.videoEl; + + const isPlayheadOverSeg = (this.currentFrame >= seg.start && this.currentFrame < seg.start + seg.length); + const isScrubbingThis = this._isDragging && (this._dragTargetId === seg.id || this._dragTargetIdRight === seg.id); + const isLiveActive = this.isPlaying && isPlayheadOverSeg; + + let drawSource = null; + if (isLiveActive && videoEl && videoEl.readyState >= 2) { + drawSource = videoEl; + } else { + if (seg.type === "motion_video" && seg.thumbnails && seg.thumbnails.length > 0) { + const targetTime = seg._scrubTargetSec !== undefined + ? seg._scrubTargetSec + : (isPlayheadOverSeg ? (this.currentFrame - seg.start + seg.trimStart) / this.getFrameRate() : seg.trimStart / this.getFrameRate()); + let nearestImg = seg.thumbnails[0].img; + let minDiff = Infinity; + for (const t of seg.thumbnails) { + const diff = Math.abs(t.time - targetTime); + if (diff < minDiff) { + minDiff = diff; + nearestImg = t.img; + } + } + drawSource = nearestImg; + } else { + drawSource = imgObj && imgObj.complete ? imgObj : null; + } + } + + if (drawSource && seg.type !== "ghost") { + const natW = drawSource.videoWidth || drawSource.naturalWidth; + const natH = drawSource.videoHeight || drawSource.naturalHeight; + + if (natW > 0) { + const imgRatio = natW / natH; + const boxRatio = pxWidth / this.motionTrackHeight; + let drawW, drawH, drawX, drawY; + if (imgRatio > boxRatio) { + drawW = pxWidth; drawH = pxWidth / imgRatio; + drawX = startX; drawY = trackY + (this.motionTrackHeight - drawH) / 2; + } else { + drawH = this.motionTrackHeight; drawW = this.motionTrackHeight * imgRatio; + drawY = trackY; drawX = startX + (pxWidth - drawW) / 2; + } + + this.ctx.save(); + this.ctx.beginPath(); + this.ctx.rect(startX, trackY + 1, pxWidth, this.motionTrackHeight - 2); + this.ctx.clip(); + + if (imgRatio > boxRatio) { + this.ctx.drawImage(drawSource, drawX, drawY, drawW, drawH); + } else { + this.ctx.drawImage(drawSource, drawX, drawY, drawW, drawH); + let leftX = drawX - drawW; + while (leftX + drawW > startX) { + this.ctx.drawImage(drawSource, leftX, drawY, drawW, drawH); + leftX -= drawW; + } + let rightX = drawX + drawW; + while (rightX < startX + pxWidth) { + this.ctx.drawImage(drawSource, rightX, drawY, drawW, drawH); + rightX += drawW; + } + } + this.ctx.restore(); + } + } + + if (pxWidth > 0 && seg.type !== "ghost") { + this.ctx.save(); + this.ctx.beginPath(); + this.ctx.rect(startX, trackY, pxWidth, this.motionTrackHeight); + this.ctx.clip(); + this.ctx.fillStyle = "rgba(0, 0, 0, 0.60)"; + this.ctx.fillRect(startX, trackY + 1, 75, 16); + this.ctx.fillStyle = "#fff"; + this.ctx.font = "bold 10px sans-serif"; + this.ctx.textAlign = "center"; + this.ctx.textBaseline = "middle"; + this.ctx.fillText("IC-LoRA Video", startX + 37, trackY + 9); + this.ctx.restore(); + + // Uploading / Loading indicator badge (bottom-left corner) + if ((seg._uploading || seg._extractingThumbs) && pxWidth > 60) { + this.ctx.save(); + this.ctx.beginPath(); + this.ctx.rect(startX, trackY, pxWidth, this.motionTrackHeight); + this.ctx.clip(); + this.ctx.font = "bold 9px sans-serif"; + const upText = seg._extractingThumbs ? "Loading..." : "Uploading..."; + const upW = this.ctx.measureText(upText).width + 10; + this.ctx.fillStyle = "rgba(0, 14, 37, 0.7)"; + this.ctx.fillRect(startX + 1, trackY + this.motionTrackHeight - 17, upW, 16); + this.ctx.fillStyle = "#fff"; + this.ctx.textAlign = "center"; + this.ctx.textBaseline = "middle"; + this.ctx.fillText(upText, startX + 1 + upW / 2, trackY + this.motionTrackHeight - 9); + this.ctx.restore(); + } + + // Filename next to IC-LoRA Video tag + if (this.node.properties.showFilenames && pxWidth > 80) { + this.ctx.save(); + this.ctx.beginPath(); + this.ctx.rect(startX, trackY, pxWidth, this.motionTrackHeight); + this.ctx.clip(); + let rawPath = seg.videoFile || ""; + let fname = rawPath.split(/[/\\]/).pop() || ""; + this.ctx.font = "9px sans-serif"; + this.ctx.textAlign = "left"; + this.ctx.textBaseline = "middle"; + const maxFileTextW = pxWidth - 75 - 10; + if (this.ctx.measureText(fname).width > maxFileTextW) { + while (fname.length > 0 && this.ctx.measureText(fname + "…").width > maxFileTextW) { + fname = fname.slice(0, -1); + } + fname += "…"; + } + const textW = this.ctx.measureText(fname).width; + this.ctx.fillStyle = "rgba(0, 0, 0, 0.50)"; + this.ctx.fillRect(startX + 76, trackY + 1, textW + 8, 16); + this.ctx.fillStyle = "#fff"; + this.ctx.fillText(fname, startX + 80, trackY + 9); + this.ctx.restore(); + } + } + + // --- Global Prompt subtitle overlay --- + const globalPromptStr = this.getGlobalPrompt(); + if (globalPromptStr && seg.type !== "ghost" && pxWidth > 24) { + const overlayH = Math.round(this.motionTrackHeight * 0.25); + const overlayY = trackY + this.motionTrackHeight - overlayH; + + this.ctx.save(); + this.ctx.beginPath(); + this.ctx.rect(startX, overlayY, pxWidth, overlayH); + this.ctx.clip(); + + // Translucent background + this.ctx.fillStyle = "rgba(0, 0, 0, 0.60)"; + this.ctx.fillRect(startX, overlayY, pxWidth, overlayH); + + // Text + const fontSize = Math.min(11, overlayH * 0.58); + this.ctx.font = `${fontSize}px sans-serif`; + this.ctx.fillStyle = "#e0e3ed"; + this.ctx.textAlign = "center"; + this.ctx.textBaseline = "middle"; + + // Measure and truncate to single line + const maxTextW = pxWidth - 10; + let label = globalPromptStr; + if (this.ctx.measureText(label).width > maxTextW) { + while (label.length > 0 && this.ctx.measureText(label + "…").width > maxTextW) { + label = label.slice(0, -1); + } + label += "…"; + } + + this.ctx.fillText(label, startX + pxWidth / 2, overlayY + overlayH / 2); + this.ctx.restore(); + } + + if (isSelected) { + this.ctx.strokeStyle = "#fff"; + this.ctx.lineWidth = 2; + this.ctx.strokeRect(startX, trackY + 1, pxWidth, this.motionTrackHeight - 2); + if (!this.isMultiSelectActive()) { + this.ctx.fillStyle = "#fff"; + this.ctx.beginPath(); + this.ctx.roundRect(startX, trackY + this.motionTrackHeight / 2 - 12, 4, 24, 2); + this.ctx.fill(); + this.ctx.beginPath(); + this.ctx.roundRect(startX + pxWidth - 4, trackY + this.motionTrackHeight / 2 - 12, 4, 24, 2); + this.ctx.fill(); + } + } else { + this.ctx.strokeStyle = "#000"; + this.ctx.lineWidth = 1.5; + this.ctx.strokeRect(startX, trackY + 1, pxWidth, this.motionTrackHeight - 2); + } + } + this.ctx.globalAlpha = 1.0; + } + + // --- Draw Audio Segments --- + for (let i = 0; i < sortedAudioSegments.length; i++) { + const seg = sortedAudioSegments[i]; + const rawStartX = (seg.start / totalFrames) * width; + const rawEndX = ((seg.start + seg.length) / totalFrames) * width; + const startX = Math.floor(rawStartX); + const pxWidth = Math.max(1, Math.floor(rawEndX) - startX); + const isSelected = this.selectedSegmentIds.includes(seg.id); + const trackY = RULER_HEIGHT + this.blockHeight; + + if ((this._isDragging && this.selectionType === "audio" && seg.id === this._dragTargetId) || (this._ghostSegmentId && seg.id === this._ghostSegmentId)) { + this.ctx.globalAlpha = 0.65; + } else { + this.ctx.globalAlpha = 1.0; + } + + if (seg.type === "ghost") { + this.ctx.fillStyle = "#1a1a1a"; + this.ctx.fillRect(startX, trackY, pxWidth, this.audioTrackHeight); + this.ctx.strokeStyle = "#555"; + this.ctx.lineWidth = 2; + this.ctx.setLineDash([5, 5]); + this.ctx.strokeRect(startX, trackY, pxWidth, this.audioTrackHeight); + this.ctx.setLineDash([]); + this.ctx.fillStyle = "#888"; + this.ctx.textAlign = "center"; + this.ctx.textBaseline = "middle"; + this.ctx.font = "bold 12px sans-serif"; + this.ctx.fillText("Drop Audio", startX + pxWidth / 2, trackY + this.audioTrackHeight / 2); + } else { + const showHandles = !this.isMultiSelectActive(); + const outlineColor = isSelected ? "#fff" : null; + this.drawAudioSegmentVisuals(this.ctx, seg, isSelected, trackY, this.audioTrackHeight, startX, pxWidth, outlineColor, showHandles); + } + this.ctx.globalAlpha = 1.0; + } + + + // --- Dim Disabled Tracks --- + this.ctx.fillStyle = "rgba(0, 0, 0, 0.65)"; + if (!this.mainTrackEnabled) { + this.ctx.fillRect(0, RULER_HEIGHT, width, this.blockHeight); + } + if (!this.audioTrackEnabled) { + this.ctx.fillRect(0, RULER_HEIGHT + this.blockHeight, width, this.audioTrackHeight); + } + if (!this.motionTrackEnabled) { + this.ctx.fillRect(0, RULER_HEIGHT + this.blockHeight + this.audioTrackHeight, width, this.motionTrackHeight); } - this.ctx.globalAlpha = 1.0; } // --- Draw Ruler & Divider AFTER segments to prevent overlap --- @@ -2110,11 +6806,13 @@ class TimelineEditor { this.ctx.fillText(zeroLabel, 4, RULER_HEIGHT / 2); // Divider - this.ctx.fillStyle = "#333"; - this.ctx.fillRect(0, RULER_HEIGHT + this.blockHeight, width, 1); + this.ctx.fillStyle = "#111"; + this.ctx.fillRect(0, RULER_HEIGHT - 1, width, 1); + this.ctx.fillRect(0, RULER_HEIGHT + this.blockHeight - 1, width, 1); + this.ctx.fillRect(0, RULER_HEIGHT + this.blockHeight + this.audioTrackHeight - 1, width, 1); // Draw gap "+" buttons - if (!this._isDragging) { + if (!this._isDragging && !this.retakeMode) { const BTN_R = 12; const gapRegions = this.getGapRegions(); for (let i = 0; i < gapRegions.length; i++) { @@ -2136,30 +6834,30 @@ class TimelineEditor { } // --- Out-of-duration shadow overlay --- - // Draw a translucent black mask over the region beyond the actual output duration - // so the user can clearly see which content will be included in the render. - const outputFrames = this.getDurationFrames(); - if (outputFrames < totalFrames) { - const cutoffX = (outputFrames / totalFrames) * width; - // Semi-transparent black overlay on both tracks - this.ctx.fillStyle = "rgba(0, 0, 0, 0.45)"; - this.ctx.fillRect(cutoffX, RULER_HEIGHT, width - cutoffX, this.blockHeight + this.audioTrackHeight); - // Subtle tinted ruler overlay - this.ctx.fillStyle = "rgba(0, 0, 0, 0.25)"; - this.ctx.fillRect(cutoffX, 0, width - cutoffX, RULER_HEIGHT); - /* - // Dashed boundary line at the output duration cutoff - this.ctx.save(); - this.ctx.strokeStyle = "rgba(255, 80, 80, 0.7)"; - this.ctx.lineWidth = 1.5; - this.ctx.setLineDash([5, 4]); - this.ctx.beginPath(); - this.ctx.moveTo(cutoffX, 0); - this.ctx.lineTo(cutoffX, CANVAS_HEIGHT); - this.ctx.stroke(); - this.ctx.setLineDash([]); - this.ctx.restore(); - */ + // Skip in retake mode — the retake region has its own overlay and the + // start/end frame widgets are locked, so this overlay would be misleading. + if (!this.retakeMode) { + const startFrames = this.getStartFrames(); + const durationFrames = this.getDurationFrames(); + const outputFrames = startFrames + durationFrames; + + if (startFrames > 0) { + const startX = (startFrames / totalFrames) * width; + this.ctx.fillStyle = "rgba(0, 0, 0, 0.6)"; + this.ctx.fillRect(0, RULER_HEIGHT, startX, this.blockHeight + this.motionTrackHeight + this.audioTrackHeight); + this.ctx.fillStyle = "rgba(0, 0, 0, 0.25)"; + this.ctx.fillRect(0, 0, startX, RULER_HEIGHT); + } + + if (outputFrames < totalFrames) { + const cutoffX = (outputFrames / totalFrames) * width; + // Semi-transparent black overlay on both tracks + this.ctx.fillStyle = "rgba(0, 0, 0, 0.6)"; + this.ctx.fillRect(cutoffX, RULER_HEIGHT, width - cutoffX, this.blockHeight + this.motionTrackHeight + this.audioTrackHeight); + // Subtle tinted ruler overlay + this.ctx.fillStyle = "rgba(0, 0, 0, 0.25)"; + this.ctx.fillRect(cutoffX, 0, width - cutoffX, RULER_HEIGHT); + } } // --- Draw Playhead --- @@ -2187,8 +6885,8 @@ class TimelineEditor { const grabBarW = 4; const grabBarH = 50; const grabBarX = this.viewport.scrollLeft + this.viewport.clientWidth - grabBarW - 3; - const grabBarY = RULER_HEIGHT + (this.blockHeight + this.audioTrackHeight - grabBarH) / 2; - + const grabBarY = RULER_HEIGHT + (this.blockHeight + this.motionTrackHeight + this.audioTrackHeight - grabBarH) / 2; + this.ctx.fillStyle = "rgba(40, 40, 40, 0.6)"; this.ctx.beginPath(); this.ctx.roundRect(grabBarX, grabBarY, grabBarW, grabBarH, 2); @@ -2198,17 +6896,44 @@ class TimelineEditor { const hBarW = 50; const hBarH = 4; const hBarX = this.viewport.scrollLeft + (this.viewport.clientWidth - hBarW) / 2; - const hBarY = this.canvasHeight - hBarH - 3; // 3px from the bottom edge - + const visibleBottom = Math.min(this.canvasHeight, this.viewport.scrollTop + this.viewport.clientHeight); + const hBarY = visibleBottom - hBarH - 3; // 3px from the visible bottom edge + this.ctx.fillStyle = "rgba(20, 20, 20, 0.8)"; this.ctx.beginPath(); this.ctx.roundRect(hBarX, hBarY, hBarW, hBarH, 2); this.ctx.fill(); + // --- Draw Selection Box Overlay --- + if (this._isSelectingBox && this._selectBoxStart && this._selectBoxCurrent) { + const sx = this._selectBoxStart.x; + const sy = this._selectBoxStart.y; + const cx = this._selectBoxCurrent.x; + const cy = this._selectBoxCurrent.y; + + const left = Math.min(sx, cx); + const top = Math.min(sy, cy); + const rectWidth = Math.abs(cx - sx); + const rectHeight = Math.abs(cy - sy); + + this.ctx.save(); + this.ctx.fillStyle = "rgba(59, 130, 246, 0.2)"; + this.ctx.fillRect(left, top, rectWidth, rectHeight); + + this.ctx.strokeStyle = "rgba(29, 78, 216, 0.9)"; + this.ctx.lineWidth = 1.5; + this.ctx.setLineDash([4, 4]); + this.ctx.strokeRect(left, top, rectWidth, rectHeight); + this.ctx.setLineDash([]); + this.ctx.restore(); + } + this.updatePlayerUI(); } - drawAudioSegmentVisuals(ctx, seg, isSelected, yOffset, trackHeight, startX, pxWidth) { + + + drawAudioSegmentVisuals(ctx, seg, isSelected, yOffset, trackHeight, startX, pxWidth, outlineColor = null, showHandles = true) { ctx.fillStyle = isSelected ? "#2a4a3a" : "#1a2a1a"; ctx.fillRect(startX, yOffset + 2, pxWidth, trackHeight - 3); @@ -2233,12 +6958,13 @@ class TimelineEditor { } } - ctx.strokeStyle = isSelected ? "#4fff8f" : "#000"; - ctx.lineWidth = 1.5; + const strokeColor = outlineColor || (isSelected ? "#4fff8f" : "#000"); + ctx.strokeStyle = strokeColor; + ctx.lineWidth = isSelected || outlineColor ? 2 : 1.5; ctx.strokeRect(startX, yOffset + 2, pxWidth, trackHeight - 3); - if (isSelected) { - ctx.fillStyle = "#4fff8f"; + if ((isSelected || outlineColor) && showHandles) { + ctx.fillStyle = strokeColor; ctx.beginPath(); ctx.roundRect(startX, yOffset + trackHeight / 2 - 12, 4, 24, 2); ctx.fill(); @@ -2267,6 +6993,24 @@ class TimelineEditor { ctx.fillText(text, startX + 6, yOffset + 8); ctx.restore(); + + // Show Uploading or Decoding badge in bottom-left if applicable + if ((seg._uploading || seg._decoding) && pxWidth > 60) { + ctx.save(); + ctx.beginPath(); + ctx.rect(startX, yOffset + 2, pxWidth, trackHeight - 3); + ctx.clip(); + ctx.font = "bold 9px sans-serif"; + const upText = seg._decoding ? "Decoding..." : "Uploading..."; + const upW = ctx.measureText(upText).width + 10; + ctx.fillStyle = "rgba(0, 14, 37, 0.7)"; + ctx.fillRect(startX + 1, yOffset + trackHeight - 17, upW, 14); + ctx.fillStyle = "#fff"; + ctx.textAlign = "center"; + ctx.textBaseline = "middle"; + ctx.fillText(upText, startX + 1 + upW / 2, yOffset + trackHeight - 10); + ctx.restore(); + } } @@ -2287,12 +7031,28 @@ class TimelineEditor { if (mouseY < RULER_HEIGHT || mouseY > this.canvasHeight) return null; - const isAudioTrack = mouseY > RULER_HEIGHT + this.blockHeight; - const trackSegments = isAudioTrack ? this.timeline.audioSegments : this.timeline.segments; - const trackType = isAudioTrack ? "audio" : "image"; + const trackType = this.getTrackFromY(mouseY); + const trackSegments = this.getSegmentArray(trackType); if (trackSegments.length === 0) return null; + // Helper to check if a segment (or its sibling video/audio counterpart) is uploading/decoding + const isSegmentProcessing = (s) => { + if (!s) return false; + if (s._uploading || s._decoding) return true; + const isVid = s.id?.endsWith("_v"); + const isAud = s.id?.endsWith("_a"); + if (isVid || isAud) { + const siblingId = isVid ? s.id.slice(0, -2) + "_a" : s.id.slice(0, -2) + "_v"; + const siblingArray = isVid ? this.timeline.audioSegments : this.timeline.segments; + const sibling = siblingArray.find(x => x.id === siblingId); + if (sibling && (sibling._uploading || sibling._decoding)) { + return true; + } + } + return false; + }; + // The variables width and totalFrames are already declared above. let sortedSegments = [...trackSegments] @@ -2313,7 +7073,9 @@ class TimelineEditor { const isLeftJoint = prevSeg && prevSeg.start + prevSeg.length === seg.start; if (!isLeftJoint) { if (Math.abs(mouseX - startX) <= HANDLE_HIT_PX) { - return { type: "edge", index: seg.originalIndex, dir: "left", track: trackType }; + if (!isSegmentProcessing(seg)) { + return { type: "edge", index: seg.originalIndex, dir: "left", track: trackType }; + } } } @@ -2322,16 +7084,24 @@ class TimelineEditor { const dx = mouseX - endX; if (Math.abs(dx) <= HANDLE_HIT_PX) { if (dx < -HANDLE_CORE) { - return { type: "edge", index: seg.originalIndex, dir: "right", track: trackType }; + if (!isSegmentProcessing(seg)) { + return { type: "edge", index: seg.originalIndex, dir: "right", track: trackType }; + } } else if (dx > HANDLE_CORE) { - return { type: "edge", index: nextSeg.originalIndex, dir: "left", track: trackType }; + if (!isSegmentProcessing(nextSeg)) { + return { type: "edge", index: nextSeg.originalIndex, dir: "left", track: trackType }; + } } else { - return { type: "joint", leftIndex: seg.originalIndex, rightIndex: nextSeg.originalIndex, track: trackType }; + if (!isSegmentProcessing(seg) && !isSegmentProcessing(nextSeg)) { + return { type: "joint", leftIndex: seg.originalIndex, rightIndex: nextSeg.originalIndex, track: trackType }; + } } } } else { if (Math.abs(mouseX - endX) <= HANDLE_HIT_PX) { - return { type: "edge", index: seg.originalIndex, dir: "right", track: trackType }; + if (!isSegmentProcessing(seg)) { + return { type: "edge", index: seg.originalIndex, dir: "right", track: trackType }; + } } } } @@ -2351,20 +7121,32 @@ class TimelineEditor { } onMouseDown(e) { + if (e.button === 2 && this.retakeMode) { + e.preventDefault(); + e.stopPropagation(); + return; + } if (e.button !== 0) return; const { x, y } = this.getMousePos(e); - const isOverDivider = Math.abs(y - (RULER_HEIGHT + this.blockHeight)) <= 4; - if (isOverDivider) { + // In retake mode: block box selection — no multi-segment operations allowed + if (e.shiftKey && !this.retakeMode) { + this._isSelectingBox = true; this._isDragging = true; - this._dragType = "divider"; - this._startBlockHeight = this.blockHeight; - this._startAudioTrackHeight = this.audioTrackHeight; - this._startY = y; + this._dragType = "box_select"; + this._selectBoxStart = { x, y }; + this._selectBoxCurrent = { x, y }; + this._selectBoxInitialSelectedIds = (e.ctrlKey || e.metaKey) ? [...this.selectedSegmentIds] : []; + this.selectedSegmentIds = [...this._selectBoxInitialSelectedIds]; + this.syncSelectionTypeAndIndex(); + this.updateUIFromSelection(); + this.render(); return; } - const isAtBottom = Math.abs(y - this.canvasHeight) <= 15; + // Canvas height and width resizing apply in both modes. + const visibleBottom = Math.min(this.canvasHeight, this.viewport.scrollTop + this.viewport.clientHeight); + const isAtBottom = Math.abs(y - visibleBottom) <= 15; if (isAtBottom) { this._isDragging = true; this._dragType = "height_resize"; @@ -2385,6 +7167,117 @@ class TimelineEditor { return; } + // Track height dividers only apply in normal timeline mode. + if (!this.retakeMode) { + const isOverDivider = Math.abs(y - (RULER_HEIGHT + this.blockHeight)) <= 8; + const isOverAudioDivider = Math.abs(y - (RULER_HEIGHT + this.blockHeight + this.audioTrackHeight)) <= 8; + if (isOverDivider) { + this._isDragging = true; + this._dragType = "divider"; + this._startBlockHeight = this.blockHeight; + this._startAudioTrackHeight = this.audioTrackHeight; + this._startY = y; + return; + } else if (isOverAudioDivider) { + this._isDragging = true; + this._dragType = "audio_divider"; + this._startMotionTrackHeight = this.motionTrackHeight; + this._startAudioTrackHeight = this.audioTrackHeight; + this._startY = y; + return; + } + } + + if (this.retakeMode) { + // If no video is loaded on the retake timeline, clicking in the timeline opens the file explorer + if (y >= RULER_HEIGHT && y <= RULER_HEIGHT + this.blockHeight) { + if (!this.timeline.retakeVideo) { + if (this.videoFileInput) { + this.videoFileInput.click(); + } + return; + } + } + + if (y < RULER_HEIGHT) { + this._isDragging = true; + this._dragType = "playhead"; + const logicalWidth = this.canvas.offsetWidth; + const totalFrames = this.getVisualDurationFrames(); + let mouseFrameX = x * (totalFrames / logicalWidth); + mouseFrameX = this.getSnappedPlayhead(mouseFrameX, logicalWidth); + const clampMax = this.timeline.retakeVideo ? (this.timeline.retakeVideo.videoDurationFrames || totalFrames) : totalFrames; + this.currentFrame = clamp(mouseFrameX, 0, clampMax); + // Pause only the RAF playback loop so we can seek the video directly during scrub. + // The video element itself keeps playing; we'll resume the loop on mouseup. + this._retakeScrubWasPlaying = this.isPlaying; + if (this.isPlaying) { + this.isPlaying = false; + this._currentPlayId = null; + } + if (this.timeline.retakeVideo && this.timeline.retakeVideo.videoEl) { + this.timeline.retakeVideo.videoEl.currentTime = this.currentFrame / this.getFrameRate(); + } + this.render(); + return; + } + + if (y >= RULER_HEIGHT && y <= RULER_HEIGHT + this.blockHeight) { + const logicalWidth = this.canvas.offsetWidth; + const totalFrames = this.getVisualDurationFrames(); + const retakeStart = this.timeline.retakeStart ?? 0; + const baseVideoDur = this.timeline.retakeVideo?.videoDurationFrames ?? totalFrames; + const retakeLength = this.timeline.retakeLength ?? baseVideoDur; + + const x1 = (retakeStart / totalFrames) * logicalWidth; + const x2 = ((retakeStart + retakeLength) / totalFrames) * logicalWidth; + const threshold = HANDLE_HIT_PX; + + if (this.timeline.retakeVideo && Math.abs(x - x1) <= threshold) { + this._isDragging = true; + this._dragType = "retake_left"; + this._dragStartX = x; + this._dragStartRetakeStart = retakeStart; + this._dragStartRetakeLength = retakeLength; + return; + } else if (this.timeline.retakeVideo && Math.abs(x - x2) <= threshold) { + this._isDragging = true; + this._dragType = "retake_right"; + this._dragStartX = x; + this._dragStartRetakeStart = retakeStart; + this._dragStartRetakeLength = retakeLength; + return; + } else if (this.timeline.retakeVideo && x > x1 && x < x2) { + this._isDragging = true; + this._dragType = "retake_center"; + this._dragStartX = x; + this._dragStartRetakeStart = retakeStart; + this._dragStartRetakeLength = retakeLength; + return; + } else { + this._isDragging = true; + this._dragType = "playhead"; + let mouseFrameX = x * (totalFrames / logicalWidth); + mouseFrameX = this.getSnappedPlayhead(mouseFrameX, logicalWidth); + const clampMax = this.timeline.retakeVideo ? (this.timeline.retakeVideo.videoDurationFrames || totalFrames) : totalFrames; + this.currentFrame = clamp(mouseFrameX, 0, clampMax); + // Pause only the RAF playback loop so we can seek the video directly during scrub. + this._retakeScrubWasPlaying = this.isPlaying; + if (this.isPlaying) { + this.isPlaying = false; + this._currentPlayId = null; + } + if (this.timeline.retakeVideo && this.timeline.retakeVideo.videoEl) { + this.timeline.retakeVideo.videoEl.currentTime = this.currentFrame / this.getFrameRate(); + } + this.render(); + return; + } + } + // Retake mode consumed the interaction — do NOT fall through to normal timeline + return; + } + if (y >= RULER_HEIGHT && y <= this.canvasHeight) { const BTN_R = 12; const gapRegions = this.getGapRegions(); @@ -2393,8 +7286,14 @@ class TimelineEditor { if (gap.widthPx < BTN_R * 2 + 8) continue; const dx = x - gap.centerX, dy2 = y - gap.centerY; if (dx * dx + dy2 * dy2 <= BTN_R * BTN_R) { - if (gap.track === "audio") { - // Direct to audio upload + const currentTrack = gap.track; + const hasCopied = this._copiedSegment || window._ltxCopiedSegment; + const copiedTrack = this._copiedSegmentTrack || window._ltxCopiedSegmentType; + const isCompatible = hasCopied && this.getCanonicalTrack(copiedTrack) === currentTrack; + + if (currentTrack === "motion" && !isCompatible) { + this.promptAddMotionInGap(gap.frameStart, gap.frameEnd); + } else if (currentTrack === "audio" && !isCompatible) { this.promptAddAudioInGap(gap.frameStart, gap.frameEnd); } else { this.showGapMenu(e.clientX, e.clientY, gap); @@ -2404,11 +7303,11 @@ class TimelineEditor { } } + const isCtrl = e.ctrlKey || e.metaKey; const hit = this.getHitTest(x, y); if (!hit) { - // Only deselect if they clicked the same track but hit empty space - const clickedTrack = y > RULER_HEIGHT + this.blockHeight ? "audio" : "image"; - if (this.selectionType === clickedTrack) { + if (!isCtrl) { + this.selectedSegmentIds = []; this.selectedIndex = -1; this.updateUIFromSelection(); } @@ -2422,7 +7321,9 @@ class TimelineEditor { const logicalWidth = this.canvas.offsetWidth; const totalFrames = this.getVisualDurationFrames(); let mouseFrameX = x * (totalFrames / logicalWidth); + mouseFrameX = this.getSnappedPlayhead(mouseFrameX, logicalWidth); this.currentFrame = clamp(mouseFrameX, 0, totalFrames); + this._liveScrubPlayhead(); this.render(); if (this.isPlaying) { this.playAudio(); @@ -2430,41 +7331,151 @@ class TimelineEditor { return; } - this.selectionType = hit.track; - const targetArray = hit.track === "audio" ? this.timeline.audioSegments : this.timeline.segments; - + const clickedTrack = hit.track; + const targetArray = this.getSegmentArray(clickedTrack); + let clickedId = null; + let clickedIdx = -1; if (hit.type === "joint") { - this.selectedIndex = hit.leftIndex; - this.updateUIFromSelection(); - this._dragType = "joint"; - this._dragTargetId = targetArray[hit.leftIndex].id; - this._dragTargetIdRight = targetArray[hit.rightIndex].id; - } else if (hit.type === "center") { - this.selectedIndex = hit.index; - this.updateUIFromSelection(); - this._dragType = "center"; + clickedIdx = hit.leftIndex; } else { - if (this.selectedIndex !== hit.index) { - this.selectedIndex = hit.index; - this.updateUIFromSelection(); + clickedIdx = hit.index; + } + if (clickedIdx !== -1 && targetArray[clickedIdx]) { + clickedId = targetArray[clickedIdx].id; + } + + if (clickedId) { + if (isCtrl) { + const sibId = clickedId.endsWith("_v") ? clickedId.slice(0, -2) + "_a" : (clickedId.endsWith("_a") ? clickedId.slice(0, -2) + "_v" : null); + const isSelected = this.selectedSegmentIds.includes(clickedId); + if (isSelected) { + this.selectedSegmentIds = this.selectedSegmentIds.filter(id => id !== clickedId && id !== sibId); + } else { + if (!this.selectedSegmentIds.includes(clickedId)) this.selectedSegmentIds.push(clickedId); + if (sibId && !this.selectedSegmentIds.includes(sibId)) this.selectedSegmentIds.push(sibId); + } + + if (this.selectedSegmentIds.length > 0) { + this.selectionType = clickedTrack; + this.selectedIndex = clickedIdx; + } else { + this.selectedIndex = -1; + } + this._multiDragClickPendingDeselect = null; + } else { + if (this.selectedSegmentIds.includes(clickedId)) { + this._multiDragClickPendingDeselect = clickedId; + } else { + this.selectedSegmentIds = [clickedId]; + const sibId = clickedId.endsWith("_v") ? clickedId.slice(0, -2) + "_a" : (clickedId.endsWith("_a") ? clickedId.slice(0, -2) + "_v" : null); + if (sibId && !this.selectedSegmentIds.includes(sibId)) this.selectedSegmentIds.push(sibId); + this.selectionType = clickedTrack; + this.selectedIndex = clickedIdx; + this._multiDragClickPendingDeselect = null; + } } - this._dragType = hit.dir; } - this._isDragging = true; - this._previewSegments = null; - this._dragStartX = x; - this._dragInitialTimeline = JSON.parse(JSON.stringify(targetArray)); + this.updateUIFromSelection(); - if (hit.type !== "joint") { - this._dragTargetId = targetArray[hit.index].id; + if (this.isMultiSelectActive()) { + this._isDragging = true; + this._dragType = "center"; + this._dragStartX = x; + this._isMultiDraggingAndMoved = false; + this._multiDragInitialSegments = { + image: this.timeline.segments.map(s => ({ ...s })), + motion: this.timeline.motionSegments.map(s => ({ ...s })), + audio: this.timeline.audioSegments.map(s => ({ ...s })) + }; + this._multiDragPreviewTimelines = null; + } else { + this.selectionType = hit.track; + if (hit.type === "joint") { + this.selectedIndex = hit.leftIndex; + this._dragType = "joint"; + this._dragTargetId = targetArray[hit.leftIndex].id; + this._dragTargetIdRight = targetArray[hit.rightIndex].id; + } else if (hit.type === "center") { + this.selectedIndex = hit.index; + this._dragType = "center"; + } else { + if (this.selectedIndex !== hit.index) { + this.selectedIndex = hit.index; + } + this._dragType = hit.dir; + } + + this._isDragging = true; + this._previewSegments = null; + this._previewSiblingSegments = null; + this._dragStartX = x; + this._dragInitialTimeline = targetArray.map(s => ({ ...s })); + this._dragInitialSiblingTimeline = this.selectionType === "motion" ? null : (this.selectionType === "audio" ? this.timeline.segments : this.timeline.audioSegments).map(s => ({ ...s })); + + if (hit.type !== "joint") { + this._dragTargetId = targetArray[hit.index].id; + } } + + if (this.isPlaying) { + this.pauseAudio(); + } + this.render(); } onMouseMove(e) { const { x: mouseX, y: mouseY } = this.getMousePos(e); + if (this._isSelectingBox && this._dragType === "box_select") { + this.canvas.style.cursor = "crosshair"; + this._selectBoxCurrent = { x: mouseX, y: mouseY }; + this.updateSelectionFromBox(); + this.render(); + return; + } + + if (this.retakeMode && !this._isDragging) { + const visibleBottom = Math.min(this.canvasHeight, this.viewport.scrollTop + this.viewport.clientHeight); + const isAtBottom = Math.abs(mouseY - visibleBottom) <= 15; + const viewRect = this.viewport.getBoundingClientRect(); + const isAtRightEdge = Math.abs(e.clientX - viewRect.right) <= 20; + + if (isAtBottom) { + this.canvas.style.cursor = "ns-resize"; + return; + } else if (isAtRightEdge) { + this.canvas.style.cursor = "ew-resize"; + return; + } + + if (mouseY >= RULER_HEIGHT && mouseY <= RULER_HEIGHT + this.blockHeight) { + const logicalWidth = this.canvas.offsetWidth; + const totalFrames = this.getVisualDurationFrames(); + const retakeStart = this.timeline.retakeStart ?? 0; + const baseVideoDur = this.timeline.retakeVideo?.videoDurationFrames ?? totalFrames; + const retakeLength = this.timeline.retakeLength ?? baseVideoDur; + + const x1 = (retakeStart / totalFrames) * logicalWidth; + const x2 = ((retakeStart + retakeLength) / totalFrames) * logicalWidth; + const threshold = HANDLE_HIT_PX; + + if (Math.abs(mouseX - x1) <= threshold || Math.abs(mouseX - x2) <= threshold) { + this.canvas.style.cursor = "ew-resize"; + } else if (mouseX > x1 && mouseX < x2) { + this.canvas.style.cursor = "move"; + } else { + this.canvas.style.cursor = "default"; + } + } else if (mouseY < RULER_HEIGHT) { + this.canvas.style.cursor = "ew-resize"; + } else { + this.canvas.style.cursor = "default"; + } + return; + } + if (!this._isDragging) { let newHoveredGapIdx = -1; const BTN_R = 12; @@ -2480,12 +7491,14 @@ class TimelineEditor { this.render(); } - const isOverDivider = Math.abs(mouseY - (RULER_HEIGHT + this.blockHeight)) <= 4; - const isAtBottom = Math.abs(mouseY - this.canvasHeight) <= 15; + const isOverDivider = Math.abs(mouseY - (RULER_HEIGHT + this.blockHeight)) <= 8; + const isOverAudioDivider = Math.abs(mouseY - (RULER_HEIGHT + this.blockHeight + this.audioTrackHeight)) <= 8; + const visibleBottom = Math.min(this.canvasHeight, this.viewport.scrollTop + this.viewport.clientHeight); + const isAtBottom = Math.abs(mouseY - visibleBottom) <= 15; const viewRect = this.viewport.getBoundingClientRect(); const isAtRightEdge = Math.abs(e.clientX - viewRect.right) <= 20; const hit = this.getHitTest(mouseX, mouseY); - if (isOverDivider || isAtBottom) { + if (isOverDivider || isOverAudioDivider || isAtBottom) { this.canvas.style.cursor = "ns-resize"; } else if (isAtRightEdge) { this.canvas.style.cursor = "ew-resize"; @@ -2505,6 +7518,162 @@ class TimelineEditor { return; } + if (this.retakeMode && this._isDragging) { + const totalFrames = this.getVisualDurationFrames(); + const logicalWidth = this.canvas.offsetWidth; + const deltaX = mouseX - this._dragStartX; + const deltaFrames = Math.round(deltaX * (totalFrames / logicalWidth)); + + const frameRate = this.getFrameRate(); + + // Handle playhead drag in retakeMode — the RAF loop is paused, so seek directly + if (this._dragType === "playhead") { + this.canvas.style.cursor = "ew-resize"; + let mouseFrameX = mouseX * (totalFrames / logicalWidth); + mouseFrameX = this.getSnappedPlayhead(mouseFrameX, logicalWidth); + const clampMax = this.timeline.retakeVideo ? (this.timeline.retakeVideo.videoDurationFrames || totalFrames) : totalFrames; + this.currentFrame = clamp(mouseFrameX, 0, clampMax); + if (this.timeline.retakeVideo && this.timeline.retakeVideo.videoEl) { + this.timeline.retakeVideo.videoEl.currentTime = this.currentFrame / frameRate; + } + this.render(); + return; + } + + if (this._dragType === "retake_left") { + this.canvas.style.cursor = "ew-resize"; + let newStart = this._dragStartRetakeStart + deltaFrames; + let newLength = this._dragStartRetakeLength - deltaFrames; + + if (this.isSnapping) { + const thresholdFrames = (15 / logicalWidth) * totalFrames; + const baseVideoDur = this.timeline.retakeVideo?.videoDurationFrames ?? totalFrames; + const candidates = [0, this.currentFrame, baseVideoDur]; + let bestStart = newStart; + let minDiff = thresholdFrames; + for (const c of candidates) { + const diff = Math.abs(newStart - c); + if (diff < minDiff) { + minDiff = diff; + bestStart = c; + } + } + if (bestStart !== newStart) { + newStart = bestStart; + newLength = this._dragStartRetakeStart + this._dragStartRetakeLength - newStart; + } + } + + if (newStart < 0) { + newStart = 0; + newLength = this._dragStartRetakeStart + this._dragStartRetakeLength; + } + if (newLength < MIN_SEGMENT_LENGTH) { + newLength = MIN_SEGMENT_LENGTH; + newStart = this._dragStartRetakeStart + this._dragStartRetakeLength - MIN_SEGMENT_LENGTH; + } + + this.timeline.retakeStart = newStart; + this.timeline.retakeLength = newLength; + + if (this.timeline.retakeVideo && this.timeline.retakeVideo.videoEl) { + this.timeline.retakeVideo.videoEl.currentTime = newStart / frameRate; + } + + this.render(); + this.updateUIFromSelection(); + return; + } + + if (this._dragType === "retake_right") { + this.canvas.style.cursor = "ew-resize"; + let newLength = this._dragStartRetakeLength + deltaFrames; + + const baseVideoDur = this.timeline.retakeVideo?.videoDurationFrames ?? totalFrames; + let newEnd = this._dragStartRetakeStart + newLength; + + if (this.isSnapping) { + const thresholdFrames = (15 / logicalWidth) * totalFrames; + const candidates = [0, this.currentFrame, baseVideoDur]; + let bestEnd = newEnd; + let minDiff = thresholdFrames; + for (const c of candidates) { + const diff = Math.abs(newEnd - c); + if (diff < minDiff) { + minDiff = diff; + bestEnd = c; + } + } + if (bestEnd !== newEnd) { + newEnd = bestEnd; + newLength = newEnd - this._dragStartRetakeStart; + } + } + + if (this._dragStartRetakeStart + newLength > baseVideoDur) { + newLength = baseVideoDur - this._dragStartRetakeStart; + } + if (newLength < MIN_SEGMENT_LENGTH) { + newLength = MIN_SEGMENT_LENGTH; + } + + this.timeline.retakeLength = newLength; + + if (this.timeline.retakeVideo && this.timeline.retakeVideo.videoEl) { + this.timeline.retakeVideo.videoEl.currentTime = (this.timeline.retakeStart + newLength) / frameRate; + } + + this.render(); + this.updateUIFromSelection(); + return; + } + + if (this._dragType === "retake_center") { + this.canvas.style.cursor = "grabbing"; + let newStart = this._dragStartRetakeStart + deltaFrames; + + if (this.isSnapping) { + const thresholdFrames = (15 / logicalWidth) * totalFrames; + const baseVideoDur = this.timeline.retakeVideo?.videoDurationFrames ?? totalFrames; + const candidates = [0, this.currentFrame, baseVideoDur]; + let bestStart = newStart; + let minDiff = thresholdFrames; + + for (const c of candidates) { + const diffLeft = Math.abs(newStart - c); + if (diffLeft < minDiff) { + minDiff = diffLeft; + bestStart = c; + } + const diffRight = Math.abs((newStart + this._dragStartRetakeLength) - c); + if (diffRight < minDiff) { + minDiff = diffRight; + bestStart = c - this._dragStartRetakeLength; + } + } + newStart = bestStart; + } + + if (newStart < 0) { + newStart = 0; + } + const baseVideoDur = this.timeline.retakeVideo?.videoDurationFrames ?? totalFrames; + if (newStart + this._dragStartRetakeLength > baseVideoDur) { + newStart = baseVideoDur - this._dragStartRetakeLength; + } + + this.timeline.retakeStart = newStart; + + if (this.timeline.retakeVideo && this.timeline.retakeVideo.videoEl) { + this.timeline.retakeVideo.videoEl.currentTime = newStart / frameRate; + } + + this.render(); + this.updateUIFromSelection(); + return; + } + } + if (this._dragType === "divider") { this.canvas.style.cursor = "ns-resize"; const deltaY = mouseY - this._startY; @@ -2527,6 +7696,35 @@ class TimelineEditor { this.blockHeight = newBlockHeight; this.audioTrackHeight = newAudioTrackHeight; + this.updateSidebarHeights(); + this.render(); + return; + } + + if (this._dragType === "audio_divider") { + this.canvas.style.cursor = "ns-resize"; + const deltaY = mouseY - this._startY; + + const minMotionH = 50; + const minAudioH = 50; + + // Divider moves down: audio gets bigger, motion gets smaller + let newAudioTrackHeight = this._startAudioTrackHeight + deltaY; + let newMotionTrackHeight = this._startMotionTrackHeight - deltaY; + + if (newAudioTrackHeight < minAudioH) { + newAudioTrackHeight = minAudioH; + newMotionTrackHeight = this._startAudioTrackHeight + this._startMotionTrackHeight - minAudioH; + } + if (newMotionTrackHeight < minMotionH) { + newMotionTrackHeight = minMotionH; + newAudioTrackHeight = this._startAudioTrackHeight + this._startMotionTrackHeight - minMotionH; + } + + this.motionTrackHeight = newMotionTrackHeight; + this.audioTrackHeight = newAudioTrackHeight; + + this.updateSidebarHeights(); this.render(); return; } @@ -2536,11 +7734,12 @@ class TimelineEditor { const deltaY = mouseY - this._startY; this.blockHeight = Math.max(100, this._startBlockHeight + deltaY); - this.canvasHeight = this.rulerHeight + this.blockHeight + this.audioTrackHeight; + this.canvasHeight = this.rulerHeight + this.blockHeight + this.motionTrackHeight + this.audioTrackHeight; this.canvas.style.height = `${this.canvasHeight}px`; this.resizeCanvas(this.canvas.offsetWidth); + this.updateSidebarHeights(); this.render(); if (this.node && this.node.computeSize) { @@ -2570,7 +7769,9 @@ class TimelineEditor { const logicalWidth = this.canvas.offsetWidth; const totalFrames = this.getVisualDurationFrames(); let mouseFrameX = mouseX * (totalFrames / logicalWidth); + mouseFrameX = this.getSnappedPlayhead(mouseFrameX, logicalWidth); this.currentFrame = clamp(mouseFrameX, 0, totalFrames); + this._liveScrubPlayhead(); this.render(); if (this.isPlaying) { this.playAudio(); // Scrub (restart from new position) @@ -2578,15 +7779,156 @@ class TimelineEditor { return; } + if (this._multiDragInitialSegments) { + this.canvas.style.cursor = "grabbing"; + this._isMultiDraggingAndMoved = true; + + const logicalWidth = this.canvas.offsetWidth; + const totalFrames = this.getVisualDurationFrames(); + const durationFrames = totalFrames; + let dragDelta = Math.round((mouseX - this._dragStartX) * (totalFrames / logicalWidth)); + + const selectedIds = this.selectedSegmentIds; + + // Group Blocking Physics Calculation + let maxLeftShift = Infinity; + let maxRightShift = Infinity; + + for (const track of ["image", "motion", "audio"]) { + const allTrackSegs = this._multiDragInitialSegments[track]; + if (!allTrackSegs) continue; + const selectedOnTrack = allTrackSegs.filter(s => selectedIds.includes(s.id)); + const nonSelectedOnTrack = allTrackSegs.filter(s => !selectedIds.includes(s.id)); + + if (selectedOnTrack.length === 0) continue; + + for (const S of selectedOnTrack) { + // Find closest non-selected segment to the left on the same track + let closestLeftEnd = 0; + for (const L of nonSelectedOnTrack) { + if (L.start + L.length <= S.start) { + closestLeftEnd = Math.max(closestLeftEnd, L.start + L.length); + } + } + const spaceLeft = S.start - closestLeftEnd; + maxLeftShift = Math.min(maxLeftShift, spaceLeft); + + // Find closest non-selected segment to the right on the same track + let closestRightStart = durationFrames; + for (const R of nonSelectedOnTrack) { + if (R.start >= S.start + S.length) { + closestRightStart = Math.min(closestRightStart, R.start); + } + } + const spaceRight = closestRightStart - (S.start + S.length); + maxRightShift = Math.min(maxRightShift, spaceRight); + } + } + + // Clamp drag delta + let clampedDragDelta = clamp(dragDelta, -maxLeftShift, maxRightShift); + + // Apply snapping if active + if (this.isSnapping) { + const thresholdFrames = (15 / logicalWidth) * totalFrames; + let bestAdjustment = null; + let minDiff = thresholdFrames; + + // Collect snap candidates + const snapCandidates = [0, this.getDurationFrames(), this.getStartFrames(), this.currentFrame]; + if (this.endFramesWidget && this.endFramesWidget.value !== undefined) { + snapCandidates.push(parseInt(this.endFramesWidget.value, 10)); + } + + for (const track of ["image", "motion", "audio"]) { + const allTrackSegs = this._multiDragInitialSegments[track]; + if (!allTrackSegs) continue; + const nonSelectedOnTrack = allTrackSegs.filter(s => !selectedIds.includes(s.id)); + for (const L of nonSelectedOnTrack) { + snapCandidates.push(L.start); + snapCandidates.push(L.start + L.length); + } + } + + // Test all selected segments against candidates + for (const track of ["image", "motion", "audio"]) { + const allTrackSegs = this._multiDragInitialSegments[track]; + if (!allTrackSegs) continue; + const selectedOnTrack = allTrackSegs.filter(s => selectedIds.includes(s.id)); + for (const S of selectedOnTrack) { + const targetStart = S.start + clampedDragDelta; + const targetEnd = S.start + S.length + clampedDragDelta; + + for (const cand of snapCandidates) { + // Check start edge + const diffStart = cand - targetStart; + if (Math.abs(diffStart) < minDiff) { + minDiff = Math.abs(diffStart); + bestAdjustment = diffStart; + } + // Check end edge + const diffEnd = cand - targetEnd; + if (Math.abs(diffEnd) < minDiff) { + minDiff = Math.abs(diffEnd); + bestAdjustment = diffEnd; + } + } + } + } + + if (bestAdjustment !== null) { + const adjustedDelta = clampedDragDelta + bestAdjustment; + if (adjustedDelta >= -maxLeftShift && adjustedDelta <= maxRightShift) { + clampedDragDelta = adjustedDelta; + } + } + } + + // Compute previews + this._multiDragPreviewTimelines = { + image: this._multiDragInitialSegments.image.map(s => { + if (selectedIds.includes(s.id)) { + return { ...s, start: s.start + clampedDragDelta }; + } + return s; + }), + motion: this._multiDragInitialSegments.motion.map(s => { + if (selectedIds.includes(s.id)) { + return { ...s, start: s.start + clampedDragDelta }; + } + return s; + }), + audio: this._multiDragInitialSegments.audio.map(s => { + if (selectedIds.includes(s.id)) { + return { ...s, start: s.start + clampedDragDelta }; + } + return s; + }) + }; + + // Scrub support for video segments being moved + for (const track of ["image", "motion"]) { + const prevSegs = this._multiDragPreviewTimelines[track]; + for (const s of prevSegs) { + if (selectedIds.includes(s.id) && (s.type === "video" || s.type === "motion_video")) { + this._liveScrubVideo(s, "start"); + } + } + } + + this.render(); + return; + } + this.canvas.style.cursor = this._dragType === "center" ? "grabbing" : this._dragType === "joint" ? "col-resize" : "ew-resize"; const logicalWidth = this.canvas.offsetWidth; const totalFrames = this.getVisualDurationFrames(); const durationFrames = totalFrames; - const dragDelta = Math.round((mouseX - this._dragStartX) * (totalFrames / logicalWidth)); + let dragDelta = Math.round((mouseX - this._dragStartX) * (totalFrames / logicalWidth)); - let t = JSON.parse(JSON.stringify(this._dragInitialTimeline)); + let t = this._dragInitialTimeline.map(s => ({ ...s })); // --- Rolling Edit (Slide Edit) --- if (this._dragType === "joint") { @@ -2600,23 +7942,62 @@ class TimelineEditor { let maxDeltaRight = origRight.length - MIN_SEGMENT_LENGTH; let maxDeltaLeft = origLeft.length - MIN_SEGMENT_LENGTH; - if (this.selectionType === "audio") { + if (this.selectionType === "audio" || origRight.type === "video") { // Drag LEFT: right clip extends left by un-trimming its head. // Can only un-trim as much as the right clip has been trimmed (trimStart >= 0). maxDeltaLeft = Math.min(maxDeltaLeft, origRight.trimStart || 0); + } + if (this.selectionType === "audio" || origLeft.type === "video") { // Drag RIGHT: left clip extends right by consuming its remaining tail audio. // Can only extend as far as the left clip's unplayed tail allows. - let availLeftTail = (origLeft.audioDurationFrames || origLeft.length) - ((origLeft.trimStart || 0) + origLeft.length); + let origDur = origLeft.audioDurationFrames || origLeft.videoDurationFrames || origLeft.length; + let availLeftTail = origDur - ((origLeft.trimStart || 0) + origLeft.length); maxDeltaRight = Math.min(maxDeltaRight, availLeftTail); } + // Apply snapping to the shared boundary position + if (this.isSnapping) { + const thresholdFrames = (15 / logicalWidth) * totalFrames; + const jointPos = origLeft.start + origLeft.length + dragDelta; + let bestJoint = jointPos; + let minDiff = thresholdFrames; + + const snapCandidates = [0, this.getDurationFrames(), this.currentFrame]; + snapCandidates.push(this.getStartFrames()); + if (this.endFramesWidget && this.endFramesWidget.value !== undefined) { + snapCandidates.push(parseInt(this.endFramesWidget.value, 10)); + } + const allTracks = [ + this.timeline.segments || [], + this.timeline.motionSegments || [], + this.timeline.audioSegments || [] + ]; + const ignoreIds = [String(this._dragTargetId), String(this._dragTargetIdRight)]; + for (const track of allTracks) { + for (const seg of track) { + if (ignoreIds.includes(String(seg.id))) continue; + snapCandidates.push(seg.start); + snapCandidates.push(seg.start + seg.length); + } + } + + for (const candidate of snapCandidates) { + const diff = Math.abs(jointPos - candidate); + if (diff < minDiff) { + minDiff = diff; + bestJoint = candidate; + } + } + dragDelta = bestJoint - (origLeft.start + origLeft.length); + } + let safeDelta = clamp(dragDelta, -maxDeltaLeft, maxDeltaRight); t[leftIdx].length = origLeft.length + safeDelta; t[rightIdx].start = origRight.start + safeDelta; t[rightIdx].length = origRight.length - safeDelta; - if (this.selectionType === "audio") { + if (this.selectionType === "audio" || t[rightIdx].type === "video") { t[rightIdx].trimStart = origRight.trimStart + safeDelta; } } @@ -2628,27 +8009,137 @@ class TimelineEditor { if (this._dragType === "right") { let newLen = t[targetIdx].length + dragDelta; + if (this.isSnapping) { + const thresholdFrames = (15 / logicalWidth) * totalFrames; + const targetEnd = t[targetIdx].start + newLen; + let bestEnd = targetEnd; + let minDiff = thresholdFrames; + + const snapCandidates = [0, this.getDurationFrames(), this.currentFrame]; + // Add start and end frames of active generation range + snapCandidates.push(this.getStartFrames()); + if (this.endFramesWidget && this.endFramesWidget.value !== undefined) { + snapCandidates.push(parseInt(this.endFramesWidget.value, 10)); + } + const allTracks = [ + this.timeline.segments || [], + this.timeline.motionSegments || [], + this.timeline.audioSegments || [] + ]; + const ignoreSegmentIds = [String(this._dragTargetId)]; + const isVid = String(this._dragTargetId).endsWith("_v"); + const isAud = String(this._dragTargetId).endsWith("_a"); + if (isVid || isAud) { + const siblingId = isVid ? String(this._dragTargetId).slice(0, -2) + "_a" : String(this._dragTargetId).slice(0, -2) + "_v"; + ignoreSegmentIds.push(siblingId); + } + + for (const track of allTracks) { + for (const seg of track) { + if (ignoreSegmentIds.includes(String(seg.id))) continue; + snapCandidates.push(seg.start); + snapCandidates.push(seg.start + seg.length); + } + } + + for (const candidate of snapCandidates) { + const diff = Math.abs(targetEnd - candidate); + if (diff < minDiff) { + minDiff = diff; + bestEnd = candidate; + } + } + newLen = bestEnd - t[targetIdx].start; + dragDelta = newLen - t[targetIdx].length; + } let maxPossibleLength = totalFrames - t[targetIdx].start; let nextSeg = t.find(s => s.start >= t[targetIdx].start + t[targetIdx].length && s.id !== t[targetIdx].id); if (nextSeg) { maxPossibleLength = nextSeg.start - t[targetIdx].start; } - if (this.selectionType === "audio") { - maxPossibleLength = Math.min(maxPossibleLength, (t[targetIdx].audioDurationFrames || t[targetIdx].length) - (t[targetIdx].trimStart || 0)); + // Check sibling track obstacles if linked + const isVid = String(this._dragTargetId).endsWith("_v"); + const isAud = String(this._dragTargetId).endsWith("_a"); + const siblingId = (isVid || isAud) ? (isVid ? String(this._dragTargetId).slice(0, -2) + "_a" : String(this._dragTargetId).slice(0, -2) + "_v") : null; + if (siblingId && this._dragInitialSiblingTimeline) { + let nextSibSeg = this._dragInitialSiblingTimeline.find(s => s.start >= t[targetIdx].start + t[targetIdx].length && s.id !== siblingId); + if (nextSibSeg) { + let sibMaxPossible = nextSibSeg.start - t[targetIdx].start; + maxPossibleLength = Math.min(maxPossibleLength, sibMaxPossible); + } + } + + if (this.selectionType === "audio" || t[targetIdx].type === "video" || t[targetIdx].type === "motion_video") { + const origDur = t[targetIdx].audioDurationFrames || t[targetIdx].videoDurationFrames || t[targetIdx].length; + maxPossibleLength = Math.min(maxPossibleLength, origDur - (t[targetIdx].trimStart || 0)); } t[targetIdx].length = Math.max(MIN_SEGMENT_LENGTH, Math.min(newLen, maxPossibleLength)); } else if (this._dragType === "left") { let newStart = t[targetIdx].start + dragDelta; + if (this.isSnapping) { + const thresholdFrames = (15 / logicalWidth) * totalFrames; + let bestStart = newStart; + let minDiff = thresholdFrames; + + const snapCandidates = [0, this.getDurationFrames(), this.currentFrame]; + // Add start and end frames of active generation range + snapCandidates.push(this.getStartFrames()); + if (this.endFramesWidget && this.endFramesWidget.value !== undefined) { + snapCandidates.push(parseInt(this.endFramesWidget.value, 10)); + } + const allTracks = [ + this.timeline.segments || [], + this.timeline.motionSegments || [], + this.timeline.audioSegments || [] + ]; + const ignoreSegmentIds = [String(this._dragTargetId)]; + const isVid = String(this._dragTargetId).endsWith("_v"); + const isAud = String(this._dragTargetId).endsWith("_a"); + if (isVid || isAud) { + const siblingId = isVid ? String(this._dragTargetId).slice(0, -2) + "_a" : String(this._dragTargetId).slice(0, -2) + "_v"; + ignoreSegmentIds.push(siblingId); + } + + for (const track of allTracks) { + for (const seg of track) { + if (ignoreSegmentIds.includes(String(seg.id))) continue; + snapCandidates.push(seg.start); + snapCandidates.push(seg.start + seg.length); + } + } + + for (const candidate of snapCandidates) { + const diff = Math.abs(newStart - candidate); + if (diff < minDiff) { + minDiff = diff; + bestStart = candidate; + } + } + newStart = bestStart; + dragDelta = newStart - t[targetIdx].start; + } let minPossibleStart = 0; let prevSeg = t.slice().reverse().find(s => s.start + s.length <= t[targetIdx].start && s.id !== t[targetIdx].id); if (prevSeg) { minPossibleStart = prevSeg.start + prevSeg.length; } - if (this.selectionType === "audio") { + // Check sibling track obstacles if linked + const isVid = String(this._dragTargetId).endsWith("_v"); + const isAud = String(this._dragTargetId).endsWith("_a"); + const siblingId = (isVid || isAud) ? (isVid ? String(this._dragTargetId).slice(0, -2) + "_a" : String(this._dragTargetId).slice(0, -2) + "_v") : null; + if (siblingId && this._dragInitialSiblingTimeline) { + let prevSibSeg = this._dragInitialSiblingTimeline.slice().reverse().find(s => s.start + s.length <= t[targetIdx].start && s.id !== siblingId); + if (prevSibSeg) { + let sibMinPossible = prevSibSeg.start + prevSibSeg.length; + minPossibleStart = Math.max(minPossibleStart, sibMinPossible); + } + } + + if (this.selectionType === "audio" || t[targetIdx].type === "video" || t[targetIdx].type === "motion_video") { minPossibleStart = Math.max(minPossibleStart, t[targetIdx].start - (t[targetIdx].trimStart || 0)); } @@ -2658,7 +8149,7 @@ class TimelineEditor { let diff = newStart - t[targetIdx].start; t[targetIdx].start = newStart; t[targetIdx].length -= diff; - if (this.selectionType === "audio") { + if (this.selectionType === "audio" || t[targetIdx].type === "video" || t[targetIdx].type === "motion_video") { t[targetIdx].trimStart += diff; } @@ -2666,22 +8157,151 @@ class TimelineEditor { let initT = this._dragInitialTimeline; let dIdx = initT.findIndex(s => s.id === this._dragTargetId); if (dIdx < 0) return; - let D = JSON.parse(JSON.stringify(initT[dIdx])); + let D = { ...initT[dIdx] }; let D_mouse_start = D.start + dragDelta; let mouseFrameX = mouseX * (totalFrames / logicalWidth); + if (this.isSnapping) { + const thresholdFrames = (15 / logicalWidth) * totalFrames; + let bestStart = D_mouse_start; + let minDiff = thresholdFrames; + + const snapCandidates = [0, this.getDurationFrames(), this.currentFrame]; + // Add start and end frames of active generation range + snapCandidates.push(this.getStartFrames()); + if (this.endFramesWidget && this.endFramesWidget.value !== undefined) { + snapCandidates.push(parseInt(this.endFramesWidget.value, 10)); + } + const allTracks = [ + this.timeline.segments || [], + this.timeline.motionSegments || [], + this.timeline.audioSegments || [] + ]; + const ignoreSegmentIds = [String(this._dragTargetId)]; + const isVid = String(this._dragTargetId).endsWith("_v"); + const isAud = String(this._dragTargetId).endsWith("_a"); + if (isVid || isAud) { + const siblingId = isVid ? String(this._dragTargetId).slice(0, -2) + "_a" : String(this._dragTargetId).slice(0, -2) + "_v"; + ignoreSegmentIds.push(siblingId); + } + + for (const track of allTracks) { + for (const seg of track) { + if (ignoreSegmentIds.includes(String(seg.id))) continue; + snapCandidates.push(seg.start); + snapCandidates.push(seg.start + seg.length); + } + } + + for (const candidate of snapCandidates) { + // Check start snap + const diffStart = Math.abs(D_mouse_start - candidate); + if (diffStart < minDiff) { + minDiff = diffStart; + bestStart = candidate; + } + // Check end snap + const diffEnd = Math.abs((D_mouse_start + D.length) - candidate); + if (diffEnd < minDiff) { + minDiff = diffEnd; + bestStart = candidate - D.length; + } + } + const rawStart = D_mouse_start; + D_mouse_start = bestStart; + const snapOffset = D_mouse_start - rawStart; + dragDelta = D_mouse_start - D.start; + mouseFrameX += snapOffset; + } + t = this._applyCenterDragPhysics(initT, D.id, D_mouse_start, mouseFrameX, durationFrames, totalFrames, logicalWidth); + + if (this._dragInitialSiblingTimeline) { + let siblingPhysics = null; + + if (this._dragTargetId.endsWith("_v") || this._dragTargetId.endsWith("_a")) { + const isVid = this._dragTargetId.endsWith("_v"); + const siblingId = isVid ? this._dragTargetId.slice(0, -2) + "_a" : this._dragTargetId.slice(0, -2) + "_v"; + siblingPhysics = this._applyCenterDragPhysics(this._dragInitialSiblingTimeline, siblingId, D_mouse_start, mouseFrameX, durationFrames, totalFrames, logicalWidth); + + // Ensure initial sync for the dragged segment so the solver starts from a good state + const activeFinal = t.find(s => s.id === this._dragTargetId); + const siblingFinal = siblingPhysics.find(s => s.id === siblingId); + + if (activeFinal && siblingFinal && activeFinal.start !== siblingFinal.start) { + const origStart = D.start; + const activeDelta = Math.abs(activeFinal.start - origStart); + const siblingDelta = Math.abs(siblingFinal.start - origStart); + const finalStart = activeDelta < siblingDelta ? activeFinal.start : siblingFinal.start; + + const finalMouseX = finalStart + D.length / 2; + t = this._applyCenterDragPhysics(initT, D.id, finalStart, finalMouseX, durationFrames, totalFrames, logicalWidth, true); + siblingPhysics = this._applyCenterDragPhysics(this._dragInitialSiblingTimeline, siblingId, finalStart, finalMouseX, durationFrames, totalFrames, logicalWidth, true); + } + } else { + siblingPhysics = this._dragInitialSiblingTimeline.map(s => ({ ...s })); + } + + // Resolve all secondary pushes to keep linked clips together + this._resolveGlobalPhysics(t, siblingPhysics, durationFrames, initT, this._dragInitialSiblingTimeline); + this._previewSiblingSegments = siblingPhysics; + } } } + const targetArray = this.getSegmentArray(this.selectionType); + this._restoreTransientProperties(t, targetArray); + + if (this._dragType === "left") { + this._liveScrubVideo(t.find(s => s.id === this._dragTargetId), "start"); + } else if (this._dragType === "right") { + this._liveScrubVideo(t.find(s => s.id === this._dragTargetId), "end"); + } else if (this._dragType === "joint") { + this._liveScrubVideo(t.find(s => s.id === this._dragTargetId), "end"); + this._liveScrubVideo(t.find(s => s.id === this._dragTargetIdRight), "start"); + } + + const syncSibling = (targetId, activeArray) => { + if (!targetId || this._dragType === "center") return; // Center drag handles physics separately above + const isVid = targetId.endsWith("_v"); + const isAud = targetId.endsWith("_a"); + if (!isVid && !isAud) return; + + const siblingId = isVid ? targetId.slice(0, -2) + "_a" : targetId.slice(0, -2) + "_v"; + if (!this._previewSiblingSegments) { + this._previewSiblingSegments = this._dragInitialSiblingTimeline.map(s => ({ ...s })); + } + const sibling = this._previewSiblingSegments.find(s => s.id === siblingId); + const active = activeArray.find(s => s.id === targetId); + + if (sibling && active) { + sibling.start = active.start; + sibling.length = active.length; + if (active.trimStart !== undefined) sibling.trimStart = active.trimStart; + } + }; + + syncSibling(this._dragTargetId, t); + if (this._dragType === "joint") syncSibling(this._dragTargetIdRight, t); + this._previewSegments = t; + + if (this._previewSiblingSegments) { + let siblingArray = null; + if (this.selectionType === "audio") siblingArray = this.timeline.segments; + else if (this.selectionType === "image") siblingArray = this.timeline.audioSegments; + if (siblingArray) { + this._restoreTransientProperties(this._previewSiblingSegments, siblingArray); + } + } + this.updateUIFromSelection(); // Live update of trim values this.render(); } - _applyCenterDragPhysics(initT, D_id, D_mouse_start, mouseFrameX, durationFrames, totalFrames, logicalWidth) { - let t_copy = JSON.parse(JSON.stringify(initT)); + _applyCenterDragPhysics(initT, D_id, D_mouse_start, mouseFrameX, durationFrames, totalFrames, logicalWidth, forceStart = false) { + let t_copy = initT.map(s => ({ ...s })); let dIdx = t_copy.findIndex(s => s.id === D_id); if (dIdx < 0) return t_copy; @@ -2699,14 +8319,16 @@ class TimelineEditor { } } - let leftBound = insertIdx > 0 ? baseSegments[insertIdx - 1].start + baseSegments[insertIdx - 1].length : 0; - let rightBound = insertIdx < baseSegments.length ? baseSegments[insertIdx].start : durationFrames; + if (!forceStart) { + let leftBound = insertIdx > 0 ? baseSegments[insertIdx - 1].start + baseSegments[insertIdx - 1].length : 0; + let rightBound = insertIdx < baseSegments.length ? baseSegments[insertIdx].start : durationFrames; - if (rightBound - leftBound >= D.length) { - D_clamped_start = clamp(D_clamped_start, leftBound, rightBound - D.length); - } else { - let gapCenter = (leftBound + rightBound) / 2; - D_clamped_start = gapCenter - D.length / 2; + if (rightBound - leftBound >= D.length) { + D_clamped_start = clamp(D_clamped_start, leftBound, rightBound - D.length); + } else { + let gapCenter = (leftBound + rightBound) / 2; + D_clamped_start = gapCenter - D.length / 2; + } } let t_test = []; @@ -2759,17 +8381,268 @@ class TimelineEditor { return result; } + _resolveGlobalPhysics(activeTimeline, siblingTimeline, durationFrames, activeInitial, siblingInitial) { + if (!siblingTimeline) return; + + let changed = true; + let iters = 0; + while (changed && iters < 10) { + changed = false; + iters++; + + let syncedActiveIndices = []; + let syncedSiblingIndices = []; + + // 1. Sync linked clips + for (let i = 0; i < activeTimeline.length; i++) { + let seg = activeTimeline[i]; + if (seg.id.endsWith("_v") || seg.id.endsWith("_a")) { + const isVid = seg.id.endsWith("_v"); + const sibId = isVid ? seg.id.slice(0, -2) + "_a" : seg.id.slice(0, -2) + "_v"; + let sibIndex = siblingTimeline.findIndex(s => s.id === sibId); + + if (sibIndex >= 0) { + let sib = siblingTimeline[sibIndex]; + if (sib.start !== seg.start) { + let origStart = seg.start; + if (activeInitial) { + const origSeg = activeInitial.find(s => s.id === seg.id); + if (origSeg) origStart = origSeg.start; + } + + let sibOrigStart = sib.start; + if (siblingInitial) { + const origSib = siblingInitial.find(s => s.id === sib.id); + if (origSib) sibOrigStart = origSib.start; + } + + const dSeg = Math.abs(seg.start - origStart); + const dSib = Math.abs(sib.start - sibOrigStart); + + // The segment that was pushed furthest dictates the new position + const targetStart = dSeg > dSib ? seg.start : sib.start; + + if (seg.start !== targetStart) { + seg.start = targetStart; + changed = true; + syncedActiveIndices.push(i); + } + if (sib.start !== targetStart) { + sib.start = targetStart; + changed = true; + syncedSiblingIndices.push(sibIndex); + } + } + } + } + } + + // 2. Resolve overlaps on both tracks by pushing outward from epicenters + if (changed) { + const sweepTrack = (track, epicenterIndices) => { + let didChange = false; + + for (let epIndex of epicenterIndices) { + // Push elements to the right of the epicenter + for (let i = epIndex + 1; i < track.length; i++) { + let prev = track[i - 1]; + let targetStart = prev.start + prev.length; + if (track[i].start < targetStart) { + track[i].start = targetStart; + didChange = true; + } + } + // Push elements to the left of the epicenter + for (let i = epIndex - 1; i >= 0; i--) { + let next = track[i + 1]; + let targetStart = next.start - track[i].length; + if (track[i].start > targetStart) { + track[i].start = targetStart; + didChange = true; + } + } + } + + // Boundary clamping to ensure nothing falls off the edges + let rightCursor = durationFrames; + for (let i = track.length - 1; i >= 0; i--) { + if (track[i].start + track[i].length > rightCursor) { + let newStart = rightCursor - track[i].length; + if (track[i].start !== newStart) { track[i].start = newStart; didChange = true; } + } + rightCursor = track[i].start; + } + + let leftCursor = 0; + for (let i = 0; i < track.length; i++) { + if (track[i].start < leftCursor) { + let newStart = leftCursor; + if (track[i].start !== newStart) { track[i].start = newStart; didChange = true; } + } + leftCursor = track[i].start + track[i].length; + } + return didChange; + }; + + sweepTrack(activeTimeline, syncedActiveIndices); + sweepTrack(siblingTimeline, syncedSiblingIndices); + } + } + } + + _restoreTransientProperties(copiedSegs, originalSegs) { + if (!copiedSegs || !originalSegs) return; + for (let ps of copiedSegs) { + const orig = originalSegs.find(s => s.id === ps.id); + if (orig) { + if (orig._uploading !== undefined) ps._uploading = orig._uploading; + if (orig._decoding !== undefined) ps._decoding = orig._decoding; + if (orig._blobUrl !== undefined) ps._blobUrl = orig._blobUrl; + if (orig._audioBuffer !== undefined) ps._audioBuffer = orig._audioBuffer; + if (orig.imgObj !== undefined) ps.imgObj = orig.imgObj; + if (orig.videoEl !== undefined) ps.videoEl = orig.videoEl; + if (orig.thumbnails !== undefined) ps.thumbnails = orig.thumbnails; + if (orig._extractingThumbs !== undefined) ps._extractingThumbs = orig._extractingThumbs; + } + } + } + onMouseUp(e) { document.body.style.userSelect = ""; + document.body.style.cursor = ""; + + if (e.button === 2 && this.retakeMode) { + e.preventDefault(); + e.stopPropagation(); + return; + } + + if (this.retakeMode) { + if (this._isDragging) { + const wasPlayheadDrag = this._dragType === "playhead"; + const wasPlaying = this._retakeScrubWasPlaying; + this._retakeScrubWasPlaying = false; + if (this.timeline.retakeVideo && this.timeline.retakeVideo._scrubTargetSec !== undefined) { + if (this.timeline.retakeVideo.videoEl) { + this.timeline.retakeVideo.videoEl.currentTime = this.timeline.retakeVideo._scrubTargetSec; + } + delete this.timeline.retakeVideo._scrubTargetSec; + } + this._isDragging = false; + this._dragType = null; + this.canvas.style.cursor = "default"; + this.commitChanges(); + // If playback was active before the scrub, resume from the new scrub position + if (wasPlayheadDrag && wasPlaying) { + this.playAudio(); + } else { + this.render(); + } + } + return; + } + + // Commit scrub target to actual video element so it's ready for playback + const commitScrub = (segs) => { + if (!segs) return; + for (const seg of segs) { + if (seg._scrubTargetSec !== undefined) { + if (seg.videoEl) seg.videoEl.currentTime = seg._scrubTargetSec; + delete seg._scrubTargetSec; + } + } + }; + + commitScrub(this.timeline.segments); + commitScrub(this.timeline.motionSegments); + commitScrub(this._previewSegments); + commitScrub(this._previewSiblingSegments); + if (this._multiDragPreviewTimelines) { + commitScrub(this._multiDragPreviewTimelines.image); + commitScrub(this._multiDragPreviewTimelines.motion); + } + if (this._isDragging) { - if (this._previewSegments) { - const targetArray = this.selectionType === "audio" ? this.timeline.audioSegments : this.timeline.segments; + if (this._dragType === "box_select") { + this._isSelectingBox = false; + this._selectBoxStart = null; + this._selectBoxCurrent = null; + this._selectBoxInitialSelectedIds = null; + this._isDragging = false; + this.canvas.style.cursor = "default"; + this.updateUIFromSelection(); + this.render(); + this.commitChanges(); + return; + } + + if (this._multiDragPreviewTimelines) { + if (this._multiDragPreviewTimelines.image) { + this.timeline.segments = this._multiDragPreviewTimelines.image.map(ps => { + const orig = this.timeline.segments.find(s => s.id === ps.id); + if (orig) { + if (orig.imgObj) ps.imgObj = orig.imgObj; + if (orig.videoEl) ps.videoEl = orig.videoEl; + if (orig.thumbnails) ps.thumbnails = orig.thumbnails; + if (orig._extractingThumbs !== undefined) ps._extractingThumbs = orig._extractingThumbs; + if (orig._uploading !== undefined) ps._uploading = orig._uploading; + if (orig._decoding !== undefined) ps._decoding = orig._decoding; + if (orig._blobUrl !== undefined) ps._blobUrl = orig._blobUrl; + if (orig._audioBuffer !== undefined) ps._audioBuffer = orig._audioBuffer; + } + return ps; + }); + } + if (this._multiDragPreviewTimelines.motion) { + this.timeline.motionSegments = this._multiDragPreviewTimelines.motion.map(ps => { + const orig = this.timeline.motionSegments.find(s => s.id === ps.id); + if (orig) { + if (orig.imgObj) ps.imgObj = orig.imgObj; + if (orig.videoEl) ps.videoEl = orig.videoEl; + if (orig.thumbnails) ps.thumbnails = orig.thumbnails; + if (orig._extractingThumbs !== undefined) ps._extractingThumbs = orig._extractingThumbs; + if (orig._uploading !== undefined) ps._uploading = orig._uploading; + if (orig._decoding !== undefined) ps._decoding = orig._decoding; + if (orig._blobUrl !== undefined) ps._blobUrl = orig._blobUrl; + if (orig._audioBuffer !== undefined) ps._audioBuffer = orig._audioBuffer; + } + return ps; + }); + } + if (this._multiDragPreviewTimelines.audio) { + this.timeline.audioSegments = this._multiDragPreviewTimelines.audio.map(ps => { + const orig = this.timeline.audioSegments.find(s => s.id === ps.id); + if (orig) { + if (orig.imgObj) ps.imgObj = orig.imgObj; + if (orig.videoEl) ps.videoEl = orig.videoEl; + if (orig.thumbnails) ps.thumbnails = orig.thumbnails; + if (orig._extractingThumbs !== undefined) ps._extractingThumbs = orig._extractingThumbs; + if (orig._uploading !== undefined) ps._uploading = orig._uploading; + if (orig._decoding !== undefined) ps._decoding = orig._decoding; + if (orig._blobUrl !== undefined) ps._blobUrl = orig._blobUrl; + if (orig._audioBuffer !== undefined) ps._audioBuffer = orig._audioBuffer; + } + return ps; + }); + } + this._multiDragPreviewTimelines = null; + } else if (this._previewSegments) { + const targetArray = this.getSegmentArray(this.selectionType); const mappedArray = this._previewSegments.map(ps => { const orig = targetArray.find(s => s.id === ps.id); let finalStart = ps.resolvedStart !== undefined ? ps.resolvedStart : ps.start; let newPs = { ...ps, start: finalStart }; - if (orig && orig.imgObj) newPs.imgObj = orig.imgObj; + if (orig) { + if (orig.imgObj) newPs.imgObj = orig.imgObj; + if (orig.videoEl) newPs.videoEl = orig.videoEl; + if (orig.thumbnails) newPs.thumbnails = orig.thumbnails; + if (orig._extractingThumbs !== undefined) newPs._extractingThumbs = orig._extractingThumbs; + if (orig._uploading !== undefined) newPs._uploading = orig._uploading; + if (orig._decoding !== undefined) newPs._decoding = orig._decoding; + if (orig._blobUrl !== undefined) newPs._blobUrl = orig._blobUrl; + if (orig._audioBuffer !== undefined) newPs._audioBuffer = orig._audioBuffer; + } delete newPs.resolvedStart; return newPs; }); @@ -2777,15 +8650,76 @@ class TimelineEditor { if (this.selectionType === "audio") { this.timeline.audioSegments = mappedArray; if (this._dragTargetId) this.selectedIndex = this.timeline.audioSegments.findIndex(s => s.id === this._dragTargetId); + } else if (this.selectionType === "motion") { + this.timeline.motionSegments = mappedArray; + if (this._dragTargetId) this.selectedIndex = this.timeline.motionSegments.findIndex(s => s.id === this._dragTargetId); } else { this.timeline.segments = mappedArray; if (this._dragTargetId) this.selectedIndex = this.timeline.segments.findIndex(s => s.id === this._dragTargetId); } } + if (this._previewSiblingSegments) { + let siblingArray = null; + if (this.selectionType === "audio") siblingArray = this.timeline.segments; + else if (this.selectionType === "image") siblingArray = this.timeline.audioSegments; + + if (siblingArray) { + const mappedSibling = this._previewSiblingSegments.map(ps => { + const orig = siblingArray.find(s => s.id === ps.id); + let finalStart = ps.resolvedStart !== undefined ? ps.resolvedStart : ps.start; + let newPs = { ...ps, start: finalStart }; + if (orig) { + if (orig.imgObj) newPs.imgObj = orig.imgObj; + if (orig.videoEl) newPs.videoEl = orig.videoEl; + if (orig.thumbnails) newPs.thumbnails = orig.thumbnails; + if (orig._extractingThumbs !== undefined) newPs._extractingThumbs = orig._extractingThumbs; + if (orig._uploading !== undefined) newPs._uploading = orig._uploading; + if (orig._decoding !== undefined) newPs._decoding = orig._decoding; + if (orig._blobUrl !== undefined) newPs._blobUrl = orig._blobUrl; + if (orig._audioBuffer !== undefined) newPs._audioBuffer = orig._audioBuffer; + } + delete newPs.resolvedStart; + return newPs; + }); + + if (this.selectionType === "audio") this.timeline.segments = mappedSibling; + else if (this.selectionType === "image") this.timeline.audioSegments = mappedSibling; + } + } + + if (this._multiDragClickPendingDeselect && !this._isMultiDraggingAndMoved) { + const clickedId = this._multiDragClickPendingDeselect; + this.selectedSegmentIds = [clickedId]; + const sibId = clickedId.endsWith("_v") ? clickedId.slice(0, -2) + "_a" : (clickedId.endsWith("_a") ? clickedId.slice(0, -2) + "_v" : null); + if (sibId && !this.selectedSegmentIds.includes(sibId)) this.selectedSegmentIds.push(sibId); + + let foundIdx = -1; + let foundTrack = "image"; + for (const track of ["image", "motion", "audio"]) { + const arr = this.getSegmentArray(track); + const idx = arr.findIndex(s => s.id === clickedId); + if (idx !== -1) { + foundIdx = idx; + foundTrack = track; + break; + } + } + if (foundIdx !== -1) { + this.selectionType = foundTrack; + this.selectedIndex = foundIdx; + } + this.updateUIFromSelection(); + } + this._isDragging = false; this._previewSegments = null; + this._previewSiblingSegments = null; this._ghostTrack = null; + this._isMultiDraggingAndMoved = false; + this._multiDragClickPendingDeselect = null; + this._multiDragInitialSegments = null; + this._multiDragPreviewTimelines = null; this.canvas.style.cursor = "default"; this.commitChanges(); } @@ -2793,68 +8727,214 @@ class TimelineEditor { // --- Backend Data Sync --- commitChanges(skipRender = false) { + if (this._suppressCommit) return; + // Deduplicate segments by ID to clean up any duplicates created by the previous onseeked bug + this.timeline.segments = this.timeline.segments.filter((seg, index, self) => index === self.findIndex((s) => s.id === seg.id)); + if (this.timeline.audioSegments) { + this.timeline.audioSegments = this.timeline.audioSegments.filter((seg, index, self) => index === self.findIndex((s) => s.id === seg.id)); + } + if (this.timeline.motionSegments) { + this.timeline.motionSegments = this.timeline.motionSegments.filter((seg, index, self) => index === self.findIndex((s) => s.id === seg.id)); + } + let sortedSegments = [...this.timeline.segments].sort((a, b) => a.start - b.start); let contiguousLengths = []; let contiguousPrompts = []; - let currentCursor = 0; + let imgStrengths = []; + + const startFrames = this.getStartFrames(); const durationFrames = this.getDurationFrames(); + if (!this.retakeMode) { + this.timeline.normalStartFrame = startFrames; + this.timeline.normalDurationFrames = durationFrames; + } + const endFrames = startFrames + durationFrames; + let currentCursor = startFrames; - // Build segment lengths clipped at the duration cutoff. - // - Gaps before the first segment, or between segments, are absorbed into the adjacent - // segment's length (same as before), but are also clipped at durationFrames. - // - Segments that start at or past the cutoff are excluded entirely. - // - Segments that cross the cutoff are trimmed so their end = durationFrames exactly. - let pendingGap = 0; - for (let seg of sortedSegments) { - // Skip segments entirely outside the duration. - if (seg.start >= durationFrames) break; + if (this.retakeMode) { + const totalFrames = this.getVisualDurationFrames(); + const retakeStart = this.timeline.retakeStart ?? 0; + const retakeLength = this.timeline.retakeLength ?? totalFrames; + const retakeEnd = retakeStart + retakeLength; + const retakePrompt = this.timeline.retakePrompt || ""; + const retakeStrength = this.timeline.retakeStrength ?? 1.0; + const globalPrompt = this.globalPromptInput ? this.globalPromptInput.value : (this.node.properties?.global_prompt || ""); - if (seg.start > currentCursor) { - // Gap between the cursor and this segment — clip it at the cutoff too. - const gapLength = Math.min(seg.start, durationFrames) - currentCursor; - if (contiguousLengths.length > 0) { - contiguousLengths[contiguousLengths.length - 1] += gapLength; - } else { - pendingGap += gapLength; - } + // 1. Preserved before + const pBeforeStart = startFrames; + const pBeforeEnd = Math.min(endFrames, retakeStart); + const pBeforeLen = pBeforeEnd - pBeforeStart; + if (pBeforeLen > 0) { + contiguousLengths.push(pBeforeLen); + contiguousPrompts.push(globalPrompt || "video"); + imgStrengths.push("0.00"); } - // Clip segment end at the duration cutoff. - const clippedEnd = Math.min(seg.start + seg.length, durationFrames); - const clippedLength = clippedEnd - seg.start; + // 2. Retake region + const rStart = Math.max(startFrames, retakeStart); + const rEnd = Math.min(endFrames, retakeEnd); + const rLen = rEnd - rStart; + if (rLen > 0) { + contiguousLengths.push(rLen); + contiguousPrompts.push(retakePrompt || "video"); + imgStrengths.push(retakeStrength.toFixed(2)); + } - contiguousLengths.push(clippedLength + pendingGap); - contiguousPrompts.push(seg.prompt || ""); - pendingGap = 0; - currentCursor = seg.start + seg.length; // advance by the real (unclipped) end for gap detection - } + // 3. Preserved after + const pAfterStart = Math.max(startFrames, retakeEnd); + const pAfterEnd = endFrames; + const pAfterLen = pAfterEnd - pAfterStart; + if (pAfterLen > 0) { + contiguousLengths.push(pAfterLen); + contiguousPrompts.push(globalPrompt || "video"); + imgStrengths.push("0.00"); + } + } else { + // Build segment lengths clipped at the duration cutoff. + // - Gaps before the first segment, or between segments, are absorbed into the adjacent + // segment's length (same as before), but are also clipped at endFrames. + // - Segments completely before startFrames or after endFrames are excluded entirely. + // - Segments that cross the boundaries are trimmed. + let pendingGap = 0; + for (let seg of sortedSegments) { + if (seg.start + seg.length <= startFrames) continue; + if (seg.start >= endFrames) break; - // If segments don't fill to the end of the duration, pad the last segment to reach it. - const clampedCursor = Math.min(currentCursor, durationFrames); - if (contiguousLengths.length > 0 && clampedCursor < durationFrames) { - contiguousLengths[contiguousLengths.length - 1] += durationFrames - clampedCursor; + const effectiveStart = Math.max(seg.start, startFrames); + + if (effectiveStart > currentCursor) { + const gapLength = Math.min(effectiveStart, endFrames) - currentCursor; + if (contiguousLengths.length > 0) { + contiguousLengths[contiguousLengths.length - 1] += gapLength; + } else { + pendingGap += gapLength; + } + } + + const clippedEnd = Math.min(seg.start + seg.length, endFrames); + const clippedLength = clippedEnd - effectiveStart; + + contiguousLengths.push(clippedLength + pendingGap); + contiguousPrompts.push(seg.prompt || ""); + pendingGap = 0; + currentCursor = Math.max(currentCursor, seg.start + seg.length); + } + + const clampedCursor = Math.min(currentCursor, endFrames); + if (contiguousLengths.length > 0 && clampedCursor < endFrames) { + contiguousLengths[contiguousLengths.length - 1] += endFrames - clampedCursor; + } } const toSave = { - segments: sortedSegments.map(s => serializeTimelineSegment(s)), - audioSegments: (this.timeline.audioSegments || []).map(s => serializeTimelineSegment(s)) + mainTrackEnabled: this.mainTrackEnabled, + audioTrackEnabled: this.audioTrackEnabled, + motionTrackEnabled: this.motionTrackEnabled, + propHeight: this.propHeight, + globalPropHeight: this.globalPropHeight, + showFilenames: !!this.node.properties.showFilenames, + overrideAudio: !!this.node.properties.overrideAudio, + inpaint_audio: !!(this.node.widgets?.find(w => w.name === "inpaint_audio")?.value), + global_prompt: this.retakeMode ? (this.timeline.global_prompt || "") : (this.globalPromptInput ? this.globalPromptInput.value : ""), + retake_global_prompt: this.retakeMode ? (this.globalPromptInput ? this.globalPromptInput.value : "") : (this.timeline.retake_global_prompt || ""), + retakeMode: this.retakeMode, + retakeStart: this.timeline.retakeStart, + retakeLength: this.timeline.retakeLength, + retakePrompt: this.timeline.retakePrompt, + retakeStrength: this.timeline.retakeStrength, + retakeVideo: this.timeline.retakeVideo ? { + fileName: this.timeline.retakeVideo.fileName, + imageFile: this.timeline.retakeVideo.imageFile, + videoDurationFrames: this.timeline.retakeVideo.videoDurationFrames, + fileSize: this.timeline.retakeVideo.fileSize, + } : null, + normalStartFrame: this.timeline.normalStartFrame, + normalDurationFrames: this.timeline.normalDurationFrames, + segments: sortedSegments.map(s => { + const { imgObj, videoEl, _isSeeking, thumbnails, _extractingThumbs, _sSecs, _lSecs, _tSecs, _dSecs, _uploading, _blobUrl, ...rest } = s; + return rest; + }), + motionSegments: (this.timeline.motionSegments || []).map(s => { + const { imgObj, videoEl, _isSeeking, thumbnails, _extractingThumbs, _sSecs, _lSecs, _tSecs, _dSecs, _uploading, _blobUrl, ...rest } = s; + return rest; + }), + audioSegments: (this.timeline.audioSegments || []).map(s => { + const { _sSecs, _lSecs, _tSecs, _dSecs, _uploading, _decoding, _blobUrl, _audioBuffer, ...rest } = s; + return rest; + }) }; const jsonStr = JSON.stringify(toSave); - if (this.timelineDataWidget) this.timelineDataWidget.value = jsonStr; + console.log("[LTXDirector debug] commitChanges: saving timelineDataWidget value:", jsonStr); + + const updateWidgetValue = (w, val) => { + if (!w) return; + const oldVal = w.value; + w.value = val; + if (this.node) { + if (this.node.properties) { + this.node.properties[w.name] = val; + } + if (this.node.onWidgetChanged) { + this.node.onWidgetChanged(w.name, val, oldVal, w); + } + } + if (w.callback) { + try { + w.callback(val); + } catch (e) { + // ignore + } + } + }; + + if (this.timelineDataWidget) { + updateWidgetValue(this.timelineDataWidget, jsonStr); + } + + if (this.node.properties) { + this.node.properties.mainTrackEnabled = this.mainTrackEnabled; + this.node.properties.audioTrackEnabled = this.audioTrackEnabled; + this.node.properties.motionTrackEnabled = this.motionTrackEnabled; + this.node.properties.audioTrackWasEnabledBeforeOverride = !!this._audioTrackWasEnabledBeforeOverride; + + if (this.node.widgets) { + for (const w of this.node.widgets) { + if (w.name && w.value !== undefined) { + this.node.properties[w.name] = w.value; + } + } + } + const overrideWidget = this.node.widgets?.find(w => w.name === "override_audio"); + if (overrideWidget) { + this.node.properties.overrideAudio = !!overrideWidget.value; + } + } + + const overrideWidget = this.node.widgets?.find(w => w.name === "override_audio"); + if (overrideWidget) { + updateWidgetValue(overrideWidget, !!this.node.properties.overrideAudio); + } if (this.localPromptsWidget) { - this.localPromptsWidget.value = contiguousPrompts.join(" | "); + updateWidgetValue(this.localPromptsWidget, contiguousPrompts.join(" | ")); } if (this.segmentLengthsWidget) { - this.segmentLengthsWidget.value = contiguousLengths.join(","); + updateWidgetValue(this.segmentLengthsWidget, contiguousLengths.join(",")); } if (this.guideStrengthWidget) { - const imgStrengths = sortedSegments - .filter(s => s.type !== "text") - .map(s => (s.guideStrength !== undefined ? s.guideStrength : 1.0).toFixed(2)); - this.guideStrengthWidget.value = imgStrengths.join(","); + let val = ""; + if (this.retakeMode) { + val = imgStrengths.join(","); + } else { + const strList = sortedSegments + .filter(s => s.type !== "text") + .filter(s => s.start + s.length > startFrames && s.start < endFrames) + .map(s => (s.guideStrength !== undefined ? s.guideStrength : 1.0).toFixed(2)); + val = strList.join(","); + } + updateWidgetValue(this.guideStrengthWidget, val); } // Keep zoom slider max in sync with the current timeline duration. @@ -2864,17 +8944,93 @@ class TimelineEditor { if (this.node && this.node.computeSize) { const sz = this.node.computeSize(); this.node.size[1] = sz[1]; - if (app.graph) app.graph.setDirtyCanvas(true, true); + if (app.graph) { + app.graph.setDirtyCanvas(true, true); + if (app.graph.change) app.graph.change(); + if (app.graph.onNodeChanged) app.graph.onNodeChanged(this.node); + if (app.graph.onStateChanged) app.graph.onStateChanged(); + } } - }, 0); + try { + const canvasEl = app.canvasEl || app.canvas?.canvas; + if (canvasEl) { + canvasEl.dispatchEvent(new PointerEvent('pointerup', { bubbles: true, cancelable: true })); + } + if (app.canvas && app.canvas.checkState) app.canvas.checkState(); + if (app.canvas && app.canvas.captureCanvasState) app.canvas.captureCanvasState(); + } catch (_) { } + }, 100); + + // Stamp exact seconds on every live segment so FPS changes can recompute + // frame values without cumulative rounding error. + this._stampSegmentSeconds(); + + if (this.isPlaying) { + this.playAudio(); // Resync audio engine with new timeline data + } if (!skipRender) this.render(); } + // Stamp _sSecs / _lSecs / _tSecs / _dSecs on every live segment + // using the current frame rate. Call this whenever segments change + // through normal timeline interactions (not FPS changes). + _stampSegmentSeconds() { + const fps = this.getFrameRate(); + if (fps <= 0) return; + for (const seg of this.timeline.segments) { + seg._sSecs = seg.start / fps; + seg._lSecs = seg.length / fps; + if (seg.trimStart !== undefined) seg._tSecs = seg.trimStart / fps; + if (seg.videoDurationFrames !== undefined) seg._dSecs = seg.videoDurationFrames / fps; + } + for (const seg of this.timeline.audioSegments) { + seg._sSecs = seg.start / fps; + seg._lSecs = seg.length / fps; + if (seg.trimStart !== undefined) seg._tSecs = seg.trimStart / fps; + if (seg.audioDurationFrames !== undefined) seg._dSecs = seg.audioDurationFrames / fps; + } + } + + // Recompute all segment frame values from their seconds snapshots at `newFPS`. + // If a segment has no snapshot yet (e.g. freshly added), fall back to scaling + // from the previous FPS so it still moves correctly. + _rebaseSegmentsToFPS(newFPS) { + if (newFPS <= 0) return; + const oldFPS = this._prevFrameRate || newFPS; + const fallbackRatio = oldFPS > 0 ? newFPS / oldFPS : 1; + for (const seg of this.timeline.segments) { + if (seg._sSecs !== undefined) { + seg.start = Math.round(seg._sSecs * newFPS); + seg.length = Math.max(1, Math.round(seg._lSecs * newFPS)); + if (seg._tSecs !== undefined) seg.trimStart = Math.round(seg._tSecs * newFPS); + if (seg._dSecs !== undefined) seg.videoDurationFrames = Math.round(seg._dSecs * newFPS); + } else { + seg.start = Math.round(seg.start * fallbackRatio); + seg.length = Math.max(1, Math.round(seg.length * fallbackRatio)); + if (seg.trimStart !== undefined) seg.trimStart = Math.round(seg.trimStart * fallbackRatio); + if (seg.videoDurationFrames !== undefined) seg.videoDurationFrames = Math.round(seg.videoDurationFrames * fallbackRatio); + } + } + for (const seg of this.timeline.audioSegments) { + if (seg._sSecs !== undefined) { + seg.start = Math.round(seg._sSecs * newFPS); + seg.length = Math.max(1, Math.round(seg._lSecs * newFPS)); + if (seg._tSecs !== undefined) seg.trimStart = Math.round(seg._tSecs * newFPS); + if (seg._dSecs !== undefined) seg.audioDurationFrames = Math.round(seg._dSecs * newFPS); + } else { + seg.start = Math.round(seg.start * fallbackRatio); + seg.length = Math.max(1, Math.round(seg.length * fallbackRatio)); + if (seg.trimStart !== undefined) seg.trimStart = Math.round(seg.trimStart * fallbackRatio); + if (seg.audioDurationFrames !== undefined) seg.audioDurationFrames = Math.round(seg.audioDurationFrames * fallbackRatio); + } + } + } + // --- Gap Region Calculation --- getGapRegions() { const totalFrames = this.getVisualDurationFrames(); - const outputFrames = this.getDurationFrames(); + const outputFrames = this.getStartFrames() + this.getDurationFrames(); const width = this.canvas.offsetWidth || this._lastWidth || 0; const gaps = []; if (!width) return gaps; @@ -2896,6 +9052,23 @@ class TimelineEditor { gaps.push({ track: 'image', frameStart: cursor, frameEnd: outputFrames, centerX: (x0 + x1) / 2, centerY: RULER_HEIGHT + this.blockHeight / 2, widthPx: x1 - x0 }); } + // Motion gaps + cursor = 0; + const sortedMot = [...this.timeline.motionSegments].sort((a, b) => a.start - b.start); + for (const seg of sortedMot) { + if (seg.start > cursor) { + const x0 = (cursor / totalFrames) * width; + const x1 = (seg.start / totalFrames) * width; + gaps.push({ track: 'motion', frameStart: cursor, frameEnd: seg.start, centerX: (x0 + x1) / 2, centerY: RULER_HEIGHT + this.blockHeight + this.audioTrackHeight + this.motionTrackHeight / 2, widthPx: x1 - x0 }); + } + cursor = seg.start + seg.length; + } + if (cursor < outputFrames) { + const x0 = (cursor / totalFrames) * width; + const x1 = (outputFrames / totalFrames) * width; + gaps.push({ track: 'motion', frameStart: cursor, frameEnd: outputFrames, centerX: (x0 + x1) / 2, centerY: RULER_HEIGHT + this.blockHeight + this.audioTrackHeight + this.motionTrackHeight / 2, widthPx: x1 - x0 }); + } + // Audio gaps cursor = 0; const sortedAud = [...this.timeline.audioSegments].sort((a, b) => a.start - b.start); @@ -2926,13 +9099,35 @@ class TimelineEditor { fi.click(); } + promptAddMotionInGap(frameStart, frameEnd) { + const fi = document.createElement("input"); + fi.type = "file"; + fi.accept = "video/*"; + fi.addEventListener("change", (ev) => { + if (ev.target.files?.[0]) this.handleMotionUpload([ev.target.files[0]], frameStart); + }); + fi.click(); + } + // --- Context Menu --- onContextMenu(e) { e.preventDefault(); + e.stopPropagation(); + + // In retake mode: suppress the normal timeline context menu entirely. + // If a retake video is loaded, show a minimal retake-specific menu instead. + if (this.retakeMode) { + if (this.timeline.retakeVideo) { + this._showRetakeContextMenu(e.clientX, e.clientY); + } + return; + } + const { x: mouseX, y: mouseY } = this.getMousePos(e); const trackHeight = this.blockHeight; const isAudioTrack = mouseY >= RULER_HEIGHT + trackHeight && mouseY <= RULER_HEIGHT + trackHeight + this.audioTrackHeight; + const isMotionTrack = mouseY >= RULER_HEIGHT + trackHeight + this.audioTrackHeight && mouseY <= RULER_HEIGHT + trackHeight + this.audioTrackHeight + this.motionTrackHeight; const isImageTrack = mouseY >= RULER_HEIGHT && mouseY <= RULER_HEIGHT + trackHeight; const logicalWidth = this.canvas.offsetWidth || 1; @@ -2942,7 +9137,10 @@ class TimelineEditor { let clickedSeg = null; let trackType = ""; - if (isAudioTrack) { + if (isMotionTrack) { + clickedSeg = this.timeline.motionSegments.find(s => cursor >= s.start && cursor <= s.start + s.length); + trackType = "motion"; + } else if (isAudioTrack) { clickedSeg = this.timeline.audioSegments.find(s => cursor >= s.start && cursor <= s.start + s.length); trackType = "audio"; } else if (isImageTrack) { @@ -2952,9 +9150,9 @@ class TimelineEditor { if (clickedSeg) { this.showContextMenu(e.clientX, e.clientY, clickedSeg, trackType); - } else if (isAudioTrack || isImageTrack) { + } else if (isMotionTrack || isImageTrack || isAudioTrack) { const gapRegions = this.getGapRegions(); - const currentTrack = isAudioTrack ? "audio" : "image"; + const currentTrack = isMotionTrack ? "motion" : (isAudioTrack ? "audio" : "image"); let gap = gapRegions.find(g => cursor >= g.frameStart && cursor <= g.frameEnd && g.track === currentTrack); if (!gap) { @@ -2971,6 +9169,107 @@ class TimelineEditor { } } + _deleteRetakeVideo() { + if (!this.timeline.retakeVideo) return; + // Clean up the video element + const vid = this.timeline.retakeVideo; + if (vid.videoEl) { + vid.videoEl.pause(); + vid.videoEl.src = ""; + vid.videoEl.load(); + } + if (vid._blobUrl) { + URL.revokeObjectURL(vid._blobUrl); + } + this.timeline.retakeVideo = null; + this.timeline.retakeStart = 0; + this.timeline.retakeLength = this.getDurationFrames(); + this.commitChanges(); + this.render(); + } + + _showRetakeContextMenu(clientX, clientY) { + this.dismissContextMenu(); + + const menu = document.createElement("div"); + menu.className = "pr-gap-menu"; + menu.style.left = `${clientX + 6}px`; + menu.style.top = `${clientY - 10}px`; + + const deleteBtn = document.createElement("button"); + deleteBtn.className = "pr-gap-menu-btn"; + deleteBtn.innerHTML = `${ICONS.trash} Delete`; + deleteBtn.style.color = "#ffaaaa"; + deleteBtn.onclick = () => { + this.dismissContextMenu(); + this._deleteRetakeVideo(); + }; + menu.appendChild(deleteBtn); + + document.body.appendChild(menu); + this._contextMenu = menu; + setTimeout(() => { + this._contextMenuDismisser = (ev) => { if (!menu.contains(ev.target)) this.dismissContextMenu(); }; + document.addEventListener("pointerdown", this._contextMenuDismisser, true); + document.addEventListener("wheel", this._contextMenuDismisser, true); + }, 0); + } + + async _checkClipboardForImage(btn) { + try { + if (navigator.permissions && navigator.permissions.query) { + const status = await navigator.permissions.query({ name: "clipboard-read" }); + if (status.state === "granted") { + const items = await navigator.clipboard.read(); + let hasImg = false; + for (const item of items) { + if (item.types.some(t => t.startsWith("image/"))) { + hasImg = true; + break; + } + } + if (!hasImg) { + btn.disabled = true; + btn.style.opacity = "0.4"; + btn.style.cursor = "not-allowed"; + btn.title = "No image found in clipboard"; + } + } else if (status.state === "denied") { + btn.disabled = true; + btn.style.opacity = "0.4"; + btn.style.cursor = "not-allowed"; + btn.title = "Clipboard permission denied"; + } + } + } catch (e) { + console.warn("Clipboard read permission query failed:", e); + } + } + + async _checkClipboardForText(btn) { + try { + if (navigator.permissions && navigator.permissions.query) { + const status = await navigator.permissions.query({ name: "clipboard-read" }); + if (status.state === "granted") { + const text = await navigator.clipboard.readText(); + if (!text || text.trim() === "") { + btn.disabled = true; + btn.style.opacity = "0.4"; + btn.style.cursor = "not-allowed"; + btn.title = "No text found in clipboard"; + } + } else if (status.state === "denied") { + btn.disabled = true; + btn.style.opacity = "0.4"; + btn.style.cursor = "not-allowed"; + btn.title = "Clipboard permission denied"; + } + } + } catch (e) { + console.warn("Clipboard read text permission query failed:", e); + } + } + showContextMenu(clientX, clientY, seg, trackType) { this.dismissContextMenu(); const menu = document.createElement("div"); @@ -2978,53 +9277,109 @@ class TimelineEditor { menu.style.left = `${clientX + 6}px`; menu.style.top = `${clientY - 10}px`; - const previewSrc = imagePreviewSrc(seg); - const isImage = trackType !== "audio" && trackType !== "text" && previewSrc; + const isImage = trackType === "image" && seg.imageB64; - if (isImage) { - const copyBtn = document.createElement("button"); - copyBtn.className = "pr-gap-menu-btn"; - copyBtn.innerHTML = `Copy Image`; - copyBtn.onclick = async () => { - try { - const res = await fetch(previewSrc); - const blob = await res.blob(); - await navigator.clipboard.write([new ClipboardItem({ [blob.type]: blob })]); - } catch (err) { - console.error("Failed to copy image", err); + const makeDivider = () => { + const d = document.createElement("div"); + d.className = "pr-settings-divider"; + return d; + }; + + // ========================================== + // 1. Define Segment options (Copy, Paste, Replace Segment, Split) + // ========================================== + const copySegBtn = document.createElement("button"); + copySegBtn.className = "pr-gap-menu-btn"; + copySegBtn.innerHTML = `Copy Segment`; + copySegBtn.onclick = () => { + this._copiedSegment = { ...seg, id: Date.now().toString() + Math.random().toString(36).substr(2, 5) }; + this._copiedSegmentTrack = trackType; + window._ltxCopiedSegment = { main: { ...seg }, sibling: null }; + window._ltxCopiedSegmentType = this.getCanonicalTrack(trackType); + if (seg.imgObj) window._ltxCopiedSegment.main.imgObj = seg.imgObj; + if (seg.videoEl) window._ltxCopiedSegment.main.videoEl = seg.videoEl; + + if (seg.id && (seg.id.endsWith("_v") || seg.id.endsWith("_a"))) { + const isVid = seg.id.endsWith("_v"); + const sibId = isVid ? seg.id.slice(0, -2) + "_a" : seg.id.slice(0, -2) + "_v"; + const sibArr = isVid ? this.timeline.audioSegments : this.timeline.segments; + const sib = sibArr.find(s => s.id === sibId); + if (sib) { + window._ltxCopiedSegment.sibling = { ...sib }; + if (sib.imgObj) window._ltxCopiedSegment.sibling.imgObj = sib.imgObj; + if (sib.videoEl) window._ltxCopiedSegment.sibling.videoEl = sib.videoEl; } - this.dismissContextMenu(); - }; - menu.appendChild(copyBtn); + } + this.dismissContextMenu(); + }; - const saveBtn = document.createElement("button"); - saveBtn.className = "pr-gap-menu-btn"; - saveBtn.innerHTML = `Save Image`; - saveBtn.onclick = () => { - const a = document.createElement("a"); - a.href = previewSrc; - a.download = "timeline_image.jpg"; - a.click(); - this.dismissContextMenu(); - }; - menu.appendChild(saveBtn); + const hasCopied = this._copiedSegment || window._ltxCopiedSegment; + const copiedTrack = this._copiedSegmentTrack || window._ltxCopiedSegmentType; + const copiedSegData = this._copiedSegment || (window._ltxCopiedSegment ? window._ltxCopiedSegment.main : null); + const copiedSibData = window._ltxCopiedSegment ? window._ltxCopiedSegment.sibling : null; - const openBtn = document.createElement("button"); - openBtn.className = "pr-gap-menu-btn"; - openBtn.innerHTML = `Open Image in New Tab`; - openBtn.onclick = () => { - const win = window.open(); - if (win) { - win.document.write(``); - win.document.close(); - } + const canPaste = hasCopied && this.getCanonicalTrack(copiedTrack) === this.getCanonicalTrack(trackType) && copiedSegData; + const pasteSegBtn = document.createElement("button"); + pasteSegBtn.className = "pr-gap-menu-btn"; + pasteSegBtn.innerHTML = `Paste Segment`; + if (!canPaste) { + pasteSegBtn.disabled = true; + pasteSegBtn.style.opacity = "0.4"; + pasteSegBtn.style.cursor = "not-allowed"; + pasteSegBtn.title = "No matching segment copied to clipboard"; + } else { + pasteSegBtn.onclick = () => { + const startFrame = Math.round(this.currentFrame); + this.pasteSegmentAtFrame(copiedSegData, this.getCanonicalTrack(copiedTrack), copiedSibData, startFrame); this.dismissContextMenu(); }; - menu.appendChild(openBtn); } + const currentTrack = trackType; + const canReplace = hasCopied && this.getCanonicalTrack(copiedTrack) === this.getCanonicalTrack(currentTrack) && copiedSegData; + const pasteReplaceBtn = document.createElement("button"); + pasteReplaceBtn.className = "pr-gap-menu-btn"; + pasteReplaceBtn.innerHTML = `Replace Segment`; + if (!canReplace) { + pasteReplaceBtn.disabled = true; + pasteReplaceBtn.style.opacity = "0.4"; + pasteReplaceBtn.style.cursor = "not-allowed"; + pasteReplaceBtn.title = "No matching segment copied to clipboard"; + } else { + pasteReplaceBtn.onclick = () => { + const newSeg = { + ...copiedSegData, + id: Date.now().toString() + Math.random().toString(36).substr(2, 5), + start: seg.start, + length: copiedSegData.length + }; + const targetArray = this.getSegmentArray(this.getCanonicalTrack(currentTrack)); + const idx = targetArray.findIndex(s => s.id === seg.id); + if (idx >= 0) targetArray[idx] = newSeg; + this.commitChanges(); + this.dismissContextMenu(); + }; + } + + let splitBtn = null; + const splitFrame = Math.round(this.currentFrame); + if (splitFrame > seg.start && splitFrame < seg.start + seg.length) { + splitBtn = document.createElement("button"); + splitBtn.className = "pr-gap-menu-btn"; + splitBtn.innerHTML = `Split at Playhead`; + splitBtn.onclick = () => { + this.splitSegmentAtPlayhead(seg, trackType); + this.dismissContextMenu(); + }; + } + + // ========================================== + // 2. Define Prompt options (if not audio) + // ========================================== + let copyPromptBtn = null; + let pastePromptBtn = null; if (trackType !== "audio") { - const copyPromptBtn = document.createElement("button"); + copyPromptBtn = document.createElement("button"); copyPromptBtn.className = "pr-gap-menu-btn"; copyPromptBtn.innerHTML = `Copy Prompt`; copyPromptBtn.onclick = async () => { @@ -3035,51 +9390,300 @@ class TimelineEditor { } this.dismissContextMenu(); }; - menu.appendChild(copyPromptBtn); - } - const copySegBtn = document.createElement("button"); - copySegBtn.className = "pr-gap-menu-btn"; - copySegBtn.innerHTML = `Copy Segment`; - copySegBtn.onclick = () => { - this._copiedSegment = { ...seg, id: Date.now().toString() + Math.random().toString(36).substr(2, 5) }; - this._copiedSegmentTrack = trackType === "audio" ? "audio" : "image"; - this.dismissContextMenu(); - }; - menu.appendChild(copySegBtn); - - const currentTrack = trackType === "audio" ? "audio" : "image"; - if (this._copiedSegment && this._copiedSegmentTrack === currentTrack) { - const pasteReplaceBtn = document.createElement("button"); - pasteReplaceBtn.className = "pr-gap-menu-btn"; - pasteReplaceBtn.innerHTML = `Paste & Replace`; - pasteReplaceBtn.onclick = () => { - const newSeg = { - ...this._copiedSegment, - id: Date.now().toString() + Math.random().toString(36).substr(2, 5), - start: seg.start, - length: this._copiedSegment.length - }; - const targetArray = currentTrack === "audio" ? this.timeline.audioSegments : this.timeline.segments; - const idx = targetArray.findIndex(s => s.id === seg.id); - if (idx >= 0) targetArray[idx] = newSeg; - this.commitChanges(); + pastePromptBtn = document.createElement("button"); + pastePromptBtn.className = "pr-gap-menu-btn"; + pastePromptBtn.innerHTML = `Paste Prompt`; + this._checkClipboardForText(pastePromptBtn); + pastePromptBtn.onclick = async () => { + try { + const text = await navigator.clipboard.readText(); + if (text) { + seg.prompt = text; + this.commitChanges(); + this.render(); + if (this.selectedIndex === this.timeline.segments.findIndex(s => s.id === seg.id)) { + this.updateUIFromSelection(); + } + } + } catch (err) { + console.error("Failed to paste prompt", err); + } this.dismissContextMenu(); }; - menu.appendChild(pasteReplaceBtn); } + // ========================================== + // 3. Define Image options (if isImage) + // ========================================== + let copyImgBtn = null; + let saveImgBtn = null; + let openImgBtn = null; + let replaceImgBtn = null; + let replaceWithFileBtn = null; + + if (isImage) { + copyImgBtn = document.createElement("button"); + copyImgBtn.className = "pr-gap-menu-btn"; + copyImgBtn.innerHTML = `Copy Image`; + copyImgBtn.onclick = async () => { + try { + const img = new Image(); + img.crossOrigin = "Anonymous"; + img.src = seg.imageB64; + await new Promise((resolve, reject) => { + img.onload = resolve; + img.onerror = reject; + }); + const canvas = document.createElement("canvas"); + canvas.width = img.width; + canvas.height = img.height; + canvas.getContext("2d").drawImage(img, 0, 0); + const blob = await new Promise(resolve => canvas.toBlob(resolve, "image/png")); + await navigator.clipboard.write([new ClipboardItem({ "image/png": blob })]); + } catch (err) { + console.error("Failed to copy image", err); + } + this.dismissContextMenu(); + }; + + saveImgBtn = document.createElement("button"); + saveImgBtn.className = "pr-gap-menu-btn"; + saveImgBtn.innerHTML = `Save Image`; + saveImgBtn.onclick = () => { + const a = document.createElement("a"); + a.href = seg.imageB64; + a.download = "timeline_image.jpg"; + a.click(); + this.dismissContextMenu(); + }; + + openImgBtn = document.createElement("button"); + openImgBtn.className = "pr-gap-menu-btn"; + openImgBtn.innerHTML = `Open Image in New Tab`; + openImgBtn.onclick = () => { + const win = window.open(); + if (win) { + win.document.write(``); + win.document.close(); + } + this.dismissContextMenu(); + }; + + replaceImgBtn = document.createElement("button"); + replaceImgBtn.className = "pr-gap-menu-btn"; + replaceImgBtn.innerHTML = `Replace with Copied Image`; + this._checkClipboardForImage(replaceImgBtn); + replaceImgBtn.onclick = async () => { + try { + const items = await navigator.clipboard.read(); + for (const item of items) { + const imageTypes = item.types.filter(type => type.startsWith("image/")); + if (imageTypes.length > 0) { + const blob = await item.getType(imageTypes[0]); + const file = new File([blob], "clipboard.png", { type: blob.type }); + + const body = new FormData(); + body.append("image", file); + body.append("subfolder", "whatdreamscost"); + const resp = await api.fetchApi("/upload/image", { method: "POST", body }); + if (resp.status === 200) { + const data = await resp.json(); + const filename = data.name; + const subfolder = data.subfolder || ""; + const imageFile = subfolder ? subfolder + "/" + filename : filename; + const imgUrl = api.apiURL(`/view?filename=${encodeURIComponent(filename)}&type=input&subfolder=${encodeURIComponent(subfolder)}`); + + const img = new Image(); + img.onload = () => { + seg.imageFile = imageFile; + seg.imageB64 = imgUrl; + seg.imgObj = img; + this.commitChanges(); + this.render(); + if (this.selectedIndex === this.timeline.segments.findIndex(s => s.id === seg.id)) { + this.updateUIFromSelection(); + } + }; + img.src = imgUrl; + } + break; + } + } + } catch (err) { + console.error("Failed to read image from clipboard", err); + } + this.dismissContextMenu(); + }; + + replaceWithFileBtn = document.createElement("button"); + replaceWithFileBtn.className = "pr-gap-menu-btn"; + replaceWithFileBtn.innerHTML = `Replace with...`; + replaceWithFileBtn.onclick = () => { + this.dismissContextMenu(); + const fi = document.createElement("input"); + fi.type = "file"; + fi.accept = "image/*"; + fi.addEventListener("change", async (ev) => { + const file = ev.target.files?.[0]; + if (!file) return; + try { + const body = new FormData(); + body.append("image", file); + body.append("subfolder", "whatdreamscost"); + const resp = await api.fetchApi("/upload/image", { method: "POST", body }); + if (resp.status === 200) { + const data = await resp.json(); + const filename = data.name; + const subfolder = data.subfolder || ""; + const imageFile = subfolder ? subfolder + "/" + filename : filename; + const imgUrl = api.apiURL(`/view?filename=${encodeURIComponent(filename)}&type=input&subfolder=${encodeURIComponent(subfolder)}`); + + const img = new Image(); + img.onload = () => { + seg.imageFile = imageFile; + seg.imageB64 = imgUrl; + seg.imgObj = img; + this.commitChanges(); + this.render(); + if (this.selectedIndex === this.timeline.segments.findIndex(s => s.id === seg.id)) { + this.updateUIFromSelection(); + } + }; + img.src = imgUrl; + } + } catch (err) { + console.error("Failed to upload replacement image", err); + } + }); + fi.click(); + }; + } + + // ========================================== + // 4. Define Convert to End Frame options (only image segment with type === "image") + // ========================================== + let toggleEndFrameBtn = null; + if (trackType === "image" && seg.type === "image") { + toggleEndFrameBtn = document.createElement("button"); + toggleEndFrameBtn.className = "pr-gap-menu-btn"; + if (seg.isEndFrame) { + toggleEndFrameBtn.innerHTML = `Convert to Start Frame`; + toggleEndFrameBtn.onclick = () => { + seg.isEndFrame = false; + this.commitChanges(); + this.render(); + this.dismissContextMenu(); + }; + } else { + toggleEndFrameBtn.innerHTML = `Convert to End Frame`; + toggleEndFrameBtn.onclick = () => { + seg.isEndFrame = true; + this.commitChanges(); + this.render(); + this.dismissContextMenu(); + }; + } + } + + // ========================================== + // 5. Define Unlink Media & Mark Selection options + // ========================================== + const isVidLink = trackType === "video" && seg.id.endsWith("_v"); + const isAudLink = trackType === "audio" && seg.id.endsWith("_a"); + let siblingForUnlink = null; + + if (isVidLink) { + siblingForUnlink = this.timeline.audioSegments.find(s => s.id === seg.id.slice(0, -2) + "_a"); + } else if (isAudLink) { + siblingForUnlink = this.timeline.segments.find(s => s.id === seg.id.slice(0, -2) + "_v"); + } + + let unlinkBtn = null; + if (siblingForUnlink) { + unlinkBtn = document.createElement("button"); + unlinkBtn.className = "pr-gap-menu-btn"; + unlinkBtn.innerHTML = `Unlink Media`; + unlinkBtn.onclick = () => { + seg.id = Date.now().toString() + Math.random().toString(36).substr(2, 5); + siblingForUnlink.id = Date.now().toString() + Math.random().toString(36).substr(2, 5); + this.commitChanges(); + this.render(); + this.dismissContextMenu(); + }; + } + + const markSelectionBtn = document.createElement("button"); + markSelectionBtn.className = "pr-gap-menu-btn"; + markSelectionBtn.innerHTML = `Mark Selection`; + markSelectionBtn.onclick = () => { + if (this.selectedSegmentIds && this.selectedSegmentIds.includes(seg.id)) { + this.markCurrentSelection(); + } else { + this.markSegment(seg); + } + this.dismissContextMenu(); + }; + + // ========================================== + // 6. Define Delete Option + // ========================================== const delBtn = document.createElement("button"); delBtn.className = "pr-gap-menu-btn"; delBtn.innerHTML = `Delete`; delBtn.style.color = "#ff4444"; delBtn.onclick = () => { - this.selectionType = trackType === "audio" ? "audio" : "image"; - const list = trackType === "audio" ? this.timeline.audioSegments : this.timeline.segments; + this.selectionType = trackType; + const list = this.getSegmentArray(trackType); this.selectedIndex = list.findIndex(s => s.id === seg.id); this.deleteSelectedSegment(); this.dismissContextMenu(); }; + + // Very top: Split at Playhead (if active/available) + if (splitBtn) { + menu.appendChild(splitBtn); + menu.appendChild(makeDivider()); + } + + // Group 1: Segment Options (Always present) + menu.appendChild(copySegBtn); + menu.appendChild(pasteSegBtn); + menu.appendChild(pasteReplaceBtn); + menu.appendChild(makeDivider()); + + // Group 2: Prompt Options (Only if not audio) + if (copyPromptBtn && pastePromptBtn) { + menu.appendChild(copyPromptBtn); + menu.appendChild(pastePromptBtn); + menu.appendChild(makeDivider()); + } + + // Group 3: Image Options (Only if isImage) + if (isImage) { + menu.appendChild(copyImgBtn); + menu.appendChild(saveImgBtn); + menu.appendChild(openImgBtn); + menu.appendChild(replaceImgBtn); + menu.appendChild(replaceWithFileBtn); + menu.appendChild(makeDivider()); + } + + // Group 4: Convert to End Frame (Only if toggleEndFrameBtn is defined) + if (toggleEndFrameBtn) { + menu.appendChild(toggleEndFrameBtn); + menu.appendChild(makeDivider()); + } + + // Group 5: Unlink Media & Mark Selection + if (unlinkBtn) { + menu.appendChild(unlinkBtn); + menu.appendChild(makeDivider()); + } + menu.appendChild(markSelectionBtn); + menu.appendChild(makeDivider()); + + // Group 6: Delete Option menu.appendChild(delBtn); document.body.appendChild(menu); @@ -3098,30 +9702,30 @@ class TimelineEditor { menu.style.left = `${clientX + 6}px`; menu.style.top = `${clientY - 10}px`; - const currentTrack = gap.track === "audio" ? "audio" : "image"; + const currentTrack = gap.track; - if (this._copiedSegment && this._copiedSegmentTrack === currentTrack) { - const pasteBtn = document.createElement("button"); - pasteBtn.className = "pr-gap-menu-btn"; - pasteBtn.innerHTML = `Paste Segment`; + const hasCopied = this._copiedSegment || window._ltxCopiedSegment; + const copiedTrack = this._copiedSegmentTrack || window._ltxCopiedSegmentType; + const copiedSegData = this._copiedSegment || (window._ltxCopiedSegment ? window._ltxCopiedSegment.main : null); + const copiedSibData = window._ltxCopiedSegment ? window._ltxCopiedSegment.sibling : null; + + const canPaste = hasCopied && this.getCanonicalTrack(copiedTrack) === this.getCanonicalTrack(currentTrack) && copiedSegData; + const pasteBtn = document.createElement("button"); + pasteBtn.className = "pr-gap-menu-btn"; + pasteBtn.innerHTML = `Paste Segment`; + if (!canPaste) { + pasteBtn.disabled = true; + pasteBtn.style.opacity = "0.4"; + pasteBtn.style.cursor = "not-allowed"; + pasteBtn.title = "No matching segment copied to clipboard"; + } else { pasteBtn.onclick = () => { const startFrame = Math.round(gap.clickedFrame !== undefined ? gap.clickedFrame : gap.frameStart); - const gapLength = gap.frameEnd - startFrame; - - const newSeg = { - ...this._copiedSegment, - id: Date.now().toString() + Math.random().toString(36).substr(2, 5), - start: startFrame, - length: Math.min(this._copiedSegment.length, gapLength) - }; - const targetArray = currentTrack === "audio" ? this.timeline.audioSegments : this.timeline.segments; - targetArray.push(newSeg); - targetArray.sort((a, b) => a.start - b.start); - this.commitChanges(); + this.pasteSegmentAtFrame(copiedSegData, this.getCanonicalTrack(copiedTrack), copiedSibData, startFrame); this.dismissContextMenu(); }; - menu.appendChild(pasteBtn); } + menu.appendChild(pasteBtn); if (currentTrack === "image") { const textBtn = document.createElement("button"); @@ -3149,6 +9753,65 @@ class TimelineEditor { fi.click(); }; menu.appendChild(imgBtn); + + const pasteImageBtn = document.createElement("button"); + pasteImageBtn.className = "pr-gap-menu-btn"; + pasteImageBtn.innerHTML = `${ICONS.upload} Paste Image`; + this._checkClipboardForImage(pasteImageBtn); + pasteImageBtn.onclick = async () => { + this.dismissContextMenu(); + try { + const items = await navigator.clipboard.read(); + for (const item of items) { + const imageTypes = item.types.filter(type => type.startsWith("image/")); + if (imageTypes.length > 0) { + const blob = await item.getType(imageTypes[0]); + const file = new File([blob], "clipboard.png", { type: blob.type }); + const startFrame = Math.round(gap.clickedFrame !== undefined ? gap.clickedFrame : gap.frameStart); + const gapLength = gap.frameEnd - startFrame; + + await this.handleImageUpload([file], startFrame, gapLength); + break; + } + } + } catch (err) { + console.error("Failed to paste image from clipboard", err); + } + }; + + const vidBtn = document.createElement("button"); + vidBtn.className = "pr-gap-menu-btn"; + vidBtn.innerHTML = ` Video Segment`; + vidBtn.onclick = () => { + this.dismissContextMenu(); + const fi = document.createElement("input"); + fi.type = "file"; fi.accept = "video/*"; + fi.addEventListener("change", (ev) => { + if (ev.target.files?.[0]) this.handleVideoUpload([ev.target.files[0]], gap.frameStart); + }); + fi.click(); + }; + + menu.appendChild(vidBtn); + menu.appendChild(pasteImageBtn); + } else if (currentTrack === "motion") { + const vidBtn = document.createElement("button"); + vidBtn.className = "pr-gap-menu-btn"; + vidBtn.innerHTML = ` Video Segment`; + vidBtn.onclick = () => { + this.dismissContextMenu(); + this.promptAddMotionInGap(gap.frameStart, gap.frameEnd); + }; + menu.appendChild(vidBtn); + } else if (currentTrack === "audio") { + const audBtn = document.createElement("button"); + audBtn.className = "pr-gap-menu-btn"; + audBtn.innerHTML = `${ICONS.audio} Audio Segment`; + audBtn.onclick = () => { + this.dismissContextMenu(); + this.promptAddAudioInGap(gap.frameStart, gap.frameEnd); + }; + menu.appendChild(audBtn); } document.body.appendChild(menu); @@ -3156,11 +9819,16 @@ class TimelineEditor { setTimeout(() => { this._contextMenuDismisser = (ev) => { if (!menu.contains(ev.target)) this.dismissContextMenu(); }; document.addEventListener("pointerdown", this._contextMenuDismisser, true); + document.addEventListener("wheel", this._contextMenuDismisser, true); }, 0); } dismissContextMenu() { if (this._contextMenu) { this._contextMenu.remove(); this._contextMenu = null; } - if (this._contextMenuDismisser) { document.removeEventListener("pointerdown", this._contextMenuDismisser, true); this._contextMenuDismisser = null; } + if (this._contextMenuDismisser) { + document.removeEventListener("pointerdown", this._contextMenuDismisser, true); + document.removeEventListener("wheel", this._contextMenuDismisser, true); + this._contextMenuDismisser = null; + } } // --- Gap Popup Menu --- @@ -3171,71 +9839,134 @@ class TimelineEditor { menu.style.left = `${clientX + 6}px`; menu.style.top = `${clientY - 10}px`; - const textBtn = document.createElement("button"); - textBtn.className = "pr-gap-menu-btn"; - textBtn.innerHTML = `${ICONS.text} Text Segment`; - textBtn.addEventListener("click", () => { - this.addSegmentInGap(gap.frameStart, gap.frameEnd, "text"); - this.dismissGapMenu(); - }); + const currentTrack = gap.track; - const imgBtn = document.createElement("button"); - imgBtn.className = "pr-gap-menu-btn"; - imgBtn.innerHTML = `${ICONS.upload} Image Segment`; - imgBtn.addEventListener("click", () => { - this.dismissGapMenu(); - const fi = document.createElement("input"); - fi.type = "file"; fi.accept = "image/*"; - fi.addEventListener("change", (ev) => { - if (ev.target.files?.[0]) { - const gapLength = gap.frameEnd - gap.frameStart; - this.handleImageUpload([ev.target.files[0]], gap.frameStart, gapLength); + if (currentTrack === "image") { + const textBtn = document.createElement("button"); + textBtn.className = "pr-gap-menu-btn"; + textBtn.innerHTML = `${ICONS.text} Text Segment`; + textBtn.addEventListener("click", () => { + this.addSegmentInGap(gap.frameStart, gap.frameEnd, "text"); + this.dismissGapMenu(); + }); + + const imgBtn = document.createElement("button"); + imgBtn.className = "pr-gap-menu-btn"; + imgBtn.innerHTML = `${ICONS.upload} Image Segment`; + imgBtn.addEventListener("click", () => { + this.dismissGapMenu(); + const fi = document.createElement("input"); + fi.type = "file"; fi.accept = "image/*"; + fi.addEventListener("change", (ev) => { + if (ev.target.files?.[0]) { + const gapLength = gap.frameEnd - gap.frameStart; + this.handleImageUpload([ev.target.files[0]], gap.frameStart, gapLength); + } + }); + fi.click(); + }); + + const pasteImageBtn = document.createElement("button"); + pasteImageBtn.className = "pr-gap-menu-btn"; + pasteImageBtn.innerHTML = `${ICONS.upload} Paste Image`; + this._checkClipboardForImage(pasteImageBtn); + pasteImageBtn.addEventListener("click", async () => { + this.dismissGapMenu(); + try { + const items = await navigator.clipboard.read(); + for (const item of items) { + const imageTypes = item.types.filter(type => type.startsWith("image/")); + if (imageTypes.length > 0) { + const blob = await item.getType(imageTypes[0]); + const file = new File([blob], "clipboard.png", { type: blob.type }); + const gapLength = gap.frameEnd - gap.frameStart; + await this.handleImageUpload([file], gap.frameStart, gapLength); + break; + } + } + } catch (err) { + console.error("Failed to paste image from clipboard", err); } }); - fi.click(); - }); - menu.appendChild(textBtn); - menu.appendChild(imgBtn); - const currentTrack = gap.track === "audio" ? "audio" : "image"; - if (this._copiedSegment && this._copiedSegmentTrack === currentTrack) { - const pasteBtn = document.createElement("button"); - pasteBtn.className = "pr-gap-menu-btn"; - pasteBtn.innerHTML = `Paste Segment`; + const vidBtn = document.createElement("button"); + vidBtn.className = "pr-gap-menu-btn"; + vidBtn.innerHTML = ` Video Segment`; + vidBtn.addEventListener("click", () => { + this.dismissGapMenu(); + const fi = document.createElement("input"); + fi.type = "file"; fi.accept = "video/*"; + fi.addEventListener("change", (ev) => { + if (ev.target.files?.[0]) { + this.handleVideoUpload([ev.target.files[0]], gap.frameStart); + } + }); + fi.click(); + }); + + menu.appendChild(textBtn); + menu.appendChild(imgBtn); + menu.appendChild(vidBtn); + menu.appendChild(pasteImageBtn); + } else if (currentTrack === "motion") { + const vidBtn = document.createElement("button"); + vidBtn.className = "pr-gap-menu-btn"; + vidBtn.innerHTML = ` Video Segment`; + vidBtn.addEventListener("click", () => { + this.dismissGapMenu(); + this.promptAddMotionInGap(gap.frameStart, gap.frameEnd); + }); + menu.appendChild(vidBtn); + } else if (currentTrack === "audio") { + const audBtn = document.createElement("button"); + audBtn.className = "pr-gap-menu-btn"; + audBtn.innerHTML = `${ICONS.audio} Audio Segment`; + audBtn.addEventListener("click", () => { + this.dismissGapMenu(); + this.promptAddAudioInGap(gap.frameStart, gap.frameEnd); + }); + menu.appendChild(audBtn); + } + + const hasCopied = this._copiedSegment || window._ltxCopiedSegment; + const copiedTrack = this._copiedSegmentTrack || window._ltxCopiedSegmentType; + const copiedSegData = this._copiedSegment || (window._ltxCopiedSegment ? window._ltxCopiedSegment.main : null); + const copiedSibData = window._ltxCopiedSegment ? window._ltxCopiedSegment.sibling : null; + + const canPaste = hasCopied && this.getCanonicalTrack(copiedTrack) === this.getCanonicalTrack(currentTrack) && copiedSegData; + const pasteBtn = document.createElement("button"); + pasteBtn.className = "pr-gap-menu-btn"; + pasteBtn.innerHTML = `Paste Segment`; + if (!canPaste) { + pasteBtn.disabled = true; + pasteBtn.style.opacity = "0.4"; + pasteBtn.style.cursor = "not-allowed"; + pasteBtn.title = "No matching segment copied to clipboard"; + } else { pasteBtn.onclick = () => { - const gapLength = gap.frameEnd - gap.frameStart; - - let finalLength = Math.min(this._copiedSegment.length, gapLength); - if (currentTrack === "image") { - finalLength = gapLength; - } - - const newSeg = { - ...this._copiedSegment, - id: Date.now().toString() + Math.random().toString(36).substr(2, 5), - start: gap.frameStart, - length: finalLength - }; - const targetArray = currentTrack === "audio" ? this.timeline.audioSegments : this.timeline.segments; - targetArray.push(newSeg); - targetArray.sort((a, b) => a.start - b.start); - this.commitChanges(); + const startFrame = Math.round(gap.frameStart); + this.pasteSegmentAtFrame(copiedSegData, this.getCanonicalTrack(copiedTrack), copiedSibData, startFrame); this.dismissGapMenu(); }; - menu.appendChild(pasteBtn); } + menu.appendChild(pasteBtn); document.body.appendChild(menu); this._gapMenu = menu; setTimeout(() => { this._gapMenuDismisser = (ev) => { if (!menu.contains(ev.target)) this.dismissGapMenu(); }; document.addEventListener("pointerdown", this._gapMenuDismisser, true); + document.addEventListener("wheel", this._gapMenuDismisser, true); }, 0); } dismissGapMenu() { if (this._gapMenu) { this._gapMenu.remove(); this._gapMenu = null; } - if (this._gapMenuDismisser) { document.removeEventListener("pointerdown", this._gapMenuDismisser, true); this._gapMenuDismisser = null; } + if (this._gapMenuDismisser) { + document.removeEventListener("pointerdown", this._gapMenuDismisser, true); + document.removeEventListener("wheel", this._gapMenuDismisser, true); + this._gapMenuDismisser = null; + } } // --- Settings Menu --- @@ -3246,18 +9977,31 @@ class TimelineEditor { // Hide all settings widgets on the node (called on init). hideSettingsWidgets() { + const isLiteGraph = !window.LiteGraph || !window.LiteGraph.vueNodesMode; + // If any settings widgets have active connections, show settings widgets instead + let hasActiveSettings = false; + for (const name of this._settingsWidgetNames) { + const hasInput = this.node.inputs?.find(i => i.name === name); + if (hasInput && hasInput.link != null) { + hasActiveSettings = true; + break; + } + } + + if (hasActiveSettings) { + this.showSettingsWidgets(); + return; + } + for (const name of this._settingsWidgetNames) { const w = this.node.widgets?.find(w => w.name === name); - if (w) hideWidget(w); - - // Also remove corresponding input slot if it exists and is NOT connected - // to prevent overlapping issues in classic ComfyUI (nodes v1) - if (this.node.inputs) { - const inputIdx = this.node.inputs.findIndex(i => i.name === name); - if (inputIdx !== -1) { - const input = this.node.inputs[inputIdx]; - if (input.link == null) { - this.node.removeInput(inputIdx); + if (w) { + hideWidget(w); + // If it was converted to an input slot but is unconnected, remove the input slot + if (isLiteGraph && this.node.inputs) { + const idx = this.node.inputs.findIndex(i => i.name === name); + if (idx !== -1 && this.node.inputs[idx].link == null) { + this.node.removeInput(idx); } } } @@ -3279,19 +10023,28 @@ class TimelineEditor { // Restore all settings widgets on the node. showSettingsWidgets() { + const isLiteGraph = !window.LiteGraph || !window.LiteGraph.vueNodesMode; for (const name of this._settingsWidgetNames) { const w = this.node.widgets?.find(w => w.name === name); if (!w) continue; - - const typeMap = { - display_mode: "combo", epsilon: "FLOAT", divisible_by: "INT", - img_compression: "INT", - }; - w.type = typeMap[name] || w._origType || "number"; - w.hidden = false; - if (w.options) w.options.hidden = false; - delete w.computeSize; - if (w.element) w.element.style.display = ""; + showWidget(w); + + // If the widget is a converted-widget but the input slot is missing, add it back! + if (isLiteGraph && w.type === "converted-widget" && this.node.inputs) { + if (!this.node.inputs.find(i => i.name === name)) { + let type = "FLOAT"; + if (name === "divisible_by" || name === "img_compression") { + type = "INT"; + } else if (name === "display_mode") { + type = "COMBO"; + } + const slot = this.node.addInput(name, type); + if (slot != null) { + const inp = this.node.inputs[this.node.inputs.length - 1]; + if (inp) inp.widget = { name }; + } + } + } } this.updateWidgetVisibility(); @@ -3308,6 +10061,273 @@ class TimelineEditor { } } + // --- Save / Load Handlers --- + async handleLoadTimeline() { + try { + if (window.showOpenFilePicker) { + const [fileHandle] = await window.showOpenFilePicker({ + types: [{ description: 'Timeline JSON', accept: { 'application/json': ['.json'] } }], + multiple: false + }); + const file = await fileHandle.getFile(); + const content = await file.text(); + this._applyLoadedTimeline(content, fileHandle); + } else { + // Fallback for browsers without showOpenFilePicker (e.g. Firefox) + const input = document.createElement("input"); + input.type = "file"; + input.accept = ".json"; + input.onchange = e => { + const file = e.target.files[0]; + if (!file) return; + const reader = new FileReader(); + reader.onload = evt => this._applyLoadedTimeline(evt.target.result, null); + reader.readAsText(file); + }; + input.click(); + } + } catch (e) { + if (e.name !== "AbortError") { + console.error("Failed to load timeline:", e); + alert("Failed to load timeline. See console for details."); + } + } + } + + _applyLoadedTimeline(jsonStr, fileHandle) { + try { + const data = JSON.parse(jsonStr); + + // Load settings if present + if (data.global_prompt !== undefined) { + if (data.retake_global_prompt !== undefined) { + this.timeline.global_prompt = data.global_prompt; + this.timeline.retake_global_prompt = data.retake_global_prompt; + } else { + this.syncGlobalPrompt(data.global_prompt); + } + } + if (data.settings) { + for (const [key, value] of Object.entries(data.settings)) { + // Handle legacy keys for backward compatibility + if (key === "startFrames" && this.startFramesWidget) { + this.startFramesWidget.value = value; + if (this.startFramesWidget.callback) this.startFramesWidget.callback(value); + continue; + } + if (key === "durationFrames" && this.durationFramesWidget) { + this.durationFramesWidget.value = value; + if (this.durationFramesWidget.callback) this.durationFramesWidget.callback(value); + continue; + } + if (key === "frameRate" && this.frameRateWidget) { + this.frameRateWidget.value = value; + if (this.frameRateWidget.callback) this.frameRateWidget.callback(value); + continue; + } + + const w = this.node.widgets?.find(x => x.name === key); + if (w) { + w.value = value; + if (w.callback) w.callback(w.value); + } + } + } + + if (this.timelineDataWidget) this.timelineDataWidget.value = JSON.stringify(data.timeline || data); + this.timeline = parseInitial(this.timelineDataWidget.value); + this.mainTrackEnabled = this.timeline.mainTrackEnabled !== false; + this.audioTrackEnabled = this.timeline.audioTrackEnabled !== false; + this.motionTrackEnabled = this.timeline.motionTrackEnabled !== false; + if (this.timeline.showFilenames !== undefined) { + this.node.properties.showFilenames = this.timeline.showFilenames; + } + if (this.timeline.overrideAudio !== undefined) { + this.node.properties.overrideAudio = this.timeline.overrideAudio; + } + if (this.timeline.inpaint_audio !== undefined) { + this.node.properties.inpaint_audio = this.timeline.inpaint_audio; + } + if (this.timeline.propHeight !== undefined) { + this.node.properties.propHeight = this.timeline.propHeight; + this.propHeight = this.timeline.propHeight; + if (this.propContainer) { + this.propContainer.style.height = `${this.propHeight}px`; + } + } + if (this.timeline.globalPropHeight !== undefined) { + this.node.properties.globalPropHeight = this.timeline.globalPropHeight; + this.globalPropHeight = this.timeline.globalPropHeight; + if (this.globalPropContainer) { + this.globalPropContainer.style.height = `${this.globalPropHeight}px`; + } + } + this.currentFileHandle = fileHandle; + this.retakeMode = this.timeline.retakeMode === true; + + this.loadMedia(); + + if (!this.retakeMode) { + this._suppressCommit = true; + if (this.timeline.normalStartFrame !== undefined && this.startFramesWidget) { + this.startFramesWidget.value = this.timeline.normalStartFrame; + if (this.startFramesWidget.callback) { + try { this.startFramesWidget.callback(this.timeline.normalStartFrame); } catch (_) {} + } + } + if (this.timeline.normalDurationFrames !== undefined && this.durationFramesWidget) { + this.durationFramesWidget.value = this.timeline.normalDurationFrames; + if (this.durationFramesWidget.callback) { + try { this.durationFramesWidget.callback(this.timeline.normalDurationFrames); } catch (_) {} + } + } + this._suppressCommit = false; + } + + this.updateRetakeUIState(); + this.updateUIFromSelection(); + this.syncWidgetsAndUI(); + this.commitChanges(true); // forces sync to UI and other widgets + + + if (this.updateInpaintToggleStyle) { + const inpaintWidget = this.node.widgets?.find(w => w.name === "inpaint_audio"); + if (inpaintWidget) this.updateInpaintToggleStyle(inpaintWidget.value); + } + + this.render(); + this.dismissSettingsMenu(); + + // Trigger ComfyUI's change-detection pipeline the same way a real user + // interaction does: by dispatching a pointerup on the canvas. This fires + // LiteGraph's onAfterChange → ChangeTracker.captureCanvasState() → + // workflowDraftStore.saveDraft() → localStorage. This is what the user + // experiences when they "move something" and it persists correctly. + setTimeout(() => { + try { + const canvasEl = app.canvasEl || app.canvas?.canvas; + if (canvasEl) { + canvasEl.dispatchEvent(new PointerEvent('pointerup', { bubbles: true, cancelable: true })); + } + // Also try the direct ChangeTracker API as a backup for both frontend versions + if (app.canvas && app.canvas.checkState) app.canvas.checkState(); + if (app.canvas && app.canvas.captureCanvasState) app.canvas.captureCanvasState(); + } catch (_) { } + }, 100); + } catch (e) { + console.error("Invalid timeline JSON:", e); + alert("Invalid timeline file."); + } + } + + _getTimelineSavePayload() { + const allSettings = {}; + const skipWidgets = ["timeline_data", "local_prompts", "segment_lengths", "guide_strength", "timeline_ui", "global_prompt"]; + + for (const w of this.node.widgets || []) { + if (!skipWidgets.includes(w.name) && w.value !== undefined) { + allSettings[w.name] = w.value; + } + } + + const normPrompt = this.retakeMode ? (this.timeline.global_prompt || "") : (this.globalPromptInput ? this.globalPromptInput.value : ""); + const retPrompt = this.retakeMode ? (this.globalPromptInput ? this.globalPromptInput.value : "") : (this.timeline.retake_global_prompt || ""); + + return JSON.stringify({ + version: 1, + settings: allSettings, + global_prompt: normPrompt, + retake_global_prompt: retPrompt, + timeline: { + mainTrackEnabled: this.mainTrackEnabled, + audioTrackEnabled: this.audioTrackEnabled, + motionTrackEnabled: this.motionTrackEnabled, + showFilenames: !!this.node.properties.showFilenames, + overrideAudio: !!this.node.properties.overrideAudio, + inpaint_audio: !!(this.node.widgets?.find(w => w.name === "inpaint_audio")?.value), + propHeight: this.propHeight, + globalPropHeight: this.globalPropHeight, + global_prompt: normPrompt, + retake_global_prompt: retPrompt, + retakeMode: this.retakeMode, + retakeStart: this.timeline.retakeStart, + retakeLength: this.timeline.retakeLength, + retakePrompt: this.timeline.retakePrompt, + retakeStrength: this.timeline.retakeStrength, + retakeVideo: this.timeline.retakeVideo ? { + fileName: this.timeline.retakeVideo.fileName, + imageFile: this.timeline.retakeVideo.imageFile, + videoDurationFrames: this.timeline.retakeVideo.videoDurationFrames, + fileSize: this.timeline.retakeVideo.fileSize, + } : null, + normalStartFrame: this.timeline.normalStartFrame, + normalDurationFrames: this.timeline.normalDurationFrames, + segments: (this.timeline.segments || []).map(s => { + const { imgObj, videoEl, _isSeeking, thumbnails, _extractingThumbs, _sSecs, _lSecs, _tSecs, _dSecs, _uploading, _blobUrl, ...rest } = s; + return rest; + }), + motionSegments: (this.timeline.motionSegments || []).map(s => { + const { imgObj, videoEl, _isSeeking, thumbnails, _extractingThumbs, _sSecs, _lSecs, _tSecs, _dSecs, _uploading, _blobUrl, ...rest } = s; + return rest; + }), + audioSegments: (this.timeline.audioSegments || []).map(s => { + const { _sSecs, _lSecs, _tSecs, _dSecs, _uploading, _decoding, _blobUrl, _audioBuffer, ...rest } = s; + return rest; + }) + } + }, null, 2); + } + + async handleSaveTimeline() { + if (!this.currentFileHandle) { + return this.handleSaveTimelineAs(); + } + + try { + const payload = this._getTimelineSavePayload(); + const writable = await this.currentFileHandle.createWritable(); + await writable.write(payload); + await writable.close(); + this.dismissSettingsMenu(); + } catch (e) { + console.error("Failed to save timeline:", e); + alert("Failed to save. You may need to use Save As."); + } + } + + async handleSaveTimelineAs() { + const payload = this._getTimelineSavePayload(); + + try { + if (window.showSaveFilePicker) { + const fileHandle = await window.showSaveFilePicker({ + suggestedName: "timeline_export.json", + types: [{ description: 'Timeline JSON', accept: { 'application/json': ['.json'] } }] + }); + const writable = await fileHandle.createWritable(); + await writable.write(payload); + await writable.close(); + this.currentFileHandle = fileHandle; + } else { + // Fallback for Firefox + const blob = new Blob([payload], { type: "application/json" }); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = "timeline_export.json"; + a.click(); + URL.revokeObjectURL(url); + // Can't track file handle via download fallback + this.currentFileHandle = null; + } + this.dismissSettingsMenu(); + } catch (e) { + if (e.name !== "AbortError") { + console.error("Failed to save timeline as:", e); + } + } + } + _makeSettingRow(label, inputEl) { const row = document.createElement("div"); row.className = "pr-settings-row"; @@ -3344,6 +10364,55 @@ class TimelineEditor { menu.appendChild(titleContainer); + // --- Save / Load / Show Widgets Grid (2x2) --- + const gridContainer = document.createElement("div"); + gridContainer.style.display = "grid"; + gridContainer.style.gridTemplateColumns = "repeat(2, 1fr)"; + gridContainer.style.gap = "6px"; + gridContainer.style.marginBottom = "4px"; + + const btnSave = document.createElement("button"); + btnSave.className = "pr-settings-toggle-btn"; + btnSave.textContent = "Save Timeline"; + btnSave.addEventListener("click", () => this.handleSaveTimeline()); + gridContainer.appendChild(btnSave); + + const btnSaveAs = document.createElement("button"); + btnSaveAs.className = "pr-settings-toggle-btn"; + btnSaveAs.textContent = "Save Timeline As"; + btnSaveAs.addEventListener("click", () => this.handleSaveTimelineAs()); + gridContainer.appendChild(btnSaveAs); + + const btnLoad = document.createElement("button"); + btnLoad.className = "pr-settings-toggle-btn"; + btnLoad.textContent = "Load Timeline"; + btnLoad.addEventListener("click", () => this.handleLoadTimeline()); + gridContainer.appendChild(btnLoad); + + // --- Show/Hide on Node Toggle --- + const toggleBtn = document.createElement("button"); + toggleBtn.className = "pr-settings-toggle-btn"; + const widgetsVisible = !!(this.node.widgets?.find(w => w.name === "display_mode" && !(w.options && w.options.hidden))); + toggleBtn.textContent = widgetsVisible ? "Hide Widgets" : "Show Widgets"; + toggleBtn.addEventListener("click", () => { + const nowVisible = !!(this.node.widgets?.find(w => w.name === "display_mode" && !(w.options && w.options.hidden))); + if (nowVisible) { + this.hideSettingsWidgets(); + const stillVisible = !!(this.node.widgets?.find(w => w.name === "display_mode" && !(w.options && w.options.hidden))); + toggleBtn.textContent = stillVisible ? "Hide Widgets" : "Show Widgets"; + } else { + this.showSettingsWidgets(); + toggleBtn.textContent = "Hide Widgets"; + } + }); + gridContainer.appendChild(toggleBtn); + + menu.appendChild(gridContainer); + + const div2 = document.createElement("hr"); + div2.className = "pr-settings-divider"; + menu.appendChild(div2); + // Helper: fire a widget's callback safely const fireCallback = (w, val) => { w.value = val; @@ -3397,9 +10466,50 @@ class TimelineEditor { menu.appendChild(this._makeSettingRow("Display Mode", ctrl)); } - const divider1 = document.createElement("hr"); - divider1.className = "pr-settings-divider"; - menu.appendChild(divider1); + + + // --- Show Filenames Toggle --- + const showFnameCtrl = document.createElement("div"); + showFnameCtrl.className = "pr-segmented-control"; + + const offSeg = document.createElement("div"); + offSeg.className = "pr-segment"; + offSeg.textContent = "Off"; + + const onSeg = document.createElement("div"); + onSeg.className = "pr-segment"; + onSeg.textContent = "On"; + + const updateFnameActive = (isEnabled) => { + if (isEnabled) { + onSeg.classList.add("active"); + offSeg.classList.remove("active"); + } else { + offSeg.classList.add("active"); + onSeg.classList.remove("active"); + } + }; + + updateFnameActive(!!this.node.properties.showFilenames); + + const onFnameSegClick = (isEnabled) => { + this.node.properties.showFilenames = isEnabled; + updateFnameActive(isEnabled); + this.render(); + this.commitChanges(true); + }; + + offSeg.addEventListener("click", () => onFnameSegClick(false)); + onSeg.addEventListener("click", () => onFnameSegClick(true)); + + showFnameCtrl.appendChild(onSeg); + showFnameCtrl.appendChild(offSeg); + + menu.appendChild(this._makeSettingRow("Show Filenames", showFnameCtrl)); + + const divider2 = document.createElement("div"); + divider2.className = "pr-settings-divider"; + menu.appendChild(divider2); // Helper to create scrubbable number control with horizontal buttons const createScrubbableNumberControl = (w, step, min, max, isFloat = false) => { @@ -3518,58 +10628,38 @@ class TimelineEditor { menu.appendChild(this._makeSettingRow("Img Compression", createScrubbableNumberControl(compWidget, 1, 0, 100, false))); } - // --- Global Prompt Toggle --- - const globalPromptWidget = this.node.widgets?.find(w => w.name === "global_prompt"); - if (globalPromptWidget) { - const cb = document.createElement("input"); - cb.type = "checkbox"; - cb.checked = !(globalPromptWidget.options && globalPromptWidget.options.hidden); - cb.style.cursor = "pointer"; - cb.addEventListener("change", () => { - const isVisible = cb.checked; - if (!globalPromptWidget.options) globalPromptWidget.options = {}; - globalPromptWidget.options.hidden = !isVisible; + // --- Divider --- + const folderDivider = document.createElement("div"); + folderDivider.className = "pr-settings-divider"; + menu.appendChild(folderDivider); - if (isVisible) { - delete globalPromptWidget.computeSize; - globalPromptWidget.hidden = false; - if (globalPromptWidget.element) globalPromptWidget.element.style.display = ""; - } else { - globalPromptWidget.computeSize = () => [0, 0]; - globalPromptWidget.hidden = true; - if (globalPromptWidget.element) globalPromptWidget.element.style.display = "none"; + // --- Workspace Folder Button --- + const btnOpenFolder = document.createElement("button"); + btnOpenFolder.className = "pr-settings-toggle-btn"; + btnOpenFolder.textContent = "Open"; + btnOpenFolder.style.width = "98px"; + btnOpenFolder.style.margin = "0"; + btnOpenFolder.addEventListener("click", async () => { + try { + const response = await api.fetchApi("/ltx_director_open_folder"); + const data = await response.json(); + if (!data.success) { + console.error("Failed to open workspace folder:", data.error || "Unknown error"); + alert("Could not open workspace folder. This option is only supported when running ComfyUI locally."); } - - // Force refresh via display mode double-toggle trick - if (this.displayModeWidget) { - const origVal = this.displayModeWidget.value; - const otherVal = origVal === "frames" ? "seconds" : "frames"; - this.displayModeWidget.value = otherVal; - if (this.displayModeWidget.callback) this.displayModeWidget.callback(otherVal); - this.displayModeWidget.value = origVal; - if (this.displayModeWidget.callback) this.displayModeWidget.callback(origVal); - } - }); - menu.appendChild(this._makeSettingRow("Use Global Prompt", cb)); - } - - - // --- Show/Hide on Node Toggle --- - const toggleBtn = document.createElement("button"); - toggleBtn.className = "pr-settings-toggle-btn"; - const widgetsVisible = !!(this.node.widgets?.find(w => w.name === "display_mode" && !(w.options && w.options.hidden))); - toggleBtn.textContent = widgetsVisible ? "Hide Widgets on Node" : "Show Widgets on Node"; - toggleBtn.addEventListener("click", () => { - const nowVisible = !!(this.node.widgets?.find(w => w.name === "display_mode" && !(w.options && w.options.hidden))); - if (nowVisible) { - this.hideSettingsWidgets(); - toggleBtn.textContent = "Show Widgets on Node"; - } else { - this.showSettingsWidgets(); - toggleBtn.textContent = "Hide Widgets on Node"; + } catch (err) { + console.error("Error opening workspace folder:", err); + alert("Error opening workspace folder: " + err.message); } }); - menu.appendChild(toggleBtn); + + menu.appendChild(this._makeSettingRow("Workspace Folder", btnOpenFolder)); + + + + + + // Position the menu below the anchor button (pop down) document.body.appendChild(menu); @@ -3591,13 +10681,18 @@ class TimelineEditor { this._settingsDismisser = (ev) => { if (!menu.contains(ev.target) && !anchorEl.contains(ev.target)) this.dismissSettingsMenu(); }; - document.addEventListener("mousedown", this._settingsDismisser); + document.addEventListener("pointerdown", this._settingsDismisser, true); + document.addEventListener("wheel", this._settingsDismisser, true); }, 0); } dismissSettingsMenu() { if (this._settingsMenu) { this._settingsMenu.remove(); this._settingsMenu = null; } - if (this._settingsDismisser) { document.removeEventListener("mousedown", this._settingsDismisser); this._settingsDismisser = null; } + if (this._settingsDismisser) { + document.removeEventListener("pointerdown", this._settingsDismisser, true); + document.removeEventListener("wheel", this._settingsDismisser, true); + this._settingsDismisser = null; + } } @@ -3609,6 +10704,11 @@ class TimelineEditor { }; this.timeline.segments.push(seg); this.timeline.segments.sort((a, b) => a.start - b.start); + + if (!this.retakeMode) { + this.growTimelineIfNeeded(seg.start + seg.length); + } + this.selectionType = "image"; this.selectedIndex = this.timeline.segments.findIndex(s => s.id === seg.id); this.updateUIFromSelection(); @@ -3633,12 +10733,25 @@ class TimelineEditor { }; this.timeline.segments.push(seg); this.timeline.segments.sort((a, b) => a.start - b.start); + + if (!this.retakeMode) { + this.growTimelineIfNeeded(seg.start + seg.length); + } + this.selectionType = "image"; this.selectedIndex = this.timeline.segments.findIndex(s => s.id === seg.id); this.updateUIFromSelection(); this.commitChanges(); } + updateSeekBarBackground() { + if (!this.seekBar) return; + const max = parseFloat(this.seekBar.max) || 1; + const val = parseFloat(this.seekBar.value) || 0; + const pct = (val / max) * 100; + this.seekBar.style.background = `linear-gradient(to right, #ff4444 0%, #ff4444 ${pct}%, #444 ${pct}%, #444 100%)`; + } + // --- Audio Player Engine --- updatePlayerUI() { if (!this.playBtn || !this.loopBtn) return; @@ -3651,6 +10764,7 @@ class TimelineEditor { if (this.seekBar) { this.seekBar.max = this.getVisualDurationFrames(); this.seekBar.value = this.currentFrame; + this.updateSeekBarBackground(); } if (this.timeCodeDisplay) { this.timeCodeDisplay.textContent = this.formatTime(this.currentFrame); @@ -3661,7 +10775,10 @@ class TimelineEditor { if (this.isPlaying) { this.pauseAudio(); } else { - if (this.currentFrame >= this.getVisualDurationFrames()) { + const playMax = this.retakeMode + ? (this.timeline.retakeVideo ? (this.timeline.retakeVideo.videoDurationFrames || this.getDurationFrames()) : this.getDurationFrames()) + : this.getVisualDurationFrames(); + if (this.currentFrame >= playMax) { this.currentFrame = 0; } this.playAudio(); @@ -3695,57 +10812,166 @@ class TimelineEditor { this.playbackStartFrame = this.currentFrame; this.playbackStartTime = this.audioContext.currentTime; - // Decode and schedule all audio segments that happen AT or AFTER currentFrame - for (let seg of this.timeline.audioSegments) { - const segStartFrame = seg.start; - const segEndFrame = seg.start + seg.length; + // Build the list of active segments to play + const segmentsToPlay = []; + + // 1. Standard Audio Segments on the audio track (only if the track is enabled and NOT in retake mode) + if (this.audioTrackEnabled && !this.retakeMode) { + if (this.timeline.audioSegments) { + for (let seg of this.timeline.audioSegments) { + segmentsToPlay.push({ + type: 'audio', + originalSeg: seg, + start: seg.start, + length: seg.length, + trimStart: seg.trimStart || 0, + audioFile: seg.audioFile, + audioB64: seg.audioB64, + _blobUrl: seg._blobUrl, + fileSize: seg.fileSize + }); + } + } + } + + // 2. Motion Video Segments (only if overrideAudio toggle is ON and NOT in retake mode) + const isOverrideAudio = !!(this.node.properties.overrideAudio || this.timeline.overrideAudio); + if (isOverrideAudio && !this.retakeMode) { + if (this.timeline.motionSegments) { + for (let seg of this.timeline.motionSegments) { + if (seg.videoFile || seg._blobUrl) { + segmentsToPlay.push({ + type: 'motion', + originalSeg: seg, + start: seg.start, + length: seg.length, + trimStart: seg.trimStart || 0, + audioFile: seg.videoFile || seg.fileName, + audioB64: null, + _blobUrl: seg._blobUrl, + fileSize: seg.fileSize + }); + } + } + } + } + + // Decode and schedule all scheduled segments that happen AT or AFTER currentFrame in the background + for (let item of segmentsToPlay) { + const segStartFrame = item.start; + const segEndFrame = item.start + item.length; if (segEndFrame <= this.currentFrame) continue; - try { - // Build audio buffer: fetch from server URL if audioFile is set, otherwise fall back to audioB64 - let audioBuffer; - if (seg.audioFile) { - const audioUrl = api.apiURL(`/view?filename=${encodeURIComponent(seg.audioFile.split("/").pop())}&type=input&subfolder=${encodeURIComponent(seg.audioFile.includes("/") ? seg.audioFile.split("/").slice(0, -1).join("/") : "")}`); - const resp = await fetch(audioUrl); - const arrayBuffer = await resp.arrayBuffer(); - audioBuffer = await this.audioContext.decodeAudioData(arrayBuffer); - } else if (seg.audioB64) { - const binaryString = window.atob(seg.audioB64); - const len = binaryString.length; - const bytes = new Uint8Array(len); - for (let i = 0; i < len; i++) { - bytes[i] = binaryString.charCodeAt(i); + (async () => { + try { + // Build mock seg object for helper compatibility + const mockSeg = { + audioFile: item.audioFile, + audioB64: item.audioB64, + _blobUrl: item._blobUrl, + fileSize: item.fileSize, + waveformPeaks: item.originalSeg.waveformPeaks + }; + + await this._getOrExtractAudio(mockSeg); + + if (this._currentPlayId !== playId || !this.isPlaying) return; + + if (mockSeg.waveformPeaks && !item.originalSeg.waveformPeaks) { + item.originalSeg.waveformPeaks = mockSeg.waveformPeaks; + this.render(); } - audioBuffer = await this.audioContext.decodeAudioData(bytes.buffer); - } else { - continue; + + if (!this._isAudioDecodingAllowed(mockSeg)) { + return; + } + + // Build audio buffer + let audioBuffer = item.originalSeg._audioBuffer; + if (!audioBuffer) { + if (mockSeg.audioFile || mockSeg._blobUrl) { + const parts = (mockSeg.audioFile || "").split(/[/\\\\]/); + const filename = parts.pop() || ''; + const subfolder = parts.join('/'); + const audioUrl = mockSeg._blobUrl || api.apiURL(`/view?filename=${encodeURIComponent(filename)}&type=input&subfolder=${encodeURIComponent(subfolder)}`); + + this._audioBufferCache = this._audioBufferCache || new Map(); + this._audioBufferPromises = this._audioBufferPromises || new Map(); + const cacheKey = mockSeg.audioFile || audioUrl; + + if (this._audioBufferCache.has(cacheKey)) { + audioBuffer = this._audioBufferCache.get(cacheKey); + } else if (this._audioBufferPromises.has(cacheKey)) { + audioBuffer = await this._audioBufferPromises.get(cacheKey); + } else { + const decodePromise = (async () => { + const resp = await fetch(audioUrl); + const arrayBuffer = await resp.arrayBuffer(); + return await this.audioContext.decodeAudioData(arrayBuffer); + })(); + this._audioBufferPromises.set(cacheKey, decodePromise); + try { + audioBuffer = await decodePromise; + this._audioBufferCache.set(cacheKey, audioBuffer); + } finally { + this._audioBufferPromises.delete(cacheKey); + } + } + item.originalSeg._audioBuffer = audioBuffer; + } else if (mockSeg.audioB64) { + const binaryString = window.atob(mockSeg.audioB64); + const len = binaryString.length; + const bytes = new Uint8Array(len); + for (let i = 0; i < len; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + audioBuffer = await this.audioContext.decodeAudioData(bytes.buffer); + item.originalSeg._audioBuffer = audioBuffer; + } else { + return; + } + } + + if (this._currentPlayId !== playId || !this.isPlaying) return; + + // Determine current playback position dynamically in Web Audio time + const currentPlayTime = this.audioContext.currentTime; + const elapsedSecSincePlayStart = currentPlayTime - this.playbackStartTime; + const currentFrameCalculated = this.playbackStartFrame + elapsedSecSincePlayStart * frameRate; + + // If playback has already moved beyond the segment end, skip playing it + if (currentFrameCalculated >= segEndFrame) return; + + let startTime, fileOffsetSec, playDurationSec; + + if (currentFrameCalculated < segStartFrame) { + // Segment starts in the future relative to current playback position + const waitFrames = segStartFrame - currentFrameCalculated; + const waitTimeSec = waitFrames / frameRate; + startTime = currentPlayTime + waitTimeSec; + fileOffsetSec = item.trimStart / frameRate; + playDurationSec = item.length / frameRate; + } else { + // Segment is already playing. Start immediately, but offset into the audio buffer + startTime = currentPlayTime; + const framesToSkip = currentFrameCalculated - segStartFrame; + fileOffsetSec = (item.trimStart + framesToSkip) / frameRate; + playDurationSec = (item.length - framesToSkip) / frameRate; + } + + if (playDurationSec <= 0) return; + + const bufferNode = this.audioContext.createBufferSource(); + bufferNode.buffer = audioBuffer; + bufferNode["connect"](this.audioContext.destination); + bufferNode.start(startTime, fileOffsetSec, playDurationSec); + + this.activeAudioNodes.push(bufferNode); + } catch (err) { + console.error("Playback decode error for segment:", err); } - if (this._currentPlayId !== playId || !this.isPlaying) return; - - const framesToSkipInSegment = Math.max(0, this.currentFrame - segStartFrame); - const waitFrames = Math.max(0, segStartFrame - this.currentFrame); - const waitTimeSec = waitFrames / frameRate; - - const fileOffsetFrames = seg.trimStart + framesToSkipInSegment; - const fileOffsetSec = fileOffsetFrames / frameRate; - - const playDurationFrames = seg.length - framesToSkipInSegment; - const playDurationSec = playDurationFrames / frameRate; - - if (playDurationSec <= 0) continue; - - const bufferNode = this.audioContext.createBufferSource(); - bufferNode.buffer = audioBuffer; - bufferNode["connect"](this.audioContext.destination); - - const startTime = this.audioContext.currentTime + waitTimeSec; - bufferNode.start(startTime, fileOffsetSec, playDurationSec); - - this.activeAudioNodes.push(bufferNode); - } catch (err) { - console.error("Playback decode error for segment:", err); - } + })(); } if (this._currentPlayId !== playId || !this.isPlaying) return; @@ -3761,22 +10987,111 @@ class TimelineEditor { const visualDurationFrames = this.getVisualDurationFrames(); const durationFrames = this.getDurationFrames(); + let loopBound, stopBound; + if (this.retakeMode) { + const retakeLimit = this.timeline.retakeVideo ? (this.timeline.retakeVideo.videoDurationFrames || durationFrames) : durationFrames; + loopBound = retakeLimit; + stopBound = retakeLimit; + } else { + loopBound = (this.playbackStartFrame >= durationFrames) ? visualDurationFrames : durationFrames; + stopBound = visualDurationFrames; + } + if (this.isLooping) { - const loopBound = (this.playbackStartFrame >= durationFrames) ? visualDurationFrames : durationFrames; if (this.currentFrame >= loopBound) { this.currentFrame = 0; this.playAudio(); // Restart playback return; } } else { - if (this.currentFrame >= visualDurationFrames) { - this.currentFrame = visualDurationFrames; + if (this.currentFrame >= stopBound) { + this.currentFrame = stopBound; this.pauseAudio(); this.render(); return; } } + // Sync video playback + if (this.retakeMode) { + if (this.timeline.retakeVideo) { + const retakeVid = this.timeline.retakeVideo; + this._ensureVideoEl(retakeVid); + if (retakeVid.videoEl) { + const expectedSec = this.currentFrame / frameRate; + if (retakeVid.videoEl.paused && !retakeVid.videoEl.seeking) { + retakeVid.videoEl.currentTime = expectedSec; + retakeVid.videoEl.muted = false; + retakeVid.videoEl.play().catch(e => console.warn("Retake video play prevented", e)); + } else if (!retakeVid.videoEl.paused && Math.abs(retakeVid.videoEl.currentTime - expectedSec) > 0.5) { + retakeVid.videoEl.currentTime = expectedSec; + } + } + } + // Pause all other video elements + const allSegments = [...(this.timeline.segments || []), ...(this.timeline.motionSegments || [])]; + for (const seg of allSegments) { + if (seg.videoEl && !seg.videoEl.paused) { + seg.videoEl.pause(); + } + } + } else { + const activeSegments = (this._isDragging && this._previewSegments && this.selectionType !== "audio") ? this._previewSegments : this.timeline.segments; + const activeSeg = activeSegments.find(s => s.type === "video" && this.currentFrame >= s.start && this.currentFrame < s.start + s.length); + const activeVideoEl = activeSeg ? activeSeg.videoEl : null; + + for (const seg of activeSegments) { + if (seg.type === "video" && seg.videoEl) { + if (seg === activeSeg) { + const expectedSec = (seg.trimStart + (this.currentFrame - seg.start)) / frameRate; + if (seg.videoEl.paused && !seg.videoEl.seeking) { + // Not playing and no seek in flight — start a fresh seek+play + seg.videoEl.currentTime = expectedSec; + seg.videoEl.play().catch(e => console.warn("Video play prevented", e)); + } else if (!seg.videoEl.paused && Math.abs(seg.videoEl.currentTime - expectedSec) > 0.5) { + // Already playing but drifted — resync + seg.videoEl.currentTime = expectedSec; + } + // If paused && seeking: a seek+play is already in flight, let it finish + } else { + // Only pause if this segment's video element is NOT shared with the currently active segment + if (seg.videoEl !== activeVideoEl && !seg.videoEl.paused) { + seg.videoEl.pause(); + } + } + } + } + } + + // Sync motion playback + if (!this.retakeMode) { + const activeMotionSegments = (this._isDragging && this._previewSegments && this.selectionType === "motion") ? this._previewSegments : this.timeline.motionSegments; + const activeMotionSeg = activeMotionSegments.find(s => s.type === "motion_video" && this.currentFrame >= s.start && this.currentFrame < s.start + s.length); + const activeMotionVideoEl = activeMotionSeg ? activeMotionSeg.videoEl : null; + + for (const seg of activeMotionSegments) { + if (seg.type === "motion_video" && seg.videoEl) { + if (seg === activeMotionSeg) { + const expectedSec = (seg.trimStart + (this.currentFrame - seg.start)) / frameRate; + if (seg.videoEl.paused && !seg.videoEl.seeking) { + // Not playing and no seek in flight — start a fresh seek+play + seg.videoEl.currentTime = expectedSec; + seg.videoEl.play().catch(e => console.warn("Video play prevented", e)); + } else if (!seg.videoEl.paused && Math.abs(seg.videoEl.currentTime - expectedSec) > 0.5) { + // Already playing but drifted — resync + seg.videoEl.currentTime = expectedSec; + } + // If paused && seeking: a seek+play is already in flight, let it finish + } else { + // Only pause if this segment's video element is NOT shared with the currently active motion segment + if (seg.videoEl !== activeMotionVideoEl && !seg.videoEl.paused) { + seg.videoEl.pause(); + } + } + } + } + } + this.render(); this._playLoopId = requestAnimationFrame(loop); }; @@ -3792,6 +11107,41 @@ class TimelineEditor { try { this.audioContext.suspend(); } catch (e) { } } + if (this.retakeMode && this.timeline.retakeVideo) { + const retakeVid = this.timeline.retakeVideo; + if (retakeVid.videoEl) { + if (!retakeVid.videoEl.paused) { + retakeVid.videoEl.pause(); + } + retakeVid.videoEl.muted = true; // Mute again on pause/stop to prevent transient audio bursts + retakeVid.videoEl.currentTime = this.currentFrame / this.getFrameRate(); + } + } else { + // Sync video segments on pause + for (const seg of this.timeline.segments) { + if (seg.type === "video" && seg.videoEl) { + if (!seg.videoEl.paused) { + seg.videoEl.pause(); + } + if (this.currentFrame >= seg.start && this.currentFrame < seg.start + seg.length) { + seg.videoEl.currentTime = (seg.trimStart + (this.currentFrame - seg.start)) / this.getFrameRate(); + } + } + } + + // Sync motion segments on pause + for (const seg of this.timeline.motionSegments) { + if (seg.type === "motion_video" && seg.videoEl) { + if (!seg.videoEl.paused) { + seg.videoEl.pause(); + } + if (this.currentFrame >= seg.start && this.currentFrame < seg.start + seg.length) { + seg.videoEl.currentTime = (seg.trimStart + (this.currentFrame - seg.start)) / this.getFrameRate(); + } + } + } + } + for (let node of this.activeAudioNodes) { try { node.stop(); } catch (e) { } try { node.disconnect(); } catch (e) { } @@ -3816,26 +11166,70 @@ const APPENDED_WIDGET_DEFAULTS = [ app.registerExtension({ name: "LTXDirector_Koolook", async beforeRegisterNodeDef(nodeType, nodeData, app) { - if ( - nodeData.name === "LTXDirector__koolook" || - nodeData.name === "LTXDirector__koolook_v1_3_2" - ) { + if (nodeData.name === "LTXDirector__koolook") { const onNodeCreated = nodeType.prototype.onNodeCreated; nodeType.prototype.onNodeCreated = function () { if (onNodeCreated) onNodeCreated.apply(this, arguments); + if (!this.properties) this.properties = {}; + const DEFAULTS = { + global_prompt: "", + mainTrackEnabled: true, + audioTrackEnabled: true, + motionTrackEnabled: true, + audioTrackWasEnabledBeforeOverride: false, + inpaint_audio: true, + override_audio: false, + overrideAudio: false, + showFilenames: true, + use_custom_audio: false, + use_custom_motion: true, + frame_rate: 24, + display_mode: "seconds", + custom_width: 0, + custom_height: 0, + resize_method: "maintain aspect ratio", + divisible_by: 32, + img_compression: 18, + guide_strength: "", + local_prompts: "", + segment_lengths: "", + timeline_data: "{}", + epsilon: 0.001, + start_second: 0.0, + end_second: 5.0, + duration_seconds: 5.0, + start_frame: 0, + end_frame: 120, + duration_frames: 120, + }; + for (const [key, val] of Object.entries(DEFAULTS)) { + if (this.properties[key] === undefined) { + this.properties[key] = val; + } + } + for (const [name, def] of APPENDED_WIDGET_DEFAULTS) { if (!this.widgets?.find(w => w.name === name)) { this.addWidget("string", name, def, () => { }); } } + const isLiteGraph = !window.LiteGraph || !window.LiteGraph.vueNodesMode; for (const w of this.widgets) { - if (HIDDEN_WIDGET_NAMES.includes(w.name)) hideWidget(w); + if (HIDDEN_WIDGET_NAMES.includes(w.name)) { + hideWidget(w); + if (isLiteGraph && this.inputs) { + const idx = this.inputs.findIndex(i => i.name === w.name); + if (idx !== -1 && this.inputs[idx].link == null) { + this.removeInput(idx); + } + } + } } // Set default width to be wider on creation (approx 2.5x default ~220px) - this.size[0] = 1000; + this.size[0] = 1375; // Force default for img_compression if not set (ComfyUI sometimes skips optional defaults) const compWidget = this.widgets?.find(w => w.name === "img_compression"); @@ -3843,19 +11237,81 @@ app.registerExtension({ compWidget.value = 18; } - // Hide global prompt by default on creation without destroying its DOM element - const globalPromptWidget = this.widgets?.find(w => w.name === "global_prompt"); - if (globalPromptWidget) { - if (!globalPromptWidget.options) globalPromptWidget.options = {}; - globalPromptWidget.options.hidden = true; - globalPromptWidget.hidden = true; - globalPromptWidget.computeSize = () => [0, 0]; - setTimeout(() => { - if (globalPromptWidget.element) globalPromptWidget.element.style.display = "none"; - }, 0); - } + const self = this; + this._syncGlobalPromptFromLink = function () { + const globalInput = self.inputs?.find(i => i.name === "global_prompt"); + if (globalInput && globalInput.link !== null && globalInput.link !== undefined) { + const link = app.graph.links[globalInput.link]; + if (link) { + const originNode = app.graph.getNodeById(link.origin_id); + if (originNode) { + // Usually string values are in widgets[0] for primitives + if (originNode.widgets && originNode.widgets.length > 0) { + const val = originNode.widgets[0].value; + if (self._timelineEditor && self._timelineEditor.globalPromptInput) { + const isRetake = self._timelineEditor.retakeMode; + const currentValInEditor = isRetake ? (self._timelineEditor.timeline.retake_global_prompt || "") : (self._timelineEditor.timeline.global_prompt || ""); + if (val !== currentValInEditor) { + if (isRetake) { + self._timelineEditor.timeline.retake_global_prompt = val; + } else { + self._timelineEditor.timeline.global_prompt = val; + } + self._timelineEditor.globalPromptInput.value = val; + if (self._timelineEditor.selectionType === "motion") { + self._timelineEditor.promptInput.value = val; + } + if (self.properties) { + self.properties.global_prompt = val; + } + } else if (self._timelineEditor.globalPromptInput.value !== val) { + self._timelineEditor.globalPromptInput.value = val; + } + } + } + } + } + } else { + if (self.properties && self._timelineEditor && self._timelineEditor.globalPromptInput) { + const val = self.properties.global_prompt || ""; + const isRetake = self._timelineEditor.retakeMode; + const currentValInEditor = isRetake ? (self._timelineEditor.timeline.retake_global_prompt || "") : (self._timelineEditor.timeline.global_prompt || ""); + if (val !== currentValInEditor) { + if (isRetake) { + self._timelineEditor.timeline.retake_global_prompt = val; + } else { + self._timelineEditor.timeline.global_prompt = val; + } + self._timelineEditor.globalPromptInput.value = val; + if (self._timelineEditor.selectionType === "motion") { + self._timelineEditor.promptInput.value = val; + } + } else if (self._timelineEditor.globalPromptInput.value !== val) { + self._timelineEditor.globalPromptInput.value = val; + } + } + } + }; + + const origOnConnectionsChange = this.onConnectionsChange; + this.onConnectionsChange = function (type, index, connected, link_info) { + if (origOnConnectionsChange) { + origOnConnectionsChange.apply(this, arguments); + } + self._syncGlobalPromptFromLink(); + }; + + const origOnDrawForeground = this.onDrawForeground; + this.onDrawForeground = function (ctx) { + if (origOnDrawForeground) { + origOnDrawForeground.apply(this, arguments); + } + self._syncGlobalPromptFromLink(); + }; const container = document.createElement("div"); + + container.style.boxSizing = "border-box"; const widget = this.addDOMWidget("timeline_ui", "timeline_ui", container, { getValue: () => "", setValue: () => { }, @@ -3863,10 +11319,12 @@ app.registerExtension({ widget.computeSize = function (width) { const canvasH = self._timelineEditor ? self._timelineEditor.canvasHeight : CANVAS_HEIGHT; - return [width, canvasH + 235]; + const propH = self._timelineEditor ? (self._timelineEditor.propHeight || 90) : 90; + const globalPropH = self._timelineEditor ? (self._timelineEditor.globalPropHeight || 60) : 60; + const nodeWidth = self.size?.[0] || width || 1375; + return [Math.max(10, nodeWidth - 30), canvasH + propH + globalPropH + 160]; }; - const self = this; setTimeout(() => { try { self._timelineEditor = new TimelineEditor(self, container, widget); @@ -3876,6 +11334,15 @@ app.registerExtension({ }, 0); }; + const onResize = nodeType.prototype.onResize; + nodeType.prototype.onResize = function (size) { + const out = onResize?.apply(this, arguments); + if (this._timelineEditor) { + requestAnimationFrame(() => this._timelineEditor?.syncLayoutToNode()); + } + return out; + }; + const onRemoved = nodeType.prototype.onRemoved; nodeType.prototype.onRemoved = function () { this._timelineEditor?.destroy(); @@ -3884,7 +11351,140 @@ app.registerExtension({ const onConfigure = nodeType.prototype.onConfigure; nodeType.prototype.onConfigure = function (info) { - const out = onConfigure?.apply(this, arguments); + // 1. Call parent/original onConfigure first, with info.widgets_values intact + const out = onConfigure ? onConfigure.apply(this, arguments) : undefined; + + if (info.properties) { + this.properties = { ...this.properties, ...info.properties }; + } + + console.log("[LTXDirector debug] onConfigure called. info.widgets_values:", info.widgets_values ? JSON.stringify(info.widgets_values) : "undefined"); + + // Helper to set widget value, sync DOM element, and trigger callbacks safely + const setWidgetValue = (w, val) => { + if (!w) return; + w.value = val; + if (w.element) { + if (w.element.type === "checkbox") { + w.element.checked = !!val; + } else { + w.element.value = val; + } + } + if (w.callback) { + try { + w.callback(val); + } catch (e) { + // ignore + } + } + }; + + // 2. Check if we have serialized properties. If so, restore widgets from properties! + if (info.properties && info.properties.has_serialized_properties) { + console.log("[LTXDirector debug] Restoring widgets from properties"); + if (this.widgets) { + for (const w of this.widgets) { + if (w.name && this.properties[w.name] !== undefined) { + setWidgetValue(w, this.properties[w.name]); + } + } + } + } else if (info.widgets_values) { + // Fallback to name-based schema mapping for older workflows + console.log("[LTXDirector debug] Restoring widgets via fallback name-based schema mapping"); + const SCHEMA_19 = [ + "start_frame", "end_frame", "duration_frames", + "timeline_data", "use_custom_audio", "use_custom_motion", "inpaint_audio", "local_prompts", "segment_lengths", + "epsilon", "frame_rate", "display_mode", "guide_strength", "custom_width", "custom_height", + "resize_method", "divisible_by", "img_compression", "timeline_ui" + ]; + const SCHEMA_21_NO_INPAINT = [ + "start_second", "end_second", "duration_seconds", "start_frame", "end_frame", "duration_frames", + "timeline_data", "local_prompts", "segment_lengths", "epsilon", "guide_strength", + "use_custom_audio", "use_custom_motion", "frame_rate", "display_mode", "custom_width", "custom_height", + "resize_method", "divisible_by", "img_compression", "timeline_ui" + ]; + const SCHEMA_22_NO_INPAINT = [ + "start_second", "end_second", "duration_seconds", "start_frame", "end_frame", "duration_frames", + "timeline_data", "local_prompts", "segment_lengths", "epsilon", "guide_strength", + "use_custom_audio", "use_custom_motion", "frame_rate", "display_mode", "custom_width", "custom_height", + "resize_method", "divisible_by", "img_compression", "override_audio", "timeline_ui" + ]; + const SCHEMA_22_WITH_INPAINT = [ + "start_second", "end_second", "duration_seconds", "start_frame", "end_frame", "duration_frames", + "timeline_data", "use_custom_audio", "use_custom_motion", "inpaint_audio", "local_prompts", "segment_lengths", + "epsilon", "frame_rate", "display_mode", "guide_strength", "custom_width", "custom_height", + "resize_method", "divisible_by", "img_compression", "timeline_ui" + ]; + const SCHEMA_23 = [ + "start_second", "end_second", "duration_seconds", "start_frame", "end_frame", "duration_frames", + "timeline_data", "use_custom_audio", "use_custom_motion", "inpaint_audio", "local_prompts", "segment_lengths", + "epsilon", "frame_rate", "display_mode", "guide_strength", "custom_width", "custom_height", + "resize_method", "divisible_by", "img_compression", "override_audio", "timeline_ui" + ]; + + const ALL_WIDGET_DEFAULTS = { + inpaint_audio: true, + override_audio: false, + use_custom_audio: false, + use_custom_motion: true, + frame_rate: 24, + display_mode: "seconds", + custom_width: 0, + custom_height: 0, + resize_method: "maintain aspect ratio", + divisible_by: 32, + img_compression: 18, + guide_strength: "", + local_prompts: "", + segment_lengths: "", + timeline_data: "{}", + epsilon: 0.001, + start_second: 0.0, + end_second: 5.0, + duration_seconds: 5.0, + start_frame: 0, + end_frame: 120, + duration_frames: 120, + }; + + let names = SCHEMA_23; + const len = info.widgets_values.length; + if (len <= 19) { + names = SCHEMA_19; + } else if (len === 21) { + names = SCHEMA_21_NO_INPAINT; + } else if (len === 22) { + if (typeof info.widgets_values[13] === "number") { + names = SCHEMA_22_NO_INPAINT; + } else { + names = SCHEMA_22_WITH_INPAINT; + } + } + + if (this.widgets) { + for (const w of this.widgets) { + const schemaIdx = names.indexOf(w.name); + if (schemaIdx !== -1 && schemaIdx < len) { + setWidgetValue(w, info.widgets_values[schemaIdx]); + } else if (ALL_WIDGET_DEFAULTS.hasOwnProperty(w.name)) { + setWidgetValue(w, ALL_WIDGET_DEFAULTS[w.name]); + } + } + } + + // Populate properties with these restored values + if (this.widgets) { + for (const w of this.widgets) { + if (w.name && w.value !== undefined) { + this.properties[w.name] = w.value; + } + } + } + this.properties.has_serialized_properties = true; + } + for (const [name, def] of APPENDED_WIDGET_DEFAULTS) { const w = this.widgets.find(x => x.name === name); if (w && (w.value == null || w.value === "")) w.value = def; @@ -3892,19 +11492,90 @@ app.registerExtension({ setTimeout(() => { if (this._timelineEditor) { - this._timelineEditor.timeline = parseInitial(this._timelineEditor.timelineDataWidget?.value); - this._timelineEditor.loadImages(); + console.log("[LTXDirector debug] setTimeout sync block called."); + console.log("[LTXDirector debug] setTimeout: timelineDataWidget value:", this._timelineEditor.timelineDataWidget?.value); + const tl = parseInitial(this._timelineEditor.timelineDataWidget?.value); + console.log("[LTXDirector debug] setTimeout: parsed timeline:", JSON.stringify(tl)); + this._timelineEditor.timeline = tl; + + // Sync editor states from the parsed timeline object (the absolute source of truth) + this._timelineEditor.mainTrackEnabled = tl.mainTrackEnabled !== false; + this._timelineEditor.audioTrackEnabled = tl.audioTrackEnabled !== false; + this._timelineEditor.motionTrackEnabled = tl.motionTrackEnabled !== false; + this._timelineEditor.retakeMode = tl.retakeMode === true; + this._timelineEditor._audioTrackWasEnabledBeforeOverride = !!this.properties.audioTrackWasEnabledBeforeOverride; + + // Sync properties to match + this.properties.mainTrackEnabled = this._timelineEditor.mainTrackEnabled; + this.properties.audioTrackEnabled = this._timelineEditor.audioTrackEnabled; + this.properties.motionTrackEnabled = this._timelineEditor.motionTrackEnabled; + this.properties.retakeMode = this._timelineEditor.retakeMode; + if (tl.showFilenames !== undefined) { + this.properties.showFilenames = tl.showFilenames; + } + if (tl.overrideAudio !== undefined) { + this.properties.overrideAudio = tl.overrideAudio; + } + if (tl.inpaint_audio !== undefined) { + this.properties.inpaint_audio = tl.inpaint_audio; + } + + // Sync widgets to match the timeline data + const inpaintWidget = this.widgets?.find(w => w.name === "inpaint_audio"); + if (inpaintWidget && tl.inpaint_audio !== undefined) { + inpaintWidget.value = tl.inpaint_audio; + } + const overrideWidget = this.widgets?.find(w => w.name === "override_audio"); + if (overrideWidget && tl.overrideAudio !== undefined) { + overrideWidget.value = tl.overrideAudio; + } + + this._timelineEditor.loadMedia(); this._timelineEditor.selectionType = "image"; this._timelineEditor.selectedIndex = clamp( this._timelineEditor.selectedIndex, -1, Math.max(-1, this._timelineEditor.timeline.segments.length - 1) ); + this._timelineEditor.updateRetakeUIState(); this._timelineEditor.updateUIFromSelection(); + this._timelineEditor.syncWidgetsAndUI(); + this._timelineEditor.syncLayoutToNode(); this._timelineEditor.render(); } }, 0); + return out; }; + + const onSerialize = nodeType.prototype.onSerialize; + nodeType.prototype.onSerialize = function (info) { + if (onSerialize) { + onSerialize.apply(this, arguments); + } + + // Sync all current widgets to properties + if (this.widgets) { + for (const w of this.widgets) { + if (w.name && w.value !== undefined) { + this.properties[w.name] = w.value; + } + } + } + + // Sync timeline editor state if it exists + if (this._timelineEditor) { + this.properties.mainTrackEnabled = this._timelineEditor.mainTrackEnabled !== false; + this.properties.audioTrackEnabled = this._timelineEditor.audioTrackEnabled !== false; + this.properties.motionTrackEnabled = this._timelineEditor.motionTrackEnabled !== false; + this.properties.audioTrackWasEnabledBeforeOverride = !!this._timelineEditor._audioTrackWasEnabledBeforeOverride; + } + + // Mark that properties have been serialized + this.properties.has_serialized_properties = true; + + // Ensure info.properties is populated with all our properties + info.properties = { ...this.properties }; + }; } }, });