diff --git a/camera_move.py b/camera_move.py index 81b0f71c..7da73a34 100644 --- a/camera_move.py +++ b/camera_move.py @@ -29,10 +29,15 @@ import torch MODES = ("off", "manual") +def finite(v, default, lo, hi): + """`v` clamped to [lo, hi]; anything unreadable (NaN) becomes `default`.""" + v = float(v) + return default if v != v else min(max(v, lo), hi) + + def clamp_pan(v): """A pan share in [-1, 1]; anything unreadable (NaN) is no pan.""" - v = float(v) - return 0.0 if v != v else min(max(v, -1.0), 1.0) + return finite(v, 0.0, -1.0, 1.0) @dataclass(frozen=True) @@ -86,7 +91,7 @@ def warp(video, move, sigma, generator): fresh noise at the current noise level `sigma` (x = (1 - sigma) * picture + sigma * noise with the picture unknown there), so they look like the rest of a noisy latent.""" c, t, h, w = video.shape - iy, ix = sources(move, t, h, w) + iy, ix = (v.to(video.device) for v in sources(move, t, h, w)) ok_y, ok_x = (iy >= 0) & (iy < h), (ix >= 0) & (ix < w) fresh = torch.randn(video.shape, generator=generator).to(video) * float(sigma) out = torch.empty_like(video) @@ -102,7 +107,7 @@ def unwarp(moved, original, move): Cells the move never showed keep `original`'s value, so the sampler's state there is left as it was.""" c, t, h, w = moved.shape - iy, ix = sources(move, t, h, w) + iy, ix = (v.to(moved.device) for v in sources(move, t, h, w)) ok_y, ok_x = (iy >= 0) & (iy < h), (ix >= 0) & (ix < w) out = original.clone() for f in range(t): diff --git a/movie_editor/frontend/pipeline_caps.js b/movie_editor/frontend/pipeline_caps.js index c5438704..681ac641 100644 --- a/movie_editor/frontend/pipeline_caps.js +++ b/movie_editor/frontend/pipeline_caps.js @@ -183,6 +183,9 @@ stas: "Steers H3's own block outputs by their packed frame layout; LTX's blocks are hooked " + "differently.", + camera_move: + "Moves the picture inside H3's packed latent (picture and sound in one sequence); it is " + + "written for that layout and is not built for LTX.", h3_video_detail: "Scales H3's own final layer, which reads video and audio on separate rows. LTX has no " + "layer of that shape, so the sampler has nothing to scale.", @@ -195,6 +198,7 @@ "h3_shadow_negative_start_percent", "h3_shadow_negative_end_percent", "h3_shadow_negative_compose", "late_guidance_strength", "late_guidance_block", "stas_alpha", "stas_block", + "camera_pan_x", "camera_pan_y", "camera_zoom", "camera_focus_x", "camera_focus_y", "camera_step", ]; // Sub-settings of identity transfer: meaningless wherever the feature itself cannot run, diff --git a/samplers.py b/samplers.py index 2a4bba6b..00043e33 100644 --- a/samplers.py +++ b/samplers.py @@ -875,6 +875,17 @@ def _step_position(args): return int(hit[0]), int(sched.shape[0]) - 1 +# Samplers that call the model exactly once per schedule step, which the camera move's +# original-frame bookkeeping relies on (heun, dpmpp_2s, dpm_2 ... call it twice: the second +# call would be read as the next step). +_CAMERA_SAFE_SAMPLERS = frozenset({ + "sample_euler", "sample_euler_ancestral", "sample_lcm", "sample_dpmpp_2m", + "sample_dpmpp_2m_sde", "sample_dpmpp_3m_sde", "sample_res_multistep", "sample_lms", + "sample_ipndm", "sample_ipndm_v", "sample_deis", "sample_gradient_estimation", + "sample_er_sde", "sample_ddpm", +}) + + class _InputSteer: """Carry a step's edit into the NEXT step's input instead of editing its answer (H3). @@ -4314,6 +4325,10 @@ class FunPackLTXAVSceneChainSampler: "picture (second pass, latent anchor or carried overlap), so it stands down") else: model, _camera_stats = self._install_camera_move(model, self._camera_move, int(seed)) + _pins = self._conditioning_value(positive, "minimax_keyframes") or [] + if any(int(p.get("resolved_frame_index", 0)) > 0 for p in _pins): + print("[FunPackSceneChain] camera move: keyframe pins after the first frame " + "stay where they are while the picture moves, so they may fight the move") _phrase_probe = self._install_phrase_probe(model, positive, latent) if _phrase_probe is not None: @@ -8767,6 +8782,7 @@ class FunPackLTXAVSceneChainSampler: try: pos = where._where(args) if pos is None: + stats["why"] = "this sampler's calls are not on the schedule" return _plain(apply_fn, args) i, n, _key, _sched = pos c = args.get("c") if isinstance(args.get("c"), dict) else {} @@ -8783,6 +8799,9 @@ class FunPackLTXAVSceneChainSampler: if span is not None: off, sz, shape = span cc, tt, hh, ww = (int(v) for v in tuple(shape)[-4:]) + if tt < 2: + stats["why"] = "one latent frame has nothing to move across" + return _plain(apply_fn, args) vid = x[..., off:off + sz].reshape(cc, tt, hh, ww) elif x.dim() == 5: off, sz, vid = None, None, x[0] @@ -9864,9 +9883,9 @@ class FunPackLTXAVSceneChainSampler: if camera_move != "off": self._camera_move = _camera.Move( pan_x=_camera.clamp_pan(camera_pan_x), pan_y=_camera.clamp_pan(camera_pan_y), - zoom=min(max(float(camera_zoom), 0.5), 2.0), - focus_x=min(max(float(camera_focus_x), 0.0), 1.0), - focus_y=min(max(float(camera_focus_y), 0.0), 1.0), + zoom=_camera.finite(camera_zoom, 1.0, 0.5, 2.0), + focus_x=_camera.finite(camera_focus_x, 0.5, 0.0, 1.0), + focus_y=_camera.finite(camera_focus_y, 0.5, 0.0, 1.0), step=min(max(int(camera_step), 1), 50)) if not self._is_h3: print("[FunPackSceneChain] camera move: Inactive | H3 only") @@ -9879,6 +9898,14 @@ class FunPackLTXAVSceneChainSampler: print("[FunPackSceneChain] camera move: Inactive | context windows split the " "clip into pieces that each see only some frames, so the move stands down") self._camera_move = None + else: + _fn = getattr(sampler, "sampler_function", None) + _name = getattr(_fn, "__name__", "") or type(sampler).__name__ + if _name not in _CAMERA_SAFE_SAMPLERS: + print(f"[FunPackSceneChain] camera move: Inactive | sampler {_name} may call " + f"the model more than once per step, which the move cannot follow; use " + f"one of: {', '.join(sorted(n.replace('sample_', '') for n in _CAMERA_SAFE_SAMPLERS))}") + self._camera_move = None if (shot_memory != "off" or tsr == "learned" or late_guidance == "learned" or stas == "learned"): if not refinement_key_input: diff --git a/tests/test_camera_move.py b/tests/test_camera_move.py index 9ced0639..a7e827c5 100644 --- a/tests/test_camera_move.py +++ b/tests/test_camera_move.py @@ -199,3 +199,25 @@ def test_the_sampler_inputs_agree_with_the_editor_defaults(): "camera_focus_x", "camera_focus_y", "camera_step"]: assert opt[name][1]["default"] == settings_card._EDITOR_DEFAULTS[name], name assert opt["camera_move"][1]["default"] == "off" + + +def test_the_move_works_on_an_accelerator_when_there_is_one(): + dev = ("cuda" if torch.cuda.is_available() + else "mps" if getattr(torch.backends, "mps", None) and torch.backends.mps.is_available() else None) + if dev is None: + import pytest + pytest.skip("no GPU here; the rental is where this runs") + x = torch.randn(2, 9, 12, 20, device=dev) + move = cm.Move(pan_x=0.5, zoom=1.3) + out = cm.warp(x, move, 0.5, _gen()) + back = cm.unwarp(out, x, move) + assert out.device == x.device and back.device == x.device and torch.isfinite(out).all() + assert torch.equal(cm.warp(x.cpu(), move, 0.5, _gen()).to(dev), out) # same as the CPU result + + +def test_only_single_call_samplers_are_allowed_and_bad_numbers_are_neutral(): + import samplers + safe = samplers._CAMERA_SAFE_SAMPLERS + assert {"sample_euler", "sample_lcm"} <= safe + assert not ({"sample_heun", "sample_dpmpp_2s_ancestral", "sample_dpm_2", "sample_dpm_adaptive"} & safe) + assert cm.finite(float("nan"), 1.0, 0.5, 2.0) == 1.0 and cm.finite(9, 1.0, 0.5, 2.0) == 2.0