diff --git a/app/legacy/settings_window.js b/app/legacy/settings_window.js index aa14aa64..3537b977 100644 --- a/app/legacy/settings_window.js +++ b/app/legacy/settings_window.js @@ -660,7 +660,7 @@ rows.append(infoRow( recording ? "Rated generations recorded (this key)" : "Recorded earlier (not recording now)", - biState ? `${runs} (${biState.n_liked} liked / ${biState.n_disliked} disliked)` + biState ? `${runs} (${biState.n_liked} liked / ${biState.n_disliked} disliked) on key '${biState.key}'` : "0", recording && runs > 0 ? true : null)); rows.append(actionRow("Save this key's measurement", "Downloads every recorded profile as one file. A rental gets replaced and " @@ -702,8 +702,8 @@ paint(); if (window.MovieEditorAPI) { window.MovieEditorAPI.blockInfluenceStatus(key) - .then((s) => { biState = s; paint(); }) - .catch(() => {}); + .then((s) => { biState = s; biError = ""; paint(); }) + .catch((e) => { biState = null; biError = String(e.message || e); paint(); }); } return box; } diff --git a/core/input_steer.py b/core/input_steer.py index 8a6dfc98..662860e2 100644 --- a/core/input_steer.py +++ b/core/input_steer.py @@ -24,6 +24,9 @@ that step: pushes are filed by the step that made them. A schedule that restarts from . import dit_hooks, log +# Below this share of the input it joins, a push is reported as barely acting (v4: 3%). +FLOOR = 0.03 + class Call: @@ -41,13 +44,29 @@ class Call: class Steer: def __init__(self, what: str): self.what = what + self.reset() + + def reset(self): + """A run starts (or ends): nothing held, nothing counted.""" self._pushes = {} # key -> {step: delta} self._seen = {} # key -> last step index fed - self._edit, self._edits = 0.0, 0 + self._made, self._delivered = 0, 0 + self._reach = 0.0 # largest delivered push, as a share of the input it joined - def effect(self): - """Mean size of the edits made, relative to the answer, or None when none were.""" - return self._edit / self._edits if self._edits else None + def attach(self, patcher, key): + """Reset around every sampling call. The node that installs the modifier is cached + by ComfyUI, so this object outlives a run; without this a run that made no edit + would report the previous run's.""" + from comfy.patcher_extension import WrappersMP + + def outer(executor, *args, **kwargs): + self.reset() + try: + return executor(*args, **kwargs) + finally: + self._pushes = {} # latent-sized tensors must not outlive the run + + patcher.add_wrapper_with_key(WrappersMP.OUTER_SAMPLE, key, outer) def _say(self, level, message, key): log.once(f"input steer {self.what}:{key}", level, f"FunPack {self.what}", message) @@ -81,10 +100,13 @@ class Steer: if i < self._seen.get(key, 0): # a new sampling pass: nothing carries over held.clear() self._seen[key] = i - push = held.get(i - 1) + push = held.pop(i - 1, None) # delivered once; a repeated sigma cannot re-deliver it if push is not None: if tuple(push.shape) == tuple(x.shape): - x = x + (1.0 - float(t.max())) * push.to(x.device, x.dtype) + add = (1.0 - float(t.max())) * push.to(x.device, x.dtype) + self._delivered += 1 + self._reach = max(self._reach, float(add.norm() / x.norm().clamp(min=1e-8))) + x = x + add else: self._say(log.WARNING, "Inactive | the latent changed size between steps, " "so that step went unsteered", "resized") @@ -95,20 +117,25 @@ class Steer: return Call(x, 0.0, final=True, keep=lambda out, _steered: out) def keep(out, steered): - try: - self._edit += float((steered - out).norm() / out.norm().clamp(min=1e-8)) - self._edits += 1 - except Exception: # noqa: BLE001 -- a measurement must never break sampling - pass + self._made += 1 held[i] = (steered - out).detach() return out return Call(x, dit_hooks.late_half(to, ahead=1), keep=keep) def _report(self): - size = self.effect() - if size is None: + """Said on the last step. Reach is what counts: an edit made is not an edit that + arrived, and one that is a sliver of the input it joins changes nothing the + model can feel.""" + if not self._made: self._say(log.ALERT, "Inactive | no step made an edit this run", "result") + elif not self._delivered: + self._say(log.ALERT, f"Inactive | {self._made} edit(s) made but none reached a later " + f"step's input", "result") + elif self._reach < FLOOR: + self._say(log.ALERT, f"Inactive | barely acts: the largest push was {self._reach:.1%} of " + f"the input it joined (under {FLOOR:.0%}); more steps or a " + f"higher strength would let it be felt", "result") else: - self._say(log.INFO, f"Active | edits averaged {size:.1%} of the picture's size, " - f"each carried into the next step's input", "result") + self._say(log.INFO, f"Active | {self._delivered} push(es) carried into later steps, the " + f"largest {self._reach:.1%} of the input it joined", "result") diff --git a/core/tests/test_input_steer.py b/core/tests/test_input_steer.py index d6f12ea7..8b46dcc3 100644 --- a/core/tests/test_input_steer.py +++ b/core/tests/test_input_steer.py @@ -105,3 +105,38 @@ def test_a_one_step_schedule_says_there_is_nothing_to_carry_into(): s, x = input_steer.Steer("t"), _x() step = s.begin(x, _t(0, 1), _to(0, 1)) assert step.final and any("1-step" in r["message"] for r in log.history()) + + +def _said(text): + return any(text in r["message"] for r in log.history()) + + +def test_a_sliver_of_a_push_is_reported_as_barely_acting_and_a_real_one_as_active(): + x, out = _x(), _x() + for size, word in ((1e-4, "barely acts"), (1.0, "Active |")): + log._reset() + s = input_steer.Steer("t") + _edit(s, 2, x, out, torch.ones(1, 1, 12) * size) + s.begin(x, _t(3), _to(3)) + assert _said(word), (size, [r["message"] for r in log.history()]) + + +def test_a_second_run_on_the_same_object_does_not_report_the_first_runs_edits(): + log._reset() + s, x, out = input_steer.Steer("t"), _x(), _x() + _edit(s, 2, x, out, torch.ones(1, 1, 12)) + s.begin(x, _t(3), _to(3)) + s.reset() # what the sampling wrapper does per run + log._reset() + s.begin(x, _t(3), _to(3)) + assert _said("Inactive | no step made an edit") and not _said("Active |") + + +def test_a_repeated_sigma_cannot_count_an_edit_that_never_arrived(): + s, x, out = input_steer.Steer("t"), _x(), _x() + sched = torch.tensor([1.0, 0.8, 0.8, 0.5, 0.0]) + to = lambda i: {"sample_sigmas": sched, "sigmas": sched[i:i + 1]} + s.begin(x, sched[2:3], to(2)).keep(out, out + 1) # index reads 1, the first match: gate 0, no edit + log._reset() + s.begin(x, sched[3:4], to(3)) + assert not _said("Active |") diff --git a/modules/sampling/block_influence/__init__.py b/modules/sampling/block_influence/__init__.py index 8212b120..531218f7 100644 --- a/modules/sampling/block_influence/__init__.py +++ b/modules/sampling/block_influence/__init__.py @@ -107,15 +107,20 @@ def routes(table, base, web): @table.post(base + "/enabled") async def _enabled(req): body = await req.json() + try: + state = measure.state(key_of(req, body)) # refuse BEFORE flipping the switch + except ValueError as exc: + return web.json_response({"why": str(exc)}, status=400) measure.set_enabled(bool(body.get("enabled"))) - return web.json_response(measure.state(key_of(req, body))) + return web.json_response({**state, "enabled": measure.enabled()}) @table.post(base + "/clear") async def _clear(req): body = await req.json() try: - measure.clear(key_of(req, body)) - return web.json_response(measure.state(key_of(req, body))) + key = measure.resolve(key_of(req, body)) + measure.clear(key) + return web.json_response(measure.state(key)) except ValueError as exc: return web.json_response({"why": str(exc)}, status=400) diff --git a/modules/sampling/block_influence/measure.py b/modules/sampling/block_influence/measure.py index f789bc04..753bbd26 100644 --- a/modules/sampling/block_influence/measure.py +++ b/modules/sampling/block_influence/measure.py @@ -145,16 +145,27 @@ def _kind(key): return kind +def resolve(key): + """The key to show. The panel asks for a placeholder ("default") when it does not know + which Taste key the runs write to, so a key with nothing recorded falls back to the key + the latest capture went to -- and the answer says which one it is.""" + if _kind(key).path().exists(): + return key + latest = registry.current().ask("taste_latest_key") + return latest if latest and _kind(latest).path().exists() else key + + def state(key) -> dict: """What Settings > Refinement & Taste shows for one key.""" + key = resolve(key) rows = _kind(key).rows() return {"key": key, "enabled": enabled(), "runs": len(rows), "min_per_group": MIN_PER_GROUP, **profile(rows)} def clear(key): - _kind(key).clear() + _kind(resolve(key)).clear() def path_of(key): - return _kind(key).path() + return _kind(resolve(key)).path() diff --git a/modules/sampling/block_influence/tests/test_block_influence.py b/modules/sampling/block_influence/tests/test_block_influence.py index 33c257e5..1c71fb08 100644 --- a/modules/sampling/block_influence/tests/test_block_influence.py +++ b/modules/sampling/block_influence/tests/test_block_influence.py @@ -173,3 +173,48 @@ def test_the_panel_routes_read_toggle_clear_and_export(tiny_h3): assert json.loads(out.text)["runs"] == 0 assert call("GET", "/b/export")[0] == 404 assert call("GET", "/b/status", key="no/slash")[0] == 400 + + +def test_a_bad_key_is_refused_before_the_switch_flips_and_a_placeholder_finds_the_real_key(tiny_h3): + from aiohttp import web + from modules.sampling import block_influence as mod + from modules.sampling.block_influence import measure + from modules.system.taste import store + import json + handlers = {} + + class Table: + def get(self, path): + return lambda fn: handlers.setdefault(("GET", path), fn) + + def post(self, path): + return lambda fn: handlers.setdefault(("POST", path), fn) + + mod.routes(Table(), "/b", web) + + class Req: + def __init__(self, body=None, key="default"): + self._body, self.rel_url = body, type("U", (), {"query": {"key": key}})() + + async def json(self): + return self._body + + out = asyncio.run(handlers[("POST", "/b/enabled")](Req({"enabled": True, "key": "no/slash"}))) + assert out.status == 400 and not measure.enabled() + patched, _ = _load(tiny_h3, key="mystyle") + measure.set_enabled(True) + tiny_h3.sample(patched) + store.rate("run-1", "liked") + state = json.loads(asyncio.run(handlers[("GET", "/b/status")](Req())).text) + assert state["key"] == "mystyle" and state["runs"] == 1 # "default" had nothing: the real key + + +def test_the_sliders_perturbed_passes_are_not_measured(tiny_h3): + from modules.sampling.block_influence import measure + from core import dit_hooks + patched, _ = _load(tiny_h3) + measure.set_enabled(True) + patched.model_options.setdefault("transformer_options", {})[dit_hooks.PROBE] = True + tiny_h3.sample(patched) # same flag the slider sets on its +/- passes + from modules.system.taste import store + assert not (store.ROOT / "fox" / "block_influence.pending.pt").exists() diff --git a/modules/sampling/dynashift/__init__.py b/modules/sampling/dynashift/__init__.py index 77abbcfd..0fcc9356 100644 --- a/modules/sampling/dynashift/__init__.py +++ b/modules/sampling/dynashift/__init__.py @@ -184,6 +184,7 @@ def install(patcher, values, key): captured = taste.collect(patcher, key, KIND, keep=4 * BANK, fresh=fresh) steer = input_steer.Steer("DynaShift") + steer.attach(patcher, key) def apply_model(executor, x, t, *args, **kwargs): named = streams.model_args(args, kwargs) diff --git a/modules/sampling/output_guidance/__init__.py b/modules/sampling/output_guidance/__init__.py index 474dd895..154142b5 100644 --- a/modules/sampling/output_guidance/__init__.py +++ b/modules/sampling/output_guidance/__init__.py @@ -70,6 +70,7 @@ def install(patcher, values, key): captured = taste.collect(patcher, key, KIND, fresh=fresh) steer = input_steer.Steer("Taste guidance") + steer.attach(patcher, key) def apply_model(executor, x, t, *args, **kwargs): named = streams.model_args(args, kwargs) diff --git a/modules/sampling/score_slider/__init__.py b/modules/sampling/score_slider/__init__.py index e5b06b2e..775ed1a3 100644 --- a/modules/sampling/score_slider/__init__.py +++ b/modules/sampling/score_slider/__init__.py @@ -98,6 +98,7 @@ def install(patcher, values, key): captured = taste.collect(patcher, key, KIND, fresh=fresh) steer = input_steer.Steer("Taste slider") + steer.attach(patcher, key) def apply_model(executor, x, t, *args, **kwargs): named = streams.model_args(args, kwargs) @@ -128,8 +129,11 @@ def install(patcher, values, key): if split is None: _say("off this run: could not find the picture in this model's latent") return base - plus = streams.video_of(executor(x, t, **{**named, "c_crossattn": c + nudge}), named) - minus = streams.video_of(executor(x, t, **{**named, "c_crossattn": c - nudge}), named) + # Perturbed-text passes are not the clip's own forward: measuring probes skip them. + aside = {**named, "transformer_options": {**(named.get("transformer_options") or {}), + dit_hooks.PROBE: True}} + plus = streams.video_of(executor(x, t, **{**aside, "c_crossattn": c + nudge}), named) + minus = streams.video_of(executor(x, t, **{**aside, "c_crossattn": c - nudge}), named) video, rebuild = split return step.keep(base, rebuild(video + (plus[0] - minus[0]) * amount)) diff --git a/modules/sampling/score_slider/tests/test_score_slider.py b/modules/sampling/score_slider/tests/test_score_slider.py index 020ee58f..4696c625 100644 --- a/modules/sampling/score_slider/tests/test_score_slider.py +++ b/modules/sampling/score_slider/tests/test_score_slider.py @@ -68,10 +68,11 @@ def test_late_steps_combine_three_passes_on_the_picture_words_only(tiny_h3): o(lambda: None) audio = torch.randn(1, 3, 5) - seen = [] + seen, flags = [], [] def executor(x, t, c_concat=None, c_crossattn=None, control=None, transformer_options=None, **kw): seen.append(c_crossattn.clone()) + flags.append(bool((transformer_options or {}).get("funpack_probe"))) return packed_av(torch.full((1, 2, 1, 2, 2), float(c_crossattn[0, :, 0].sum())), audio)[0] c = torch.ones(1, 4, dim) @@ -93,6 +94,7 @@ def test_late_steps_combine_three_passes_on_the_picture_words_only(tiny_h3): seen.clear() answer = call(2) # the one gated push (lands on step 4) assert len(seen) == 3 + assert flags[-3:] == [False, True, True] # the +/- passes are marked: not the clip's own plus = seen[1] assert torch.equal(plus[0, 2], c[0, 2]) and not torch.equal(plus[0, 0], c[0, 0]) assert torch.equal(answer, x0) # the model's own answer comes back diff --git a/modules/sampling/trajectory_guidance/__init__.py b/modules/sampling/trajectory_guidance/__init__.py index d558c0ca..52efa1e3 100644 --- a/modules/sampling/trajectory_guidance/__init__.py +++ b/modules/sampling/trajectory_guidance/__init__.py @@ -76,6 +76,7 @@ def install(patcher, values, key): captured = taste.collect(patcher, key, KIND, fresh=fresh) steer = input_steer.Steer("Early taste guidance") + steer.attach(patcher, key) def apply_model(executor, x, t, *args, **kwargs): named = streams.model_args(args, kwargs)