fix: input steer reports reach not size, resets per run; block influence validates before toggling, finds the real key
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5.5
parent
7dab350961
commit
313d703846
@@ -660,7 +660,7 @@
|
||||
rows.append(infoRow(
|
||||
recording ? "Rated generations recorded (this key)"
|
||||
: "Recorded earlier (not recording now)",
|
||||
biState ? `${runs} (${biState.n_liked} liked / ${biState.n_disliked} disliked)`
|
||||
biState ? `${runs} (${biState.n_liked} liked / ${biState.n_disliked} disliked) on key '${biState.key}'`
|
||||
: "0", recording && runs > 0 ? true : null));
|
||||
rows.append(actionRow("Save this key's measurement",
|
||||
"Downloads every recorded profile as one file. A rental gets replaced and "
|
||||
@@ -702,8 +702,8 @@
|
||||
paint();
|
||||
if (window.MovieEditorAPI) {
|
||||
window.MovieEditorAPI.blockInfluenceStatus(key)
|
||||
.then((s) => { biState = s; paint(); })
|
||||
.catch(() => {});
|
||||
.then((s) => { biState = s; biError = ""; paint(); })
|
||||
.catch((e) => { biState = null; biError = String(e.message || e); paint(); });
|
||||
}
|
||||
return box;
|
||||
}
|
||||
|
||||
+42
-15
@@ -24,6 +24,9 @@ that step: pushes are filed by the step that made them. A schedule that restarts
|
||||
|
||||
from . import dit_hooks, log
|
||||
|
||||
# Below this share of the input it joins, a push is reported as barely acting (v4: 3%).
|
||||
FLOOR = 0.03
|
||||
|
||||
|
||||
|
||||
class Call:
|
||||
@@ -41,13 +44,29 @@ class Call:
|
||||
class Steer:
|
||||
def __init__(self, what: str):
|
||||
self.what = what
|
||||
self.reset()
|
||||
|
||||
def reset(self):
|
||||
"""A run starts (or ends): nothing held, nothing counted."""
|
||||
self._pushes = {} # key -> {step: delta}
|
||||
self._seen = {} # key -> last step index fed
|
||||
self._edit, self._edits = 0.0, 0
|
||||
self._made, self._delivered = 0, 0
|
||||
self._reach = 0.0 # largest delivered push, as a share of the input it joined
|
||||
|
||||
def effect(self):
|
||||
"""Mean size of the edits made, relative to the answer, or None when none were."""
|
||||
return self._edit / self._edits if self._edits else None
|
||||
def attach(self, patcher, key):
|
||||
"""Reset around every sampling call. The node that installs the modifier is cached
|
||||
by ComfyUI, so this object outlives a run; without this a run that made no edit
|
||||
would report the previous run's."""
|
||||
from comfy.patcher_extension import WrappersMP
|
||||
|
||||
def outer(executor, *args, **kwargs):
|
||||
self.reset()
|
||||
try:
|
||||
return executor(*args, **kwargs)
|
||||
finally:
|
||||
self._pushes = {} # latent-sized tensors must not outlive the run
|
||||
|
||||
patcher.add_wrapper_with_key(WrappersMP.OUTER_SAMPLE, key, outer)
|
||||
|
||||
def _say(self, level, message, key):
|
||||
log.once(f"input steer {self.what}:{key}", level, f"FunPack {self.what}", message)
|
||||
@@ -81,10 +100,13 @@ class Steer:
|
||||
if i < self._seen.get(key, 0): # a new sampling pass: nothing carries over
|
||||
held.clear()
|
||||
self._seen[key] = i
|
||||
push = held.get(i - 1)
|
||||
push = held.pop(i - 1, None) # delivered once; a repeated sigma cannot re-deliver it
|
||||
if push is not None:
|
||||
if tuple(push.shape) == tuple(x.shape):
|
||||
x = x + (1.0 - float(t.max())) * push.to(x.device, x.dtype)
|
||||
add = (1.0 - float(t.max())) * push.to(x.device, x.dtype)
|
||||
self._delivered += 1
|
||||
self._reach = max(self._reach, float(add.norm() / x.norm().clamp(min=1e-8)))
|
||||
x = x + add
|
||||
else:
|
||||
self._say(log.WARNING, "Inactive | the latent changed size between steps, "
|
||||
"so that step went unsteered", "resized")
|
||||
@@ -95,20 +117,25 @@ class Steer:
|
||||
return Call(x, 0.0, final=True, keep=lambda out, _steered: out)
|
||||
|
||||
def keep(out, steered):
|
||||
try:
|
||||
self._edit += float((steered - out).norm() / out.norm().clamp(min=1e-8))
|
||||
self._edits += 1
|
||||
except Exception: # noqa: BLE001 -- a measurement must never break sampling
|
||||
pass
|
||||
self._made += 1
|
||||
held[i] = (steered - out).detach()
|
||||
return out
|
||||
|
||||
return Call(x, dit_hooks.late_half(to, ahead=1), keep=keep)
|
||||
|
||||
def _report(self):
|
||||
size = self.effect()
|
||||
if size is None:
|
||||
"""Said on the last step. Reach is what counts: an edit made is not an edit that
|
||||
arrived, and one that is a sliver of the input it joins changes nothing the
|
||||
model can feel."""
|
||||
if not self._made:
|
||||
self._say(log.ALERT, "Inactive | no step made an edit this run", "result")
|
||||
elif not self._delivered:
|
||||
self._say(log.ALERT, f"Inactive | {self._made} edit(s) made but none reached a later "
|
||||
f"step's input", "result")
|
||||
elif self._reach < FLOOR:
|
||||
self._say(log.ALERT, f"Inactive | barely acts: the largest push was {self._reach:.1%} of "
|
||||
f"the input it joined (under {FLOOR:.0%}); more steps or a "
|
||||
f"higher strength would let it be felt", "result")
|
||||
else:
|
||||
self._say(log.INFO, f"Active | edits averaged {size:.1%} of the picture's size, "
|
||||
f"each carried into the next step's input", "result")
|
||||
self._say(log.INFO, f"Active | {self._delivered} push(es) carried into later steps, the "
|
||||
f"largest {self._reach:.1%} of the input it joined", "result")
|
||||
|
||||
@@ -105,3 +105,38 @@ def test_a_one_step_schedule_says_there_is_nothing_to_carry_into():
|
||||
s, x = input_steer.Steer("t"), _x()
|
||||
step = s.begin(x, _t(0, 1), _to(0, 1))
|
||||
assert step.final and any("1-step" in r["message"] for r in log.history())
|
||||
|
||||
|
||||
def _said(text):
|
||||
return any(text in r["message"] for r in log.history())
|
||||
|
||||
|
||||
def test_a_sliver_of_a_push_is_reported_as_barely_acting_and_a_real_one_as_active():
|
||||
x, out = _x(), _x()
|
||||
for size, word in ((1e-4, "barely acts"), (1.0, "Active |")):
|
||||
log._reset()
|
||||
s = input_steer.Steer("t")
|
||||
_edit(s, 2, x, out, torch.ones(1, 1, 12) * size)
|
||||
s.begin(x, _t(3), _to(3))
|
||||
assert _said(word), (size, [r["message"] for r in log.history()])
|
||||
|
||||
|
||||
def test_a_second_run_on_the_same_object_does_not_report_the_first_runs_edits():
|
||||
log._reset()
|
||||
s, x, out = input_steer.Steer("t"), _x(), _x()
|
||||
_edit(s, 2, x, out, torch.ones(1, 1, 12))
|
||||
s.begin(x, _t(3), _to(3))
|
||||
s.reset() # what the sampling wrapper does per run
|
||||
log._reset()
|
||||
s.begin(x, _t(3), _to(3))
|
||||
assert _said("Inactive | no step made an edit") and not _said("Active |")
|
||||
|
||||
|
||||
def test_a_repeated_sigma_cannot_count_an_edit_that_never_arrived():
|
||||
s, x, out = input_steer.Steer("t"), _x(), _x()
|
||||
sched = torch.tensor([1.0, 0.8, 0.8, 0.5, 0.0])
|
||||
to = lambda i: {"sample_sigmas": sched, "sigmas": sched[i:i + 1]}
|
||||
s.begin(x, sched[2:3], to(2)).keep(out, out + 1) # index reads 1, the first match: gate 0, no edit
|
||||
log._reset()
|
||||
s.begin(x, sched[3:4], to(3))
|
||||
assert not _said("Active |")
|
||||
|
||||
@@ -107,15 +107,20 @@ def routes(table, base, web):
|
||||
@table.post(base + "/enabled")
|
||||
async def _enabled(req):
|
||||
body = await req.json()
|
||||
try:
|
||||
state = measure.state(key_of(req, body)) # refuse BEFORE flipping the switch
|
||||
except ValueError as exc:
|
||||
return web.json_response({"why": str(exc)}, status=400)
|
||||
measure.set_enabled(bool(body.get("enabled")))
|
||||
return web.json_response(measure.state(key_of(req, body)))
|
||||
return web.json_response({**state, "enabled": measure.enabled()})
|
||||
|
||||
@table.post(base + "/clear")
|
||||
async def _clear(req):
|
||||
body = await req.json()
|
||||
try:
|
||||
measure.clear(key_of(req, body))
|
||||
return web.json_response(measure.state(key_of(req, body)))
|
||||
key = measure.resolve(key_of(req, body))
|
||||
measure.clear(key)
|
||||
return web.json_response(measure.state(key))
|
||||
except ValueError as exc:
|
||||
return web.json_response({"why": str(exc)}, status=400)
|
||||
|
||||
|
||||
@@ -145,16 +145,27 @@ def _kind(key):
|
||||
return kind
|
||||
|
||||
|
||||
def resolve(key):
|
||||
"""The key to show. The panel asks for a placeholder ("default") when it does not know
|
||||
which Taste key the runs write to, so a key with nothing recorded falls back to the key
|
||||
the latest capture went to -- and the answer says which one it is."""
|
||||
if _kind(key).path().exists():
|
||||
return key
|
||||
latest = registry.current().ask("taste_latest_key")
|
||||
return latest if latest and _kind(latest).path().exists() else key
|
||||
|
||||
|
||||
def state(key) -> dict:
|
||||
"""What Settings > Refinement & Taste shows for one key."""
|
||||
key = resolve(key)
|
||||
rows = _kind(key).rows()
|
||||
return {"key": key, "enabled": enabled(), "runs": len(rows), "min_per_group": MIN_PER_GROUP,
|
||||
**profile(rows)}
|
||||
|
||||
|
||||
def clear(key):
|
||||
_kind(key).clear()
|
||||
_kind(resolve(key)).clear()
|
||||
|
||||
|
||||
def path_of(key):
|
||||
return _kind(key).path()
|
||||
return _kind(resolve(key)).path()
|
||||
|
||||
@@ -173,3 +173,48 @@ def test_the_panel_routes_read_toggle_clear_and_export(tiny_h3):
|
||||
assert json.loads(out.text)["runs"] == 0
|
||||
assert call("GET", "/b/export")[0] == 404
|
||||
assert call("GET", "/b/status", key="no/slash")[0] == 400
|
||||
|
||||
|
||||
def test_a_bad_key_is_refused_before_the_switch_flips_and_a_placeholder_finds_the_real_key(tiny_h3):
|
||||
from aiohttp import web
|
||||
from modules.sampling import block_influence as mod
|
||||
from modules.sampling.block_influence import measure
|
||||
from modules.system.taste import store
|
||||
import json
|
||||
handlers = {}
|
||||
|
||||
class Table:
|
||||
def get(self, path):
|
||||
return lambda fn: handlers.setdefault(("GET", path), fn)
|
||||
|
||||
def post(self, path):
|
||||
return lambda fn: handlers.setdefault(("POST", path), fn)
|
||||
|
||||
mod.routes(Table(), "/b", web)
|
||||
|
||||
class Req:
|
||||
def __init__(self, body=None, key="default"):
|
||||
self._body, self.rel_url = body, type("U", (), {"query": {"key": key}})()
|
||||
|
||||
async def json(self):
|
||||
return self._body
|
||||
|
||||
out = asyncio.run(handlers[("POST", "/b/enabled")](Req({"enabled": True, "key": "no/slash"})))
|
||||
assert out.status == 400 and not measure.enabled()
|
||||
patched, _ = _load(tiny_h3, key="mystyle")
|
||||
measure.set_enabled(True)
|
||||
tiny_h3.sample(patched)
|
||||
store.rate("run-1", "liked")
|
||||
state = json.loads(asyncio.run(handlers[("GET", "/b/status")](Req())).text)
|
||||
assert state["key"] == "mystyle" and state["runs"] == 1 # "default" had nothing: the real key
|
||||
|
||||
|
||||
def test_the_sliders_perturbed_passes_are_not_measured(tiny_h3):
|
||||
from modules.sampling.block_influence import measure
|
||||
from core import dit_hooks
|
||||
patched, _ = _load(tiny_h3)
|
||||
measure.set_enabled(True)
|
||||
patched.model_options.setdefault("transformer_options", {})[dit_hooks.PROBE] = True
|
||||
tiny_h3.sample(patched) # same flag the slider sets on its +/- passes
|
||||
from modules.system.taste import store
|
||||
assert not (store.ROOT / "fox" / "block_influence.pending.pt").exists()
|
||||
|
||||
@@ -184,6 +184,7 @@ def install(patcher, values, key):
|
||||
|
||||
captured = taste.collect(patcher, key, KIND, keep=4 * BANK, fresh=fresh)
|
||||
steer = input_steer.Steer("DynaShift")
|
||||
steer.attach(patcher, key)
|
||||
|
||||
def apply_model(executor, x, t, *args, **kwargs):
|
||||
named = streams.model_args(args, kwargs)
|
||||
|
||||
@@ -70,6 +70,7 @@ def install(patcher, values, key):
|
||||
|
||||
captured = taste.collect(patcher, key, KIND, fresh=fresh)
|
||||
steer = input_steer.Steer("Taste guidance")
|
||||
steer.attach(patcher, key)
|
||||
|
||||
def apply_model(executor, x, t, *args, **kwargs):
|
||||
named = streams.model_args(args, kwargs)
|
||||
|
||||
@@ -98,6 +98,7 @@ def install(patcher, values, key):
|
||||
|
||||
captured = taste.collect(patcher, key, KIND, fresh=fresh)
|
||||
steer = input_steer.Steer("Taste slider")
|
||||
steer.attach(patcher, key)
|
||||
|
||||
def apply_model(executor, x, t, *args, **kwargs):
|
||||
named = streams.model_args(args, kwargs)
|
||||
@@ -128,8 +129,11 @@ def install(patcher, values, key):
|
||||
if split is None:
|
||||
_say("off this run: could not find the picture in this model's latent")
|
||||
return base
|
||||
plus = streams.video_of(executor(x, t, **{**named, "c_crossattn": c + nudge}), named)
|
||||
minus = streams.video_of(executor(x, t, **{**named, "c_crossattn": c - nudge}), named)
|
||||
# Perturbed-text passes are not the clip's own forward: measuring probes skip them.
|
||||
aside = {**named, "transformer_options": {**(named.get("transformer_options") or {}),
|
||||
dit_hooks.PROBE: True}}
|
||||
plus = streams.video_of(executor(x, t, **{**aside, "c_crossattn": c + nudge}), named)
|
||||
minus = streams.video_of(executor(x, t, **{**aside, "c_crossattn": c - nudge}), named)
|
||||
video, rebuild = split
|
||||
return step.keep(base, rebuild(video + (plus[0] - minus[0]) * amount))
|
||||
|
||||
|
||||
@@ -68,10 +68,11 @@ def test_late_steps_combine_three_passes_on_the_picture_words_only(tiny_h3):
|
||||
o(lambda: None)
|
||||
|
||||
audio = torch.randn(1, 3, 5)
|
||||
seen = []
|
||||
seen, flags = [], []
|
||||
|
||||
def executor(x, t, c_concat=None, c_crossattn=None, control=None, transformer_options=None, **kw):
|
||||
seen.append(c_crossattn.clone())
|
||||
flags.append(bool((transformer_options or {}).get("funpack_probe")))
|
||||
return packed_av(torch.full((1, 2, 1, 2, 2), float(c_crossattn[0, :, 0].sum())), audio)[0]
|
||||
|
||||
c = torch.ones(1, 4, dim)
|
||||
@@ -93,6 +94,7 @@ def test_late_steps_combine_three_passes_on_the_picture_words_only(tiny_h3):
|
||||
seen.clear()
|
||||
answer = call(2) # the one gated push (lands on step 4)
|
||||
assert len(seen) == 3
|
||||
assert flags[-3:] == [False, True, True] # the +/- passes are marked: not the clip's own
|
||||
plus = seen[1]
|
||||
assert torch.equal(plus[0, 2], c[0, 2]) and not torch.equal(plus[0, 0], c[0, 0])
|
||||
assert torch.equal(answer, x0) # the model's own answer comes back
|
||||
|
||||
@@ -76,6 +76,7 @@ def install(patcher, values, key):
|
||||
|
||||
captured = taste.collect(patcher, key, KIND, fresh=fresh)
|
||||
steer = input_steer.Steer("Early taste guidance")
|
||||
steer.attach(patcher, key)
|
||||
|
||||
def apply_model(executor, x, t, *args, **kwargs):
|
||||
named = streams.model_args(args, kwargs)
|
||||
|
||||
Reference in New Issue
Block a user