feat: phrase blend -- per-blend strength override, [a|b:0.8]
Default 0.5 shift left phraseA dominating the video. [phraseA|phraseB:N] overrides strength per blend. Fixed a shared regex gap along the way: every weight/strength markup in this file (not just blend) silently rejected leading-dot decimals like ".8" -- fell through to literal text with no error, reproducing the exact "override did nothing" bug this feature exists to fix. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
digital-garbage
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
1f54ad4519
commit
8a6ab12ce6
+10
-3
@@ -12264,7 +12264,14 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner):
|
||||
return out
|
||||
|
||||
def _v2_apply_h3_phrase_blend(self, conditioning_list, clip, link_texts=None):
|
||||
"""`[phraseA|phraseB]` — pull phraseA's rows toward phraseB's, in place.
|
||||
"""`[phraseA|phraseB(:strength)?]` — pull phraseA's rows toward phraseB's, in place.
|
||||
|
||||
`strength` defaults to `h3_token_weights.BLEND_STRENGTH_DEFAULT` (0.5) when omitted;
|
||||
each span carries its own value (parsed per-blend, not a single run-wide setting),
|
||||
since a half-strength pull still reads as mostly phraseA — the rest of the sentence
|
||||
(style, postfix, grammar around the phrase) is 100% built around phraseA's words
|
||||
regardless of how far the mean shift moves, so getting phraseB's influence to show
|
||||
up at all in the video can need pushing well past 0.5.
|
||||
|
||||
phraseA is what `_v2_encode_prompt` already put in the prompt (see
|
||||
`funpack_h3_blended` in `_v2_encode_prompt`); prefix, postfix and every other word
|
||||
@@ -12390,7 +12397,7 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner):
|
||||
continue
|
||||
new_cond = cond.clone()
|
||||
done = 0
|
||||
for start, end, alt_phrase in spans:
|
||||
for start, end, alt_phrase, strength in spans:
|
||||
toks = _tw.token_spans_from_offsets(offsets, [(start, end, 1.0)])
|
||||
if not toks:
|
||||
skipped_n += 1
|
||||
@@ -12430,7 +12437,7 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner):
|
||||
mean_b = alt_cond[:, blo:bhi, :].mean(dim=1, keepdim=True).to(
|
||||
dtype=new_cond.dtype, device=new_cond.device)
|
||||
new_cond[:, lo:hi, :] = new_cond[:, lo:hi, :] \
|
||||
+ _tw.BLEND_STRENGTH_DEFAULT * (mean_b - mean_a)
|
||||
+ float(strength) * (mean_b - mean_a)
|
||||
done += 1
|
||||
if done:
|
||||
cond = new_cond
|
||||
|
||||
+29
-11
@@ -24,8 +24,14 @@ import math
|
||||
import re
|
||||
from typing import Optional
|
||||
|
||||
# A signed number a weight/strength may be written as, including a leading-dot shorthand
|
||||
# (".8") that `-?\d+(?:\.\d+)?` alone would silently refuse -- refused text falls through to
|
||||
# being read as literal phrase content instead of a weight, with nothing to say the number
|
||||
# was never applied.
|
||||
_NUM = r"-?(?:\d+\.\d*|\.\d+|\d+)"
|
||||
|
||||
# (phrase:1.3) / (phrase:-0.4). Escaped \( is left alone so a literal bracket still works.
|
||||
_WEIGHTED = re.compile(r"(?<!\\)\(([^():]*?):\s*(-?\d+(?:\.\d+)?)\s*\)")
|
||||
_WEIGHTED = re.compile(r"(?<!\\)\(([^():]*?):\s*(" + _NUM + r")\s*\)")
|
||||
|
||||
# Below this a weight is treated as "remove entirely" rather than log(0) = -inf, which
|
||||
# produces NaN the moment a query attends to nothing else.
|
||||
@@ -70,7 +76,7 @@ def parse(text: str):
|
||||
# Square brackets so a shortcut replacement's parentheses and colons can sit inside; only a
|
||||
# `:number` directly before the `@t0-t1]` tail is a weight.
|
||||
_TIMED = re.compile(
|
||||
r"(?<!\\)\[([^\[\]]*?)(?::\s*(-?\d+(?:\.\d+)?))?\s*@\s*(\d+(?:\.\d+)?)\s*-\s*(\d+(?:\.\d+)?)\s*\]")
|
||||
r"(?<!\\)\[([^\[\]]*?)(?::\s*(" + _NUM + r"))?\s*@\s*(\d+(?:\.\d+)?)\s*-\s*(\d+(?:\.\d+)?)\s*\]")
|
||||
# H3's latent time grid: latent frame k spans FRAME_PER_TOKEN[k % 5] pixel frames at 24 fps.
|
||||
FPS = 24
|
||||
FRAME_PER_TOKEN = (1, 4, 4, 4, 4)
|
||||
@@ -78,7 +84,7 @@ FRAME_PER_TOKEN = (1, 4, 4, 4, 4)
|
||||
|
||||
# [phrase:1.5] - a weight with no window, in brackets so a shortcut replacement's own
|
||||
# parentheses cannot break it (the reason timed phrases use brackets at all).
|
||||
_BRACKET_WEIGHTED = re.compile(r"(?<!\\)\[([^\[\]@]*?):\s*(-?\d+(?:\.\d+)?)\s*\]")
|
||||
_BRACKET_WEIGHTED = re.compile(r"(?<!\\)\[([^\[\]@]*?):\s*(" + _NUM + r")\s*\]")
|
||||
|
||||
# [phraseA|phraseB]: BLEND. phraseA stays in the prompt Studio encodes (prefix/postfix and
|
||||
# every other phrase are exactly what was typed); phraseB is a second sentence re-encoded
|
||||
@@ -104,9 +110,20 @@ _BRACKET_WEIGHTED = re.compile(r"(?<!\\)\[([^\[\]@]*?):\s*(-?\d+(?:\.\d+)?)\s*\]
|
||||
# always found a way through anyway (`dog:5@2`, `dog@1—2` with an em dash, `@` followed by
|
||||
# an unrelated dash many words later). Blacklisting a look is not a fixable boundary; not
|
||||
# blacklisting at all is consistent with how the rest of this file already treats it.
|
||||
_BLENDED = re.compile(r"(?<!\\)\[([^\[\]|]+)\|([^\[\]|]+)\]")
|
||||
# Default blend amount: half the distance from phraseA's average hidden state toward
|
||||
# phraseB's. Not exposed as a Studio dial yet — one number, changeable here, not a UI knob.
|
||||
#
|
||||
# An OPTIONAL `:strength` on phraseB overrides `BLEND_STRENGTH_DEFAULT` for that one blend
|
||||
# — `[cat|dog:0.8]`. This deliberately claims the same trailing "colon then a bare number"
|
||||
# shape that would otherwise be ambiguous with literal phraseB text, the same way
|
||||
# `_BRACKET_WEIGHTED` already claims that shape for its own phrase. Non-greedy so an
|
||||
# ordinary internal colon ("shot: walks") is still just text — only a colon-number
|
||||
# immediately before the closing `]` is read as a strength override.
|
||||
_BLENDED = re.compile(r"(?<!\\)\[([^\[\]|]+)\|([^\[\]|]+?)(?::\s*(" + _NUM + r"))?\]")
|
||||
# Default blend amount when no `:strength` is given: half the distance from phraseA's
|
||||
# average hidden state toward phraseB's. Applies as a bare fraction, same scale as a typed
|
||||
# strength (0 = no shift, 1 = fully phraseB's mean, room to overshoot past 1 if asked). Not
|
||||
# clamped: 0 is a legitimate no-op, and a negative value pushes phraseA's rows AWAY from
|
||||
# phraseB's mean instead of toward it. Deliberately left open, not validated -- same policy
|
||||
# as every other weight/strength value in this file.
|
||||
BLEND_STRENGTH_DEFAULT = 0.5
|
||||
|
||||
_MARKUP = re.compile(_TIMED.pattern + "|" + _BLENDED.pattern + "|" + _WEIGHTED.pattern
|
||||
@@ -117,7 +134,7 @@ def parse_markup(text: str):
|
||||
"""Strip ALL THREE prompt markups in one left-to-right pass.
|
||||
|
||||
-> (clean_text, weighted [(start, end, weight)], timed [(start, end, weight, t0, t1)],
|
||||
blended [(start, end, alt_phrase)]).
|
||||
blended [(start, end, alt_phrase, strength)]).
|
||||
One pass, so every span list indexes the same clean text; stripping them one after the
|
||||
other would leave the first list's offsets pointing into a string that no longer exists.
|
||||
"""
|
||||
@@ -133,13 +150,14 @@ def parse_markup(text: str):
|
||||
if t1 > t0 and phrase.strip():
|
||||
timed.append((start, start + len(phrase),
|
||||
float(m.group(2)) if m.group(2) else 1.0, t0, t1))
|
||||
elif m.group(5) is not None: # [phraseA|phraseB]
|
||||
elif m.group(5) is not None: # [phraseA|phraseB(:strength)?]
|
||||
phrase, alt = m.group(5), m.group(6)
|
||||
strength = float(m.group(7)) if m.group(7) is not None else BLEND_STRENGTH_DEFAULT
|
||||
if phrase.strip() and alt.strip():
|
||||
blended.append((start, start + len(phrase), alt.strip()))
|
||||
blended.append((start, start + len(phrase), alt.strip(), strength))
|
||||
else: # (phrase:w) or [phrase:w]
|
||||
phrase = m.group(7) if m.group(7) is not None else m.group(9)
|
||||
weight = m.group(8) if m.group(8) is not None else m.group(10)
|
||||
phrase = m.group(8) if m.group(8) is not None else m.group(10)
|
||||
weight = m.group(9) if m.group(9) is not None else m.group(11)
|
||||
weighted.append((start, start + len(phrase), float(weight)))
|
||||
out.append(phrase)
|
||||
pos = m.end()
|
||||
|
||||
@@ -112,7 +112,7 @@ def expand_prompt_with_windows(text: str, variables=None):
|
||||
which never sees the encoded string when a wired node owns the prompt - needs them
|
||||
handed over separately, as char spans on the clean text.
|
||||
-> (clean_expanded_text, timed [(start, end, weight, t0, t1)],
|
||||
weighted [(start, end, weight)], blended [(start, end, alt_phrase)])
|
||||
weighted [(start, end, weight)], blended [(start, end, alt_phrase, strength)])
|
||||
"""
|
||||
expanded = expand_prompt_fragment(text)
|
||||
if expanded and variables:
|
||||
|
||||
@@ -827,18 +827,38 @@ def test_a_blended_phrase_is_parsed_and_phrase_a_stays_in_the_clean_text():
|
||||
assert clean == "a cat sits on a mat"
|
||||
assert weighted == []
|
||||
assert timed == []
|
||||
assert blended == [(6, 10, "stands")]
|
||||
assert blended == [(6, 10, "stands", tw.BLEND_STRENGTH_DEFAULT)]
|
||||
|
||||
|
||||
def test_a_blend_alongside_a_weight_and_a_window_in_one_pass():
|
||||
clean, weighted, timed, blended = tw.parse_markup(
|
||||
"a cat [sits|stands] then [runs@1-2] (fast:1.5)")
|
||||
assert clean == "a cat sits then runs fast"
|
||||
assert blended == [(6, 10, "stands")]
|
||||
assert blended == [(6, 10, "stands", tw.BLEND_STRENGTH_DEFAULT)]
|
||||
assert timed == [(16, 20, 1.0, 1.0, 2.0)]
|
||||
assert weighted == [(21, 25, 1.5)]
|
||||
|
||||
|
||||
def test_a_blend_strength_suffix_overrides_the_default():
|
||||
clean, weighted, timed, blended = tw.parse_markup("a cat [sits|stands:0.8] man")
|
||||
assert clean == "a cat sits man"
|
||||
assert blended == [(6, 10, "stands", 0.8)]
|
||||
|
||||
|
||||
def test_a_leading_dot_strength_is_a_number_not_literal_text():
|
||||
"""`:.8`, `:5.` and a negative leading-dot -- the natural shorthand for typing a decimal
|
||||
-- must parse as numbers here, same as everywhere else a weight/strength is written in
|
||||
this file. `-?\\d+(?:\\.\\d+)?` alone silently rejects a bare leading dot, which falls
|
||||
through to the alt phrase being read as literal text (":.8" stays in the sentence) with
|
||||
no error and no log line -- exactly reproducing the "my strength override did nothing"
|
||||
complaint this syntax exists to fix. Shared `_NUM` fixes this for every weight-style
|
||||
markup in the file, not just blend."""
|
||||
assert tw.parse_markup("a [cat|dog:.8] man")[3] == [(2, 5, "dog", 0.8)]
|
||||
assert tw.parse_markup("a [cat|dog:5.] man")[3] == [(2, 5, "dog", 5.0)]
|
||||
assert tw.parse_markup("a [cat|dog:-.5] man")[3] == [(2, 5, "dog", -0.5)]
|
||||
assert tw.parse_markup("a (cat:.8) man")[1] == [(2, 5, 0.8)]
|
||||
|
||||
|
||||
def test_an_empty_alternate_phrase_is_not_a_blend():
|
||||
assert tw.parse_markup("a cat [sits| ]")[3] == []
|
||||
|
||||
@@ -990,7 +1010,7 @@ def test_the_blended_span_is_shifted_by_half_the_mean_difference(refiner, monkey
|
||||
text = "a cat sits on a mat" # "sits" is chars [6:10]
|
||||
cond = torch.zeros(1, len(text), 4)
|
||||
cond[:, 6:10, :] = 1.0
|
||||
meta = {"funpack_h3_blended": [(6, 10, "stands")], "funpack_h3_timed_text": text,
|
||||
meta = {"funpack_h3_blended": [(6, 10, "stands", 0.5)], "funpack_h3_timed_text": text,
|
||||
"minimax_token_tags": [1] * len(text)}
|
||||
out = refiner._v2_apply_h3_phrase_blend([[cond, meta]], _H3Clip())
|
||||
result = out[0][0]
|
||||
@@ -999,6 +1019,20 @@ def test_the_blended_span_is_shifted_by_half_the_mean_difference(refiner, monkey
|
||||
assert torch.allclose(result[:, 10:, :], torch.zeros(1, len(text) - 10, 4))
|
||||
|
||||
|
||||
def test_a_higher_blend_strength_shifts_further_toward_the_alternate(refiner, monkeypatch):
|
||||
"""Same setup as above, but strength 0.8 instead of the 0.5 default: span should land at
|
||||
1.0 + 0.8*(5.0-1.0) = 4.2 — this is what the user asked for after reporting phraseA
|
||||
dominating visually at the default strength."""
|
||||
_quiet(monkeypatch)
|
||||
text = "a cat sits on a mat"
|
||||
cond = torch.zeros(1, len(text), 4)
|
||||
cond[:, 6:10, :] = 1.0
|
||||
meta = {"funpack_h3_blended": [(6, 10, "stands", 0.8)], "funpack_h3_timed_text": text,
|
||||
"minimax_token_tags": [1] * len(text)}
|
||||
out = refiner._v2_apply_h3_phrase_blend([[cond, meta]], _H3Clip())
|
||||
assert torch.allclose(out[0][0][:, 6:10, :], torch.full((1, 4, 4), 4.2))
|
||||
|
||||
|
||||
def test_an_untagged_entry_is_left_completely_alone(refiner, monkeypatch):
|
||||
_quiet(monkeypatch)
|
||||
cond = torch.zeros(1, 8, 4)
|
||||
@@ -1014,7 +1048,7 @@ def test_wired_conditioning_blends_from_the_editors_link_texts(refiner, monkeypa
|
||||
phrases. `choose_encoded_text` verifies "cat runs" against the tensor's own tag run
|
||||
(3 reference tokens + 8 text tokens) before trusting it."""
|
||||
_quiet(monkeypatch)
|
||||
link = {"prompt": "cat runs", "full_prompt": "cat runs", "blended": [[4, 8, "stands"]]}
|
||||
link = {"prompt": "cat runs", "full_prompt": "cat runs", "blended": [[4, 8, "stands", 0.5]]}
|
||||
cond = torch.zeros(1, 11, 4)
|
||||
cond[:, 7:11, :] = 1.0 # "runs" sits after the 3-token ref block
|
||||
meta = {"funpack_conditioning_owner": "wired", "minimax_token_tags": [0, 0, 0] + [1] * 8}
|
||||
@@ -1036,7 +1070,7 @@ def test_wired_conditioning_is_skipped_not_silently_dropped_when_nothing_matches
|
||||
feature=lambda *a, **k: logged.append(a)),
|
||||
raising=False)
|
||||
link = {"prompt": "totally different text", "full_prompt": "totally different text",
|
||||
"blended": [[4, 8, "stands"]]}
|
||||
"blended": [[4, 8, "stands", 0.5]]}
|
||||
cond = torch.zeros(1, 11, 4)
|
||||
cond[:, 7:11, :] = 1.0
|
||||
meta = {"funpack_conditioning_owner": "wired", "minimax_token_tags": [0, 0, 0] + [1] * 8}
|
||||
@@ -1057,25 +1091,25 @@ def test_a_non_h3_clip_is_left_alone(refiner, monkeypatch):
|
||||
import minimax_h3
|
||||
monkeypatch.setattr(minimax_h3, "is_h3_clip", lambda c: False)
|
||||
cond = torch.zeros(1, 8, 4)
|
||||
meta = {"funpack_h3_blended": [(0, 3, "x")], "funpack_h3_timed_text": "cat runs"}
|
||||
meta = {"funpack_h3_blended": [(0, 3, "x", 0.5)], "funpack_h3_timed_text": "cat runs"}
|
||||
out = refiner._v2_apply_h3_phrase_blend([[cond, meta]], _H3Clip())
|
||||
assert out[0][0] is cond
|
||||
|
||||
|
||||
# --- adversarial-review fixes: a colon in the alt phrase, double-apply, partial failure ----
|
||||
|
||||
def test_an_alt_phrase_shaped_like_a_weight_is_accepted_as_literal_text():
|
||||
"""[cat|dog:1.5] is a blend whose alt phrase is the literal string "dog:1.5" -- colon
|
||||
and digit intact. It is NOT stripped before its separate re-encode (there is no bracket
|
||||
left around it once substituted into the sentence), so Qwen reads it as literal
|
||||
punctuation in that phrase's own sentence -- a quality footnote, not a leak, and the
|
||||
same trade-off `[cat (sitting):1.5]` already gets elsewhere in this file (kept as
|
||||
literal pass-through text, not re-parsed). Blacklisting "looks like a weight" was tried
|
||||
and abandoned: it also caught ordinary colons in prose with nothing to do with weights,
|
||||
and a DIFFERENT shape (`dog:5@2`, an em-dash instead of a hyphen) always found a way
|
||||
through anyway -- see git history on this test for the two reverted attempts."""
|
||||
def test_a_trailing_colon_number_on_the_alt_phrase_is_now_a_strength_override():
|
||||
"""[cat|dog:1.5] used to keep "dog:1.5" as literal alt-phrase text (colon and digit both
|
||||
just part of the phrase, a deliberate accepted quality trade-off -- see
|
||||
feedback_dont_blacklist_by_shape). Per-blend strength syntax repurposes exactly this
|
||||
shape on purpose: a colon immediately followed by a number and the closing bracket is
|
||||
now read as `strength`, not literal text, same as `(phrase:weight)` elsewhere in this
|
||||
file. Only a colon in that exact tail position triggers it -- `dog:5@2` below is NOT a
|
||||
valid number-then-bracket tail, so it stays literal, and an ordinary colon anywhere else
|
||||
in either phrase (`test_an_ordinary_colon_in_either_phrase_is_a_perfectly_fine_blend`)
|
||||
is untouched."""
|
||||
clean, weighted, timed, blended = tw.parse_markup("a [cat|dog:1.5] man")
|
||||
assert blended == [(2, 5, "dog:1.5")]
|
||||
assert blended == [(2, 5, "dog", 1.5)]
|
||||
assert clean == "a cat man"
|
||||
|
||||
|
||||
@@ -1098,13 +1132,13 @@ def test_an_ordinary_colon_in_either_phrase_is_a_perfectly_fine_blend():
|
||||
the construct just never matched. A colon that isn't sitting at the very end next to a
|
||||
bare number is ordinary prompt text and must be allowed through untouched."""
|
||||
clean, weighted, timed, blended = tw.parse_markup("a [wide shot: walks|runs fast] man")
|
||||
assert blended == [(2, 18, "runs fast")]
|
||||
assert blended == [(2, 18, "runs fast", tw.BLEND_STRENGTH_DEFAULT)]
|
||||
assert clean == "a wide shot: walks man"
|
||||
|
||||
|
||||
def test_an_ordinary_at_sign_in_either_phrase_is_a_perfectly_fine_blend():
|
||||
clean, weighted, timed, blended = tw.parse_markup("a [call me@work|call me@home] man")
|
||||
assert blended == [(2, 14, "call me@home")]
|
||||
assert blended == [(2, 14, "call me@home", tw.BLEND_STRENGTH_DEFAULT)]
|
||||
assert clean == "a call me@work man"
|
||||
|
||||
|
||||
@@ -1128,7 +1162,7 @@ def test_a_malformed_time_window_shaped_alt_phrase_is_still_a_blend(alt):
|
||||
test_an_alt_phrase_shaped_like_a_weight_is_accepted_as_literal_text for why blocking
|
||||
by shape was tried and abandoned rather than fixed further."""
|
||||
clean, weighted, timed, blended = tw.parse_markup(f"a [cat|{alt}] man")
|
||||
assert blended == [(2, 5, alt)]
|
||||
assert blended == [(2, 5, alt, tw.BLEND_STRENGTH_DEFAULT)]
|
||||
assert clean == "a cat man"
|
||||
|
||||
|
||||
@@ -1141,7 +1175,7 @@ def test_applying_the_blend_twice_does_not_compound(refiner, monkeypatch):
|
||||
text = "a cat sits on a mat"
|
||||
cond = torch.zeros(1, len(text), 4)
|
||||
cond[:, 6:10, :] = 1.0
|
||||
meta = {"funpack_h3_blended": [(6, 10, "stands")], "funpack_h3_timed_text": text,
|
||||
meta = {"funpack_h3_blended": [(6, 10, "stands", 0.5)], "funpack_h3_timed_text": text,
|
||||
"minimax_token_tags": [1] * len(text)}
|
||||
once = refiner._v2_apply_h3_phrase_blend([[cond, meta]], _H3Clip())
|
||||
twice = refiner._v2_apply_h3_phrase_blend(once, _H3Clip())
|
||||
@@ -1157,7 +1191,7 @@ def test_applying_a_wired_blend_twice_with_the_same_link_texts_does_not_compound
|
||||
would re-tag and re-blend it without a separate, never-popped 'already handled'
|
||||
marker."""
|
||||
_quiet(monkeypatch)
|
||||
link = {"prompt": "cat runs", "full_prompt": "cat runs", "blended": [[4, 8, "stands"]]}
|
||||
link = {"prompt": "cat runs", "full_prompt": "cat runs", "blended": [[4, 8, "stands", 0.5]]}
|
||||
cond = torch.zeros(1, 11, 4)
|
||||
cond[:, 7:11, :] = 1.0
|
||||
meta = {"funpack_conditioning_owner": "wired", "minimax_token_tags": [0, 0, 0] + [1] * 8}
|
||||
@@ -1175,7 +1209,7 @@ def test_one_failed_span_is_reported_not_swallowed(refiner, monkeypatch, capsys)
|
||||
text = "a cat sits on a mat"
|
||||
cond = torch.zeros(1, len(text), 4)
|
||||
cond[:, 6:10, :] = 1.0
|
||||
meta = {"funpack_h3_blended": [(6, 10, "stands"), (100, 104, "nowhere")],
|
||||
meta = {"funpack_h3_blended": [(6, 10, "stands", 0.5), (100, 104, "nowhere", 0.5)],
|
||||
"funpack_h3_timed_text": text, "minimax_token_tags": [1] * len(text)}
|
||||
out = refiner._v2_apply_h3_phrase_blend([[cond, meta]], _H3Clip())
|
||||
assert torch.allclose(out[0][0][:, 6:10, :], torch.full((1, 4, 4), 3.0))
|
||||
|
||||
Reference in New Issue
Block a user