From e1d9c8f2ee9f609d73a0519cc472e2c3ada2fd64 Mon Sep 17 00:00:00 2001 From: digital-garbage <133010426+digital-garbage@users.noreply.github.com> Date: Wed, 7 Oct 2026 18:46:54 +0300 Subject: [PATCH] feat: add a liked detail only where that body part is already named A like of a short detail such as "detailed lips" can append that sentence at the end of a later shot that already says lips. The original sentence is left alone. Shots that already describe the part, or never name it, get nothing. Two bad-composition votes more than the likes drop that detail. --- app/ui/features/settings/shotmemory.js | 4 +- modules/conditioning/shot_camera/__init__.py | 6 + modules/conditioning/shot_camera/engine.py | 129 ++++++++++++++++++ modules/conditioning/shot_camera/memory.py | 72 +++++++++- modules/conditioning/shot_camera/nodes.py | 18 ++- .../shot_camera/tests/test_node.py | 26 ++++ .../shot_camera/tests/test_shot_camera.py | 24 ++++ 7 files changed, 273 insertions(+), 6 deletions(-) diff --git a/app/ui/features/settings/shotmemory.js b/app/ui/features/settings/shotmemory.js index 7cc7aa44..2361bf47 100644 --- a/app/ui/features/settings/shotmemory.js +++ b/app/ui/features/settings/shotmemory.js @@ -20,8 +20,10 @@ export const shotMemory = (app) => function mount() { if (error) return page.set([c.banner.warn({ text: `Could not read the camera's memory: ${error}` })]); if (!data) return page.set([c.hint.default({ text: "Loading…" })]); const words = data.words || [], views = data.views || []; - if (!words.length && !views.length && !(data.arms || []).length) return page.set([c.emptyState.default({ icon: "◎", title: "Nothing learned yet", hint: "In a pipeline with the Shot camera node, turn on camera moves or shot views in Engine, then rate renders: the camera remembers which words and views you liked." })]); + const details = data.details || []; + if (!words.length && !views.length && !(data.arms || []).length && !details.length) return page.set([c.emptyState.default({ icon: "◎", title: "Nothing learned yet", hint: "In a pipeline with the Shot camera node, turn on camera moves, shot views or liked details in Engine, then rate renders: the camera remembers which words, views and details you liked." })]); page.set([c.hint.default({ text: `${data.prompts} prompts seen · ${data.shots} shots reviewed. Forget anything that taught it the wrong thing.` }), + ...(details.length ? [c.label.section({ text: "Liked details" }), ...details.map((d) => row(d.phrase, `${d.kept ? "will be added" : "dropped"} · liked ${d.good.toFixed(1)} · disliked ${d.bad.toFixed(1)}`, "detail", d.phrase))] : []), ...(views.length ? [c.label.section({ text: "Views" }), ...views.map((v) => row(v.view, `liked ${v.good.toFixed(1)} · disliked ${v.bad.toFixed(1)}`, "view", v.view))] : []), ...(words.length ? [c.label.section({ text: "Words the camera aims at" }), ...words.map((w) => row(w.word, `picked ${w.picks} · replaced ${w.rejects} · seen ${w.seen}`, "word", w.word)), data.more ? c.hint.default({ text: `and ${data.more} more` }) : null] : []), diff --git a/modules/conditioning/shot_camera/__init__.py b/modules/conditioning/shot_camera/__init__.py index b71e2f60..116deb3e 100644 --- a/modules/conditioning/shot_camera/__init__.py +++ b/modules/conditioning/shot_camera/__init__.py @@ -8,6 +8,7 @@ pick the move. Off by default: * cut within the same shot: "[Shot 1] DANCE KISS" and "[Shot 1] DANCE DANCE" are cut between the shortcuts, so each part gets a time, a view and a focus. Its chance slider is hidden until this is on * views: your own trusted list (side view, from above, POV...), never on shot 1, never the same twice in a row * camera moves: one to three per shot, aimed at the most specific thing in it ("'s hand", not "") +* liked details: a short detail you liked ("detailed lips") may be added, once, at the end of a later shot that already names that body part. Its chance slider is hidden until this is on Deterministic: the same prompt gives the same shots; "Variation" re-rolls. Ratings teach it: which views and moves are liked (a dislike that blames the picture alone teaches nothing here), and the words your prompts keep returning to are favoured. @@ -44,6 +45,11 @@ SETTINGS = { "hint": "Later shots open on a view (side, from above, POV…), never the same twice in a row."}, "shot_views_chance": {"type": "float", "default": 0.4, "min": 0.0, "max": 1.0, "step": 0.05, "label": "Chance a shot gets a view", "when": {"shot_views": True}}, + "detail_notes": {"type": "bool", "default": False, "label": "Liked details", + "hint": "When a shot already names a body part, a short detail you liked of that part (such as \"detailed lips\") may be added once, as its own sentence at the end of the shot."}, + "detail_notes_chance": {"type": "float", "default": 0.5, "min": 0.0, "max": 1.0, "step": 0.05, "label": "Chance of a liked detail", + "hint": "1 = always add a matching detail, 0 = never. Ratings nudge the values in between. A detail that keeps coming back bad is dropped.", + "when": {"detail_notes": True}}, "variation": {"type": "int", "default": 0, "min": 0, "max": 9999, "label": "Variation", "hint": "The same prompt always gets the same shots. Change this number to draw different ones."}, } diff --git a/modules/conditioning/shot_camera/engine.py b/modules/conditioning/shot_camera/engine.py index 6404fe52..376a229e 100644 --- a/modules/conditioning/shot_camera/engine.py +++ b/modules/conditioning/shot_camera/engine.py @@ -49,6 +49,12 @@ GENERIC = {"camera", "video", "scene", "shot", "frame", "background", "foregroun PARTS = {"lip", "mouth", "tongue", "tooth", "teeth", "eye", "eyes", "face", "cheek", "hand", "finger", "hair", "neck", "shoulder", "waist", "hip", "leg", "foot", "feet", "arm", "skin", "expression", "smile", "gaze", "back", "chin", "brow"} +# A liked detail may name one of these. Hair, face, skin and the like are appearance: the old +# refiner refused to auto-inject those, and a like of a clip cannot tell the person from the act. +DETAIL_PARTS = PARTS - {"hair", "face", "skin", "expression", "smile", "gaze", "back"} +# Adjectives that locate a part ("lower lip") are not a detail worth repeating everywhere. +_NOT_A_DETAIL = {"left", "right", "other", "same", "whole", "own", "both", "each", "another", + "first", "next", "last", "upper", "lower", "front", "back"} PLACES = {"room", "street", "kitchen", "bedroom", "bed", "table", "floor", "stage", "hall", "garden", "beach", "forest", "city", "car", "office", "bathroom", "field"} MOVES_DETAIL = ("The camera pushes in toward {x}.", "The camera racks focus to {x}.", @@ -203,6 +209,129 @@ def candidates(picture): return out +def _content_amods(root): + """Adjectives that describe `root`, ignoring ones that only say which side it is on.""" + return sorted((c for c in root.children + if c.dep_ == "amod" and c.is_alpha and c.lemma_.lower() not in _NOT_A_DETAIL), + key=lambda t: t.i) + + +def detail_phrases(picture): + """[(lemma, phrase)] for a short detail already written on a body part ("detailed lips"). + + A bare noun is not a detail. Neither is a place, a person, or hair and face: those are who + the clip is about, and a like must not paste them onto the next one. With no tagger, one + adjective in front of a body-part word is the same idea.""" + hidden = _hide(picture or "") + nlp = _spacy() + if nlp is None: + return _detail_phrases_plain(hidden) + out, seen = [], set() + for root in (c.root for c in nlp(hidden).noun_chunks): + lemma = root.lemma_.lower() + if lemma not in DETAIL_PARTS: + continue + amods = _content_amods(root) + if not amods: + continue + phrase = " ".join(c.text.lower() for c in amods) + " " + root.text.lower() + phrase = re.sub(r"\s+", " ", phrase).strip() + if not phrase or phrase in seen or len(phrase.split()) > 5: + continue + seen.add(phrase) + out.append((lemma, phrase)) + return out + + +def _detail_phrases_plain(hidden): + parts = "|".join(sorted(DETAIL_PARTS, key=len, reverse=True)) + skip = DETERMINERS | POSSESSIVE_PRONOUNS | _NOT_A_DETAIL | {"and", "with", "from", "into", "onto", "over"} + out, seen = [], set() + for m in re.finditer(rf"\b([A-Za-z]{{3,}})\s+({parts})s?\b", hidden): + adj = m.group(1).lower() + if adj in skip: + continue + word = m.group(2).lower() + lemma = word if word in DETAIL_PARTS else word[:-1] + phrase = f"{adj} {m.group(2).lower()}" + if phrase in seen: + continue + seen.add(phrase) + out.append((lemma, phrase)) + return out + + +def qualified_lemmas(picture): + """Body-part lemmas this shot already describes with an adjective. A second detail would + argue with that adjective, so nothing is added onto them.""" + hidden = _hide(picture or "") + nlp = _spacy() + if nlp is None: + return {lemma for lemma, _phrase in _detail_phrases_plain(hidden)} + out = set() + for tok in nlp(hidden): + if tok.lemma_.lower() in DETAIL_PARTS and any(c.dep_ == "amod" and c.is_alpha for c in tok.children): + out.add(tok.lemma_.lower()) + return out + + +def detail_phrases_in(text): + """Every detail phrase in a prompt, one per wording, read per shot so the sound track is skipped.""" + marks = list(SHOT.finditer(text or "")) + chunks = ([text] if not marks else + [text[m.end():(marks[i + 1].start() if i + 1 < len(marks) else len(text))] for i, m in enumerate(marks)]) + out, seen = [], set() + for chunk in chunks: + picture, _sound = _split_sound(chunk) + for lemma, phrase in detail_phrases(picture): + if phrase not in seen: + seen.add(phrase) + out.append((lemma, phrase)) + return out + + +def _as_sentence(phrase): + phrase = (phrase or "").strip().rstrip(".") + return (phrase[:1].upper() + phrase[1:] + ".") if phrase else "" + + +def add_detail_notes(text, seed=0, chance=0.5, bank=()): + """-> (prompt, {"added": [{"shot", "lemma", "phrase"}], "why"}). + + A banked detail is added once, as its own sentence at the end of a shot, and only when that + shot already names the body part, the part is not already described, and the words are not + already there. One detail per shot: the one ratings have kept most. The original sentence + is never rewritten, so it cannot be made ungrammatical. `chance` of 0 or 1 is kept as set; + anything between is the caller's, already tilted per phrase.""" + marks = list(SHOT.finditer(text or "")) + info = {"added": [], "why": ""} + if not marks: + info["why"] = "no shots" + return text, info + usable = [b for b in bank or () if isinstance(b, dict) and b.get("lemma") and b.get("phrase") and float(b.get("good") or 0) >= 1] + if not usable: + info["why"] = "nothing learned yet" + return text, info + out = [text[:marks[0].start()]] + for i, m in enumerate(marks): + end = marks[i + 1].start() if i + 1 < len(marks) else len(text) + picture, sound = _split_sound(text[m.end():end]) + present = {c[0] for c in candidates(picture)} + taken = qualified_lemmas(picture) + low = picture.lower() + options = [b for b in usable + if b["lemma"] in present and b["lemma"] not in taken and b["phrase"].lower() not in low] + options.sort(key=lambda b: (-(float(b["good"]) - float(b["bad"])), -float(b["good"]), b["phrase"])) + if options and random.Random(f"{seed}:detail:{i}:{options[0]['phrase']}").random() < float(options[0].get("chance", chance)): + sentence = _as_sentence(options[0]["phrase"]) + picture = picture.rstrip() + " " + sentence + " " + info["added"].append({"shot": int(m.group(1)), "lemma": options[0]["lemma"], "phrase": options[0]["phrase"]}) + out.append(m.group(0) + picture + sound) + if not info["added"]: + info["why"] = "nothing matched" + return "".join(out), info + + def _score(c, prior=None): """How much `c` deserves the camera: what it is in its own prompt (an owned part, an object of a verb), plus, when given, `prior`'s bonus for words the user's prompts keep returning to.""" diff --git a/modules/conditioning/shot_camera/memory.py b/modules/conditioning/shot_camera/memory.py index 75cc4716..85eb879e 100644 --- a/modules/conditioning/shot_camera/memory.py +++ b/modules/conditioning/shot_camera/memory.py @@ -40,7 +40,7 @@ def _path(): _LOCK = threading.RLock() # the worker thread (observe, record_run) and the rating route write the same file -_DICTS = ("picks", "rejects", "seen", "views", "arms", "runs", "rated") +_DICTS = ("picks", "rejects", "seen", "views", "arms", "runs", "rated", "details") def _locked(fn): @@ -200,6 +200,61 @@ def rate_arms(used, sign, k=1.0): return len(used) +def detail_chance(chance, good, bad): + """The chance of adding one learned detail. 0 and 1 stay as the user set them. In between, + a detail that was liked is more likely and one that was blamed is less, gently: one vote + cannot flip it.""" + chance = float(chance) + if chance <= 0.0 or chance >= 1.0: + return max(0.0, min(1.0, chance)) + rate = (float(good) + 1.0) / (float(good) + float(bad) + 2.0) + return max(0.0, min(1.0, chance * 2.0 * rate)) + + +def detail_bank(): + """Details a like has taught, still allowed to be added: good at least once, and not + suppressed by bad-composition votes outrunning the likes.""" + rows = [] + for phrase, row in (_read().get("details") or {}).items(): + if not isinstance(row, dict): + continue + good, bad = float(row.get("good") or 0), float(row.get("bad") or 0) + lemma = str(row.get("lemma") or "") + if good < 1 or not lemma or (bad >= 2 and bad > good): + continue + rows.append({"lemma": lemma, "phrase": str(phrase), "good": good, "bad": bad}) + return rows + + +def _detail_pairs(run, sign): + """What a like credits (details written in that prompt) or a dislike blames (details this + run actually added).""" + if sign > 0: + from . import engine + return engine.detail_phrases_in((run or {}).get("text") or "") + return [(d.get("lemma") or "", str(d.get("phrase") or "").strip().lower()) + for d in (run or {}).get("details") or [] + if isinstance(d, dict) and str(d.get("phrase") or "").strip()] + + +def _apply_detail_lesson(run, sign, undo=False): + """Credit or blame the details of one run. `undo` takes that same lesson back. -> phrases touched.""" + pairs = [(lemma, phrase) for lemma, phrase in _detail_pairs(run, sign) if phrase] + if not pairs or not sign: + return 0 + delta = -1.0 if undo else 1.0 + field = "good" if sign > 0 else "bad" + data = _read() + store = data.setdefault("details", {}) + for lemma, phrase in pairs: + row = store.setdefault(phrase, {"lemma": lemma or "", "good": 0.0, "bad": 0.0}) + if lemma: + row["lemma"] = lemma + row[field] = max(0.0, round(float(row.get(field) or 0) + delta, 4)) + _save(data) + return len(pairs) + + def summary(limit=40): """What is remembered, for the person to read and prune: words (picked / replaced / merely recurring) and views (good / bad, with the per-trait rows folded under their view).""" @@ -212,7 +267,16 @@ def summary(limit=40): views.sort(key=lambda r: -(r["good"] + r["bad"])) arms = [{"arm": k, "good": g, "bad": b} for k, (g, b) in arm_stats().items() if not k.startswith("word:")] arms.sort(key=lambda r: -(r["good"] + r["bad"])) + details = [] + for phrase, row in (data.get("details") or {}).items(): + if not isinstance(row, dict): + continue + good, bad = float(row.get("good") or 0), float(row.get("bad") or 0) + details.append({"phrase": phrase, "lemma": row.get("lemma") or "", "good": good, "bad": bad, + "kept": good >= 1 and not (bad >= 2 and bad > good)}) + details.sort(key=lambda r: (-(r["good"] + r["bad"]), r["phrase"])) return {"words": words[:limit], "more": max(0, len(words) - limit), "views": views, "arms": arms, + "details": details[:limit], "prompts": int(data.get("prompts", 0)), "shots": int(data.get("shots", 0)), "kept": int(data.get("kept", 0))} @@ -238,6 +302,8 @@ def forget(kind, name=None): views.pop(k) elif kind == "arm": existed = (data.get("arms") or {}).pop(name, None) is not None + elif kind == "detail": + existed = (data.get("details") or {}).pop(str(name or "").strip().lower(), None) is not None else: raise ValueError(f"unknown kind {kind!r}") if existed: # a forgotten lesson must not be rebuilt by re-rating the run that taught it @@ -333,7 +399,9 @@ def on_rating(prompt_id, rating, axis=None): if before: rate_views(run.get("views"), before, -1.0) rate_arms(run.get("arms"), before, -1.0) - n = (rate_views(run.get("views"), sign) + rate_arms(run.get("arms"), sign)) if sign else 0 + _apply_detail_lesson(run, before, undo=True) + n = (rate_views(run.get("views"), sign) + rate_arms(run.get("arms"), sign) + + _apply_detail_lesson(run, sign)) if sign else 0 data = _read() data.setdefault("rated", {})[pid] = sign data["rated"] = dict(list(data["rated"].items())[-MAX_RUNS:]) diff --git a/modules/conditioning/shot_camera/nodes.py b/modules/conditioning/shot_camera/nodes.py index 1ffe5e20..c7329f46 100644 --- a/modules/conditioning/shot_camera/nodes.py +++ b/modules/conditioning/shot_camera/nodes.py @@ -66,7 +66,19 @@ def rewrite(text, values, seconds=None, pieces=None): memory.observe(engine.content_fingerprint(before), [w for r in report for w in r.get("lemmas", [])]) moved = [r for r in report if r["move"]] said.append(f"moves: {len(moved)} of {len(report)} shots" + ("" if moved else " (" + "; ".join(f"shot {r['shot']} {r['why']}" for r in report) + ")")) - return text, "; ".join(said), {"views": views, "arms": arms} + details = [] + if values.get("detail_notes") and engine.SHOT.search(text or ""): + base = float(values.get("detail_notes_chance", 0.5)) + bank = [] + for row in memory.detail_bank(): + row = dict(row) + row["chance"] = memory.detail_chance(base, row["good"], row["bad"]) + bank.append(row) + text, info = engine.add_detail_notes(text, seed=seed, chance=base, bank=bank) + details = info.get("added") or [] + said.append("details: " + (", ".join(f"shot {d['shot']} {d['phrase']}" for d in details) if details else info.get("why") or "none")) + return text, "; ".join(said), {"views": views, "arms": arms, "details": details, + "text": text if values.get("detail_notes") else None} class FunPackShotCamera(io.ComfyNode): @@ -93,7 +105,7 @@ class FunPackShotCamera(io.ComfyNode): @classmethod def execute(cls, text: str, length: int, frame_rate: float, settings=None) -> io.NodeOutput: values = _values(settings) - if not any(values.get(k) for k in ("camera_moves", "shot_cuts", "shot_views", "cut_same_shot")) or not engine.SHOT.search(text or ""): + if not any(values.get(k) for k in ("camera_moves", "shot_cuts", "shot_views", "cut_same_shot", "detail_notes")) or not engine.SHOT.search(text or ""): return io.NodeOutput(text, "unchanged") try: out, said, chose = rewrite(text, values, seconds=length / frame_rate if frame_rate else None) @@ -101,7 +113,7 @@ class FunPackShotCamera(io.ComfyNode): log.failed("FunPack Shot Camera", exc) return io.NodeOutput(text, f"left as written ({exc})") pid = _prompt_id() - if pid and (chose["views"] or chose["arms"]): + if pid and (chose["views"] or chose["arms"] or chose.get("text") is not None): try: memory.record_run(pid, chose) except Exception as exc: # noqa: BLE001 -- only the learning is lost, not the run diff --git a/modules/conditioning/shot_camera/tests/test_node.py b/modules/conditioning/shot_camera/tests/test_node.py index e5b68f2e..df29b01b 100644 --- a/modules/conditioning/shot_camera/tests/test_node.py +++ b/modules/conditioning/shot_camera/tests/test_node.py @@ -1,6 +1,8 @@ """The node's rewrite, the rating hook and the module contract.""" import importlib +import pytest + from .. import memory, nodes @@ -46,6 +48,30 @@ def test_rating_teaches_once_and_skips_picture_only(tmp_path, monkeypatch): assert memory.on_rating("unknown", "liked") == 0 +def test_a_like_teaches_a_detail_and_a_bad_composition_can_retire_it(tmp_path, monkeypatch): + pytest.importorskip("spacy") + monkeypatch.setenv("SHOT_CAMERA_MEMORY", str(tmp_path / "m.json")) + m = _mod() + assert m.SETTINGS["detail_notes"]["type"] == "bool" and "when" not in m.SETTINGS["detail_notes"] + assert m.SETTINGS["detail_notes_chance"]["when"] == {"detail_notes": True} + text = "[Shot 1] She kisses his lips." + memory.record_run("liked", {"views": [], "arms": [], "text": "[Shot 1] She has detailed lips.", "details": []}) + assert memory.on_rating("liked", "disliked", "image") == 0 + assert memory.detail_bank() == [] + assert memory.on_rating("liked", "liked") == 1 + assert memory.detail_bank()[0]["phrase"] == "detailed lips" + out, said, chose = nodes.rewrite(text, {"detail_notes": True, "detail_notes_chance": 1}) + assert "Detailed lips." in out and "shot 1 detailed lips" in said + assert out.startswith("[Shot 1] She kisses his lips.") + memory.record_run("bad1", {"views": [], "arms": [], "text": out, "details": chose["details"]}) + memory.record_run("bad2", {"views": [], "arms": [], "text": out, "details": chose["details"]}) + memory.on_rating("bad1", "disliked", "composition") + memory.on_rating("bad2", "disliked", "composition") + assert memory.detail_bank() == [] + quiet, _, _ = nodes.rewrite(text, {"detail_notes": True, "detail_notes_chance": 1}) + assert quiet == text + + def test_contract(): m = _mod() assert m.PROVIDES["on_rating"]("nope", "liked") == 0 diff --git a/modules/conditioning/shot_camera/tests/test_shot_camera.py b/modules/conditioning/shot_camera/tests/test_shot_camera.py index 534e6679..b97a3bc3 100644 --- a/modules/conditioning/shot_camera/tests/test_shot_camera.py +++ b/modules/conditioning/shot_camera/tests/test_shot_camera.py @@ -15,6 +15,30 @@ MUSIC = "Upbeat jazz music with a soft bassline plays throughout." PROMPT = HEAD + S1 + S2 + S3 + MUSIC +def test_a_liked_detail_is_added_only_where_that_body_part_already_is(): + """The sentence already written is left as it is. The detail is a new sentence at the end, + and only in a shot that already names that part without describing it.""" + pytest.importorskip("spacy") + bank = [{"lemma": "lip", "phrase": "detailed lips", "good": 2, "bad": 0, "chance": 1}] + bare = "[Shot 1] She kisses his lips. The music plays." + out, info = sc.add_detail_notes(bare, seed=1, chance=1, bank=bank) + assert info["added"] == [{"shot": 1, "lemma": "lip", "phrase": "detailed lips"}] + assert out.index("his lips") < out.index("Detailed lips.") + assert out.index("Detailed lips.") < out.index("music") + assert "She kisses his lips" in out + + described = "[Shot 1] She presses her wet lips to his cheek." + kept, none = sc.add_detail_notes(described, seed=1, chance=1, bank=bank) + assert kept == described and none["added"] == [] + + elsewhere = "[Shot 1] A lamp sits on the table while the music plays." + same, skipped = sc.add_detail_notes(elsewhere, seed=1, chance=1, bank=bank) + assert same == elsewhere and skipped["added"] == [] + + never, _ = sc.add_detail_notes(bare, seed=1, chance=0, bank=[{**bank[0], "chance": 0}]) + assert never == bare + + def test_no_shot_markers_leaves_the_prompt_alone(): assert sc.add_camera_moves("A woman walks. Music plays.") == ("A woman walks. Music plays.", [])