Compare commits
1
Commits
main
...
feat/multiref
| Author | SHA1 | Date | |
|---|---|---|---|
 alisson-anjosandClaude Opus 5.5
|
7db9eca1c9 |
+47
-26
@@ -1002,10 +1002,19 @@ SAM3_URL = "https://huggingface.co/Comfy-Org/sam3.1/resolve/main/checkpoints/sam
|
||||
_SAM3: dict[str, Any] = {}
|
||||
_MASK_CACHE: dict[tuple, dict] = {}
|
||||
# global mask settings (the planner's Mask card); a shot only says what to segment (text / points / key frame)
|
||||
DEFAULT_MASK = {"text": "", "points": [], "key": 0, "threshold": 0.5, "max_objects": 4, "invert": False,
|
||||
DEFAULT_MASK = {"text": "", "points": [], "key": 0, "keys": [], "threshold": 0.5, "max_objects": 4, "invert": False,
|
||||
"fill_holes": True, "temporal_expand": 2, "blockify": 0, "padding": 0.15, "expand": 16,
|
||||
"feather": 12, "paste": "box"}
|
||||
SHOT_MASK_KEYS = ("text", "points", "key")
|
||||
SHOT_MASK_KEYS = ("text", "points", "key", "keys")
|
||||
|
||||
|
||||
def point_keys(spec: dict) -> list[tuple[int, list]]:
|
||||
"""The shot's key frames with their points, in frame order: spec['keys'] ([{key, points}], 1.116), else the single
|
||||
key frame of older plans (points + key)."""
|
||||
keys = {int(k.get("key") or 0): list(k.get("points") or []) for k in spec.get("keys") or [] if k.get("points")}
|
||||
if not keys and spec.get("points"):
|
||||
keys = {int(spec.get("key") or 0): list(spec["points"])}
|
||||
return sorted(keys.items())
|
||||
|
||||
|
||||
def _sam3_path() -> str:
|
||||
@@ -1074,7 +1083,7 @@ def plan_spec(p: dict, m: dict | None, path: str = "") -> dict:
|
||||
|
||||
|
||||
def spec_has_mask(spec: dict) -> bool:
|
||||
return bool(spec.get("ext") or spec.get("points") or str(spec.get("text") or "").strip())
|
||||
return bool(spec.get("ext") or point_keys(spec) or str(spec.get("text") or "").strip())
|
||||
|
||||
|
||||
def ext_masks(ext: dict, analysis: dict, start: int, length: int, w: int, h: int) -> torch.Tensor:
|
||||
@@ -1207,28 +1216,34 @@ def _track(model, imgs: torch.Tensor, init: torch.Tensor | None, cond, spec: dic
|
||||
|
||||
|
||||
def segment_frames(imgs: torch.Tensor, spec: dict) -> torch.Tensor:
|
||||
"""[N,H,W,3] -> [N,H,W] masks with SAM 3: tracked from the points on the key frame, or from the text."""
|
||||
"""[N,H,W,3] -> [N,H,W] masks with SAM 3, from the text or from the points of the key frames: each key frame's
|
||||
points select the subject there and it is tracked from that frame up to the next key frame (before the first one,
|
||||
backward from it), so a later key frame corrects the mask from its frame on."""
|
||||
from comfy_extras.nodes_sam3 import SAM3_Detect
|
||||
model, clip = _sam3()
|
||||
text = str(spec.get("text") or "").strip()
|
||||
cond = clip.encode_from_tokens_scheduled(clip.tokenize(text)) if text else None
|
||||
pts = spec.get("points") or []
|
||||
N, H, W = imgs.shape[:3]
|
||||
if not pts:
|
||||
keys = [(max(0, min(N - 1, k)), pts) for k, pts in point_keys(spec)]
|
||||
if not keys:
|
||||
if cond is None:
|
||||
raise ValueError("the shot's mask needs a text prompt or points")
|
||||
return _track(model, imgs, None, cond, spec)
|
||||
key = max(0, min(N - 1, int(spec.get("key") or 0)))
|
||||
pos = [{"x": p["x"] * W, "y": p["y"] * H} for p in pts if p.get("label", 1)]
|
||||
neg = [{"x": p["x"] * W, "y": p["y"] * H} for p in pts if not p.get("label", 1)]
|
||||
first = SAM3_Detect.execute(model=model, image=imgs[key:key + 1], positive_coords=json.dumps(pos),
|
||||
negative_coords=json.dumps(neg), threshold=float(spec["threshold"]),
|
||||
refine_iterations=2).args[0][:1].float()
|
||||
fwd = _track(model, imgs[key:], first, None, spec) # key frame -> end
|
||||
if key == 0:
|
||||
return fwd
|
||||
back = _track(model, imgs[:key + 1].flip(0), first, None, spec).flip(0) # key frame -> start
|
||||
return torch.cat([back[:-1], fwd], 0)
|
||||
|
||||
def detect(k, pts):
|
||||
pos = [{"x": p["x"] * W, "y": p["y"] * H} for p in pts if p.get("label", 1)]
|
||||
neg = [{"x": p["x"] * W, "y": p["y"] * H} for p in pts if not p.get("label", 1)]
|
||||
return SAM3_Detect.execute(model=model, image=imgs[k:k + 1], positive_coords=json.dumps(pos),
|
||||
negative_coords=json.dumps(neg), threshold=float(spec["threshold"]),
|
||||
refine_iterations=2).args[0][:1].float()
|
||||
out = torch.zeros(N, H, W)
|
||||
for i, (k, pts) in enumerate(keys):
|
||||
seed = detect(k, pts)
|
||||
if i == 0 and k > 0: # first key frame -> start
|
||||
out[:k + 1] = _track(model, imgs[:k + 1].flip(0), seed, None, spec).flip(0)
|
||||
end = keys[i + 1][0] if i + 1 < len(keys) else N # this key frame -> the next one
|
||||
out[k:end] = _track(model, imgs[k:end], seed, None, spec)
|
||||
return out
|
||||
|
||||
|
||||
def _mask_key(path: str, analysis: dict, start: int, length: int, spec: dict) -> tuple:
|
||||
@@ -1244,7 +1259,7 @@ def _mask_key(path: str, analysis: dict, start: int, length: int, spec: dict) ->
|
||||
else:
|
||||
e = None
|
||||
return (path, os.path.getmtime(path), float(analysis["fps"]), int(start), int(length),
|
||||
json.dumps({k: spec[k] for k in ("text", "points", "key", "threshold", "max_objects")}, sort_keys=True), e)
|
||||
json.dumps({"keys": point_keys(spec), **{k: spec[k] for k in ("text", "threshold", "max_objects")}}, sort_keys=True), e)
|
||||
|
||||
|
||||
def shot_mask(path: str, analysis: dict, start: int, length: int, spec: dict) -> dict:
|
||||
@@ -1268,8 +1283,8 @@ def shot_mask(path: str, analysis: dict, start: int, length: int, spec: dict) ->
|
||||
frames = _read_frames(path, src[idx], (w, h), stage="Reading the shot")
|
||||
imgs = torch.from_numpy(np.stack(frames).astype(np.float32) / 255.0)
|
||||
_status(f"Segmenting {len(frames)} frames with SAM 3", force=True)
|
||||
what = f"{len(spec.get('points') or [])} points on frame {int(spec.get('key') or 0)}" if spec.get("points") \
|
||||
else f"'{spec.get('text', '')}'"
|
||||
keys = point_keys(spec)
|
||||
what = (", ".join(f"{len(p)} points on frame {k}" for k, p in keys)) if keys else f"'{spec.get('text', '')}'"
|
||||
print(f"[BFS Shot] SAM 3: segmenting {what} on {len(frames)} frames ({w}x{h}, timeline {start}-{start + length - 1})…",
|
||||
flush=True)
|
||||
t0 = time.time()
|
||||
@@ -1480,13 +1495,15 @@ def uncrop(result: torch.Tensor, shot: dict, edge_hardness: float = 0.0, match_c
|
||||
|
||||
def mask_preview(path: str, analysis: dict, start: int, length: int, spec: dict, count: int = 6,
|
||||
color: str = "#ff283c", opacity: float = 0.55) -> dict:
|
||||
"""A few frames of the shot with the selected region tinted and the crop box, as data URLs."""
|
||||
"""`count` frames of the shot (0 = every frame, smaller, for the points window's player) with the selected region
|
||||
tinted and the crop box, as data URLs."""
|
||||
import cv2
|
||||
r = shot_mask(path, analysis, start, length, spec)
|
||||
masks, (w, h), box = r["masks"], r["size"], r["box"]
|
||||
src = _timeline(analysis["n_src"], analysis["fps_src"], float(analysis["fps"]))
|
||||
picks = sorted(set(int(round(i)) for i in np.linspace(0, length - 1, min(count, length))))
|
||||
picks = list(range(length)) if count <= 0 else sorted(set(int(round(i)) for i in np.linspace(0, length - 1, min(count, length))))
|
||||
frames = _read_frames(path, src[np.clip(np.array(picks) + start, 0, analysis["n"] - 1)], (w, h))
|
||||
small = 384 / max(w, h) if count <= 0 and max(w, h) > 384 else 1.0 # every frame: ~10 KB each
|
||||
try:
|
||||
tint = np.array([int(color[i:i + 2], 16) for i in (1, 3, 5)], dtype=np.float32)
|
||||
alpha = min(0.8, max(0.05, float(opacity)))
|
||||
@@ -1500,7 +1517,9 @@ def mask_preview(path: str, analysis: dict, start: int, length: int, spec: dict,
|
||||
if box:
|
||||
x0, y0, x1, y1 = box_px(box, w, h)
|
||||
cv2.rectangle(img, (x0, y0), (x1 - 1, y1 - 1), (255, 210, 90), 2)
|
||||
ok, buf = cv2.imencode(".jpg", cv2.cvtColor(img, cv2.COLOR_RGB2BGR), [cv2.IMWRITE_JPEG_QUALITY, 80])
|
||||
if small < 1.0:
|
||||
img = cv2.resize(img, (int(w * small) // 2 * 2, int(h * small) // 2 * 2), interpolation=cv2.INTER_AREA)
|
||||
ok, buf = cv2.imencode(".jpg", cv2.cvtColor(img, cv2.COLOR_RGB2BGR), [cv2.IMWRITE_JPEG_QUALITY, 80 if small == 1.0 else 70])
|
||||
out.append({"f": start + i, "src": "data:image/jpeg;base64," + base64.b64encode(buf.tobytes()).decode()})
|
||||
cover = float(masks.float().mean()) if masks.numel() else 0.0
|
||||
return {"box": box, "frames": out, "coverage": cover, "empty": box is None}
|
||||
@@ -1519,7 +1538,8 @@ TARGET_Q = ("Describe the main subject in this picture (a person, an animal, a c
|
||||
def target_crop(path: str, analysis: dict, start: int, length: int, spec: dict, pad: float = 0.12):
|
||||
"""The selected subject (the shot's SAM 3 points / text, on its key frame) cropped from that frame, RGB uint8."""
|
||||
src = _timeline(analysis["n_src"], analysis["fps_src"], float(analysis["fps"]))
|
||||
key = max(0, min(length - 1, int(spec.get("key") or length // 2)))
|
||||
keys = point_keys(spec)
|
||||
key = max(0, min(length - 1, keys[0][0] if keys else int(spec.get("key") or length // 2)))
|
||||
fr = _read_frames(path, src[np.clip(np.array([start + key]), 0, analysis["n"] - 1)], None)[0]
|
||||
if spec.get("ext"):
|
||||
sm = shot_mask(path, analysis, start, length, spec)["masks"]
|
||||
@@ -3965,8 +3985,8 @@ def color_marks(plan: dict, m: dict) -> list[dict]:
|
||||
"key": m.get("color_guide_key2")}] if m.get("color_guide_text2") or m.get("color_guide_points2") else [])
|
||||
marks = [{"picture": max(1, int(m.get("color_guide_picture") or 1))}]
|
||||
marks += [{"picture": max(1, int(r.get("picture") or 2)), "text": str(r.get("text") or "").strip(),
|
||||
"points": list(r.get("points") or []), "key": int(r.get("key") or 0)}
|
||||
for r in extra if isinstance(r, dict) and (str(r.get("text") or "").strip() or r.get("points"))]
|
||||
"points": list(r.get("points") or []), "key": int(r.get("key") or 0), "keys": list(r.get("keys") or [])}
|
||||
for r in extra if isinstance(r, dict) and (str(r.get("text") or "").strip() or r.get("points") or r.get("keys"))]
|
||||
for x in marks:
|
||||
x["color"] = picture_color(plan, x["picture"])
|
||||
return marks
|
||||
@@ -3985,6 +4005,7 @@ def extra_mark_masks(shot: dict, log: str = "") -> list[tuple]:
|
||||
out = []
|
||||
for x in extra:
|
||||
spec = dict(DEFAULT_MASK, text=x.get("text") or "", points=x.get("points") or [], key=x.get("key") or 0,
|
||||
keys=x.get("keys") or [],
|
||||
max_objects=1)
|
||||
out.append((shot_mask(ms["path"], ms["analysis"], ms["start"], ms["length"], spec)["masks"], x))
|
||||
return out
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
[project]
|
||||
name = "bfsnodes"
|
||||
description = "Custom ComfyUI nodes for reference-guided video: BFS Sampler (guide-driven IC-LoRAs, auto crop, mask inpainting, temporal chunking), LTX Multiple Controls / Identity Transfer / Edit Anything, Head-Swap and Anime2Real conditioning, MiniMax-H3 latent guides, RoPE layouts and loaders, LoRA Surgery, and Shot Planner / Shot Loop (tabbed panel): split long videos into shots at camera cuts, face-tracked cast, SAM 3 mask per shot with full frame, paste, mask, crop and crop + mask generation modes, optional translucent Colored Mark overlay for manually selected regions, optional Qwen3-VL to suggest settings, describe references ({details}) and the person to replace ({target}), continuity between shots, H3 conditioning with an optional duet and prompt writer, join with the original timing, audio and a side-by-side comparison. H3 Duet: pin a source clip beside the video, training-free."
|
||||
version = "1.115.1"
|
||||
version = "1.116.0"
|
||||
license = {file = "LICENSE"}
|
||||
dependencies = [
|
||||
"torch",
|
||||
|
||||
@@ -1088,6 +1088,52 @@ class MultiRefTest(unittest.TestCase):
|
||||
self.assertNotIn("ref_image", SL.BFSShotPlanner.INPUT_TYPES()["optional"])
|
||||
|
||||
|
||||
class KeyFrameSegmentTest(unittest.TestCase):
|
||||
"""Points on several frames of a shot: each key frame starts the track up to the next one."""
|
||||
|
||||
def setUp(self):
|
||||
import json as _json
|
||||
|
||||
class Detect: # the mask of a key frame carries its first point's x (in pixels), to see who filled a frame
|
||||
@staticmethod
|
||||
def execute(model, image, positive_coords, **kw):
|
||||
return types.SimpleNamespace(args=(torch.full((1, 4, 4), _json.loads(positive_coords)[0]["x"]),))
|
||||
mod = types.ModuleType("comfy_extras.nodes_sam3"); mod.SAM3_Detect = Detect
|
||||
sys.modules.setdefault("comfy_extras", types.ModuleType("comfy_extras"))
|
||||
self._old = (sys.modules.get("comfy_extras.nodes_sam3"), SL._sam3, SL._track)
|
||||
sys.modules["comfy_extras.nodes_sam3"] = mod
|
||||
SL._sam3 = lambda: (None, None)
|
||||
self.tracks = []
|
||||
|
||||
def track(model, imgs, seed, cond, spec):
|
||||
self.tracks.append((imgs.shape[0], float(seed.flatten()[0])))
|
||||
return seed.expand(imgs.shape[0], -1, -1).clone()
|
||||
SL._track = track
|
||||
|
||||
def tearDown(self):
|
||||
mod, SL._sam3, SL._track = self._old
|
||||
if mod is None:
|
||||
sys.modules.pop("comfy_extras.nodes_sam3", None)
|
||||
else:
|
||||
sys.modules["comfy_extras.nodes_sam3"] = mod
|
||||
|
||||
def test_each_key_frame_corrects_the_mask_from_its_frame_on(self):
|
||||
pt = lambda x: [{"x": x, "y": 0.5, "label": 1}]
|
||||
spec = dict(SL.DEFAULT_MASK, keys=[{"key": 6, "points": pt(0.75)}, {"key": 2, "points": pt(0.25)}])
|
||||
out = SL.segment_frames(torch.zeros(10, 4, 4, 3), spec)
|
||||
self.assertEqual([float(out[f, 0, 0]) for f in range(10)], [1.0] * 6 + [3.0] * 4) # frames 0-5 from key 2, 6-9 from key 6
|
||||
self.assertEqual(self.tracks, [(3, 1.0), (4, 1.0), (4, 3.0)]) # back 2->0, forward 2->5, forward 6->9
|
||||
self.assertTrue(SL.spec_has_mask(dict(SL.DEFAULT_MASK, keys=[{"key": 3, "points": pt(0.5)}])))
|
||||
self.assertFalse(SL.spec_has_mask(dict(SL.DEFAULT_MASK, keys=[{"key": 3, "points": []}])))
|
||||
|
||||
def test_a_plan_with_one_key_frame_works_as_before(self):
|
||||
spec = dict(SL.DEFAULT_MASK, points=[{"x": 0.5, "y": 0.5, "label": 1}], key=4)
|
||||
self.assertEqual(SL.point_keys(spec), [(4, spec["points"])])
|
||||
out = SL.segment_frames(torch.zeros(8, 4, 4, 3), spec)
|
||||
self.assertEqual(self.tracks, [(5, 2.0), (4, 2.0)])
|
||||
self.assertEqual(float(out.sum()), 2.0 * 8 * 16)
|
||||
|
||||
|
||||
class DescribeShortTest(unittest.TestCase):
|
||||
def test_short_details_fit_inside_the_prompt(self):
|
||||
self.assertEqual(SL.tidy_details('"The green goblin in a torn leather vest."\nextra', {"describe_preset": "short"}),
|
||||
|
||||
+53
-22
@@ -70,7 +70,8 @@ function Panel(io) {
|
||||
const maskPrev = ref({}); // "start-end" -> {frames, box, coverage, empty} from /bfs/shotloop/mask
|
||||
const showMasks = ref(true); // overlay the mask preview on the shot cards
|
||||
const vlmSug = ref({}); // "start-end" -> {segment, shot, people, recommend, reason} from the VLM
|
||||
const modal = reactive({ open: false, idx: -1, key: 0, points: [], text: "", region: 1, groups: {}, prev: null, busy: "" });
|
||||
const modal = reactive({ open: false, idx: -1, key: 0, points: [], text: "", region: 1, groups: {}, prev: null, stale: false,
|
||||
showMask: true, playing: 0, busy: "" });
|
||||
const segPeople = ref({}); // "start-end" -> {people, main} from /bfs/shotloop/plan
|
||||
const targetCrop = ref({}); // "start-end" -> data URL of the subject the description was made from
|
||||
const tab = ref(store.get("bfs.shotloop.tab", "shots"));
|
||||
@@ -352,15 +353,17 @@ const meta = i => { while (plan.segs.length <= i) plan.segs.push({}); return pla
|
||||
} catch (e) { error.value = String(e.message || e); }
|
||||
busy.value = "";
|
||||
}
|
||||
// intoModal: every frame of the shot, for the points window's player (count 0)
|
||||
async function previewMask(i, spec, intoModal = false) {
|
||||
const s = segs.value[i]; if (!s) return;
|
||||
const label = "Segmenting with SAM 3… (the first time downloads/loads the model)";
|
||||
if (intoModal) { modal.busy = label; busySince.value = Date.now(); Object.assign(status, { stage: "", done: 0, total: 0 }); } else busy.value = label;
|
||||
error.value = "";
|
||||
try {
|
||||
const r = await api.fetchApi("/bfs/shotloop/mask", { method: "POST", body: JSON.stringify({ plan: { ...plan }, index: i, mask: spec || null }) });
|
||||
const body = { plan: { ...plan }, index: i, mask: spec || null, ...(intoModal ? { count: 0 } : {}) };
|
||||
const r = await api.fetchApi("/bfs/shotloop/mask", { method: "POST", body: JSON.stringify(body) });
|
||||
const j = await r.json(); if (j.error) throw new Error(j.error);
|
||||
if (intoModal) modal.prev = j; else maskPrev.value = { ...maskPrev.value, [segKey(s)]: j };
|
||||
if (intoModal) { modal.prev = j; modal.stale = false; modal.showMask = true; } else maskPrev.value = { ...maskPrev.value, [segKey(s)]: j };
|
||||
} catch (e) { error.value = String(e.message || e); }
|
||||
if (intoModal) modal.busy = ""; else busy.value = "";
|
||||
}
|
||||
@@ -371,29 +374,52 @@ const meta = i => { while (plan.segs.length <= i) plan.segs.push({}); return pla
|
||||
if (s && maskPrev.value[segKey(s)]) { const mp = { ...maskPrev.value }; delete mp[segKey(s)]; maskPrev.value = mp; }
|
||||
};
|
||||
const setMaskCfg = (k, v) => { plan.mask_cfg = { ...plan.mask_cfg, [k]: v }; save(); maskPrev.value = {}; };
|
||||
// Colored Mark regions in the points window: 1 = the shot's own selection (its mask), 2.. = its extra regions
|
||||
// The points window. A selection (region 1 = the shot's own mask, 2.. = its extra Colored Mark regions) has key
|
||||
// frames, each with its own points: SAM 3 starts from a key frame and tracks up to the next one, so points added on a
|
||||
// later frame correct the mask from there on. modal.key is the frame shown, modal.points its points (being edited).
|
||||
const keyFramesOf = m => (Array.isArray(m?.keys) && m.keys.length ? m.keys.map(k => ({ key: +k.key || 0, points: [...(k.points || [])] }))
|
||||
: (m?.points || []).length ? [{ key: +m.key || 0, points: [...m.points] }] : []);
|
||||
const pointGroup = () => modal.groups[modal.region];
|
||||
const keepFrame = () => { // the frame being edited back into its selection's key frames
|
||||
const g = pointGroup(); if (!g) return;
|
||||
g.keys = g.keys.filter(k => k.key !== modal.key);
|
||||
if (modal.points.length) g.keys = [...g.keys, { key: modal.key, points: [...modal.points] }].sort((a, b) => a.key - b.key);
|
||||
g.text = modal.text;
|
||||
};
|
||||
const showFrame = f => {
|
||||
keepFrame();
|
||||
modal.key = Math.max(0, Math.min((segs.value[modal.idx]?.len || 1) - 1, f));
|
||||
modal.points = [...(pointGroup()?.keys.find(k => k.key === modal.key)?.points || [])];
|
||||
};
|
||||
const dropKeyFrame = f => {
|
||||
const g = pointGroup(); g.keys = g.keys.filter(k => k.key !== f);
|
||||
if (f === modal.key) modal.points = [];
|
||||
modal.stale = !!modal.prev;
|
||||
};
|
||||
const openPoints = (i, region = 1) => {
|
||||
const s = segs.value[i]; if (!s) return;
|
||||
const g1 = { key: s.mask.key ?? Math.floor(s.len / 2), points: [...(s.mask.points || [])], text: s.mask.text || "" };
|
||||
const groups = { 1: g1 };
|
||||
if (s.color_guide) s.regions.forEach((r, j) => { groups[j + 2] = { key: r.key ?? g1.key, points: [...(r.points || [])], text: r.text || "" }; });
|
||||
const g = groups[region] || g1;
|
||||
Object.assign(modal, { open: true, idx: i, region: groups[region] ? region : 1, groups, key: g.key, points: [...g.points],
|
||||
text: g.text, prev: null, busy: "" });
|
||||
const groups = { 1: { keys: keyFramesOf(s.mask), text: s.mask.text || "" } };
|
||||
if (s.color_guide) s.regions.forEach((r, j) => { groups[j + 2] = { keys: keyFramesOf(r), text: r.text || "" }; });
|
||||
const r0 = groups[region] ? region : 1, key = groups[r0].keys[0]?.key ?? Math.floor(s.len / 2);
|
||||
Object.assign(modal, { open: true, idx: i, region: r0, groups, key, points: [...(groups[r0].keys[0]?.points || [])],
|
||||
text: groups[r0].text, prev: null, stale: false, showMask: true, busy: "" });
|
||||
};
|
||||
const switchPointRegion = region => {
|
||||
modal.groups[modal.region] = { key: modal.key, points: [...modal.points], text: modal.text };
|
||||
const next = modal.groups[region] || { key: modal.key, points: [], text: "" };
|
||||
modal.region = region; modal.points = [...next.points]; modal.text = next.text; modal.prev = null;
|
||||
keepFrame();
|
||||
modal.region = region; modal.text = pointGroup()?.text || "";
|
||||
modal.points = [...(pointGroup()?.keys.find(k => k.key === modal.key)?.points || [])]; modal.prev = null;
|
||||
};
|
||||
// the selection being edited as a mask spec (what Segment sends)
|
||||
const pointSpec = () => { keepFrame(); const g = pointGroup(); return { keys: g.keys, points: g.keys[0]?.points || [], key: g.keys[0]?.key ?? modal.key, text: g.text }; };
|
||||
const commitPointGroups = i => {
|
||||
modal.groups[modal.region] = { key: modal.key, points: [...modal.points], text: modal.text };
|
||||
keepFrame();
|
||||
const m = meta(i), g1 = modal.groups[1];
|
||||
m.mask = { ...(m.mask || {}), points: [...g1.points], key: g1.key, text: g1.text };
|
||||
// keys is what 1.116 reads; points + key (the first key frame) for older versions
|
||||
m.mask = { ...(m.mask || {}), keys: g1.keys, points: g1.keys[0]?.points || [], key: g1.keys[0]?.key ?? 0, text: g1.text };
|
||||
if (segs.value[i]?.color_guide) {
|
||||
m.color_guide_regions = markRegions(m).map((r, j) => {
|
||||
const g = modal.groups[j + 2];
|
||||
return g ? { ...r, points: [...g.points], key: g.key, text: g.text } : r;
|
||||
return g ? { ...r, keys: g.keys, points: g.keys[0]?.points || [], key: g.keys[0]?.key ?? 0, text: g.text } : r;
|
||||
});
|
||||
}
|
||||
};
|
||||
@@ -421,7 +447,11 @@ const meta = i => { while (plan.segs.length <= i) plan.segs.push({}); return pla
|
||||
};
|
||||
const savePoints = () => {
|
||||
const i = modal.idx; commitPointGroups(i); save();
|
||||
if (modal.prev) maskPrev.value = { ...maskPrev.value, [segKey(segs.value[modal.idx])]: modal.prev };
|
||||
if (modal.prev && !modal.stale) { // the card's strip shows 6 of the window's frames
|
||||
const fr = modal.prev.frames, pick = [...new Set(Array.from({ length: Math.min(6, fr.length) }, (_, k) => Math.round(k * (fr.length - 1) / Math.max(1, Math.min(6, fr.length) - 1))))];
|
||||
maskPrev.value = { ...maskPrev.value, [segKey(segs.value[modal.idx])]: { ...modal.prev, frames: pick.map(k => fr[k]) } };
|
||||
}
|
||||
clearInterval(modal.playing); modal.playing = 0;
|
||||
modal.open = false;
|
||||
};
|
||||
const targets = (scope, from) => segs.value.map((_, i) => i).filter(i => i !== from && (scope === "all" || i > from));
|
||||
@@ -431,7 +461,8 @@ const meta = i => { while (plan.segs.length <= i) plan.segs.push({}); return pla
|
||||
const src = segs.value[fromIdx]; const rel = src ? key / Math.max(1, src.len - 1) : 0.5;
|
||||
[fromIdx, ...targets(scope, fromIdx)].forEach(i => {
|
||||
const x = segs.value[i], m = meta(i);
|
||||
m.mask = { ...(m.mask || {}), points: points.map(p => ({ ...p })), key: Math.round(rel * Math.max(0, x.len - 1)), text: text ?? (m.mask || {}).text };
|
||||
const key = Math.round(rel * Math.max(0, x.len - 1));
|
||||
m.mask = { ...(m.mask || {}), keys: [{ key, points: points.map(p => ({ ...p })) }], points: points.map(p => ({ ...p })), key, text: text ?? (m.mask || {}).text };
|
||||
});
|
||||
maskPrev.value = {}; save();
|
||||
};
|
||||
@@ -444,13 +475,13 @@ const meta = i => { while (plan.segs.length <= i) plan.segs.push({}); return pla
|
||||
// part of that track, so the subject is found wherever it moves (copied points can fall on the background)
|
||||
const saveTrack = () => {
|
||||
const s0 = segs.value[modal.idx]; if (!s0 || !modal.points.length) return;
|
||||
const own = segs.value.filter((x, i) => i !== modal.idx && (x.mask.text || (x.mask.points || []).length)).length;
|
||||
const own = segs.value.filter((x, i) => i !== modal.idx && (x.mask.text || (x.mask.points || []).length || (x.mask.keys || []).length)).length;
|
||||
if (own && !window.confirm(`${own} other shot(s) have their own mask text or points, which would win over the track. Clear them so every shot uses the track?`)) {
|
||||
// keep them: those shots keep their own selection
|
||||
} else {
|
||||
segs.value.forEach((_, i) => { const m = meta(i); if (m.mask) { m.mask = { ...m.mask, points: [], text: "" }; } });
|
||||
segs.value.forEach((_, i) => { const m = meta(i); if (m.mask) { m.mask = { ...m.mask, points: [], keys: [], text: "" }; } });
|
||||
}
|
||||
const m = meta(modal.idx); m.mask = { ...(m.mask || {}), points: [], text: "" };
|
||||
const m = meta(modal.idx); m.mask = { ...(m.mask || {}), points: [], keys: [], text: "" };
|
||||
plan.track = { points: modal.points.map(p => ({ ...p })), frame: s0.start + modal.key, text: modal.text || "",
|
||||
threshold: plan.mask_cfg?.threshold ?? 0.5 };
|
||||
maskPrev.value = {}; save(); modal.open = false;
|
||||
@@ -873,7 +904,7 @@ const meta = i => { while (plan.segs.length <= i) plan.segs.push({}); return pla
|
||||
refreshFiles, analyze, autoSplit, analyzeContent, findPeople, setCast, describeTarget, previewMask, setMask, setMaskCfg,
|
||||
openPoints, switchPointRegion, savePoints, savePointsAll, saveTrack, clearTrack, pointsTo, clearMask, copyFrom, analyseVLM, setVlmCfg,
|
||||
applySug, applyAllSug, setCastOpt, setGlobalMaskVideo, setFilter, pickFile, progress, setVideo, splitAt, mergeNext, removeCut, select, drag,
|
||||
frameAt, thumbFor, playShot, playAll, stop, seek, setRef, putRefs, uploadRef, dropRefs, pictures, describePictures, setPictureDetail, setPictureCfg,
|
||||
frameAt, thumbFor, playShot, playAll, stop, seek, setRef, putRefs, uploadRef, dropRefs, showFrame, dropKeyFrame, pointSpec, keyFramesOf, pictures, describePictures, setPictureDetail, setPictureCfg,
|
||||
setMarkPicture, addMarkRegion, setMarkRegion, removeMarkRegion, setPictureColor,
|
||||
autoFor, writtenOf, setAuto, setWritten, writePrompts,
|
||||
};
|
||||
|
||||
@@ -209,6 +209,11 @@ const CSS = `
|
||||
.bsl .pt.pos{background:#3ccf6b}.bsl .pt.neg{background:#e8455a}
|
||||
.bsl .pt.cg.pos{background:var(--point-color,#00ff00)}
|
||||
.bsl .pt.other{opacity:.5}
|
||||
.bsl .kslider{position:relative;flex:1;min-width:160px;padding-bottom:8px}
|
||||
.bsl .kmarks{position:absolute;left:8px;right:8px;bottom:0;height:8px}
|
||||
.bsl .kmark{position:absolute;width:8px;height:8px;margin-left:-4px;border-radius:50%;background:var(--mark,#3ccf6b);opacity:.55;cursor:pointer;border:1px solid #000}
|
||||
.bsl .kmark.on{opacity:1}
|
||||
.bsl .kchip{display:inline-flex;gap:1px}.bsl .kchip.on button:first-child{border-color:var(--acc)}
|
||||
.bsl .mreg{display:flex;gap:6px;align-items:center;flex-wrap:wrap;margin-top:6px}
|
||||
`;
|
||||
|
||||
|
||||
@@ -15,68 +15,106 @@ export function busyBar(c, msg) {
|
||||
}
|
||||
|
||||
export function pointsModal(c) {
|
||||
const { modal, segs, plan, api, setMeta } = c;
|
||||
const { modal, segs, plan, api } = c;
|
||||
const ms = segs.value[modal.idx];
|
||||
if (!modal.open || !ms) return null;
|
||||
// Colored Mark: region 1 = the shot's own selection, 2.. = its extra regions; each one is a picture, in its color
|
||||
const regionPicture = r => (r === 1 ? ms.color_guide_picture : ms.regions[r - 2]?.picture) || 1;
|
||||
const regionColor = r => pictureColor(plan, regionPicture(r));
|
||||
const regionColor = r => (ms.color_guide ? pictureColor(plan, regionPicture(r)) : "#3ccf6b");
|
||||
const regionsN = ms.color_guide ? 1 + ms.regions.length : 1;
|
||||
const pointOverlays = ms.color_guide
|
||||
? Array.from({ length: regionsN }, (_, q) => q + 1).flatMap(region => (region === modal.region ? modal.points : modal.groups[region]?.points || []).map((p, k) => h("div", {
|
||||
class: ["pt", "cg", region !== modal.region && "other", p.label ? "pos" : "neg"],
|
||||
style: `left:${p.x * 100}%;top:${p.y * 100}%;--point-color:${regionColor(region)}`,
|
||||
title: `Region ${region} · Picture ${regionPicture(region)}${region === modal.region ? " · click to remove" : ""}`,
|
||||
onPointerdown: e => { if (region !== modal.region) return; e.stopPropagation(); modal.points = modal.points.filter((_, j) => j !== k); modal.prev = null; },
|
||||
})))
|
||||
: modal.points.map((p, k) => h("div", { class: ["pt", p.label ? "pos" : "neg"], style: `left:${p.x * 100}%;top:${p.y * 100}%`,
|
||||
title: "click to remove", onPointerdown: e => { e.stopPropagation(); modal.points = modal.points.filter((_, j) => j !== k); modal.prev = null; } }));
|
||||
const regions = Array.from({ length: regionsN }, (_, q) => q + 1);
|
||||
// the points of a region on the frame shown (the one being edited are in modal.points)
|
||||
const pointsAt = r => (r === modal.region ? modal.points : modal.groups[r]?.keys.find(k => k.key === modal.key)?.points || []);
|
||||
const keysOf = r => {
|
||||
const ks = new Set((modal.groups[r]?.keys || []).map(k => k.key));
|
||||
if (r === modal.region) { if (modal.points.length) ks.add(modal.key); else ks.delete(modal.key); }
|
||||
return [...ks].sort((a, b) => a - b);
|
||||
};
|
||||
const edited = () => { modal.stale = !!modal.prev; };
|
||||
const pointOverlays = regions.flatMap(r => pointsAt(r).map((p, k) => h("div", {
|
||||
class: ["pt", ms.color_guide && "cg", r !== modal.region && "other", p.label ? "pos" : "neg"],
|
||||
style: `left:${p.x * 100}%;top:${p.y * 100}%;--point-color:${regionColor(r)}`,
|
||||
title: r === modal.region ? "click to remove" : `Picture ${regionPicture(r)}`,
|
||||
onPointerdown: e => { if (r !== modal.region) return; e.stopPropagation(); modal.points = modal.points.filter((_, j) => j !== k); edited(); },
|
||||
})));
|
||||
const regionButton = r => {
|
||||
const k = regionPicture(r), name = ms.names[k - 1];
|
||||
return h("button", { class: modal.region === r ? "pri" : "", onClick: () => c.switchPointRegion(r),
|
||||
title: `Region ${r}: becomes <Picture ${k}> (${markName(regionColor(r))})` }, [
|
||||
name ? h("img", { class: "mthumb", src: c.viewUrl(name), style: `--mark:${regionColor(r)}` }) : null,
|
||||
` Picture ${k} · ${markName(regionColor(r))} (${(r === modal.region ? modal.points : modal.groups[r]?.points || []).length})`]);
|
||||
` Picture ${k} · ${markName(regionColor(r))} (${keysOf(r).length} key frame${keysOf(r).length === 1 ? "" : "s"})`]);
|
||||
};
|
||||
// after Segment: the mask over every frame of the shot (modal.prev.frames, one per frame)
|
||||
const masked = modal.prev && modal.showMask ? modal.prev.frames.find(f => f.f === ms.start + modal.key) : null;
|
||||
const play = () => {
|
||||
if (modal.playing) { clearInterval(modal.playing); modal.playing = 0; return; }
|
||||
modal.playing = setInterval(() => {
|
||||
if (modal.key >= ms.len - 1 || !modal.open) { clearInterval(modal.playing); modal.playing = 0; return; }
|
||||
c.showFrame(modal.key + 1);
|
||||
}, 1000 / Math.min(24, plan.fps || 24));
|
||||
};
|
||||
const marks = h("div", { class: "kmarks" }, regions.flatMap(r => keysOf(r).map(f => h("span", {
|
||||
class: ["kmark", r === modal.region && "on"], style: `left:${(f / Math.max(1, ms.len - 1)) * 100}%;--mark:${regionColor(r)}`,
|
||||
title: `key frame ${ms.start + f}`, onClick: () => c.showFrame(f) }))));
|
||||
return h(Teleport, { to: "body" }, h("div", { class: "bsl", style: "background:none;padding:0" }, h("div", { class: "mmodal",
|
||||
onKeydown: e => e.stopPropagation(), onPointerdown: e => { if (e.target === e.currentTarget) modal.open = false; } }, [
|
||||
h("div", { class: "mbox" }, [
|
||||
onKeydown: e => {
|
||||
e.stopPropagation();
|
||||
if (e.target.tagName === "INPUT" && e.target.type === "text") return;
|
||||
if (e.key === "ArrowLeft") c.showFrame(modal.key - 1);
|
||||
if (e.key === "ArrowRight") c.showFrame(modal.key + 1);
|
||||
}, onPointerdown: e => { if (e.target === e.currentTarget) modal.open = false; } }, [
|
||||
h("div", { class: "mbox", tabindex: 0 }, [
|
||||
h("div", { class: "row", style: "margin-bottom:6px" }, [h("span", { class: "sect" }, `Shot #${modal.idx + 1} · points`),
|
||||
ms.color_guide ? h("div", { class: "row" }, [hint("One point set per marked person; switch to edit each. All use the frame selected below."),
|
||||
...Array.from({ length: regionsN }, (_, q) => regionButton(q + 1))]) : null,
|
||||
hint("click = keep · right-click or shift+click = exclude · click a point to remove it")]),
|
||||
ms.color_guide ? h("div", { class: "row" }, regions.map(regionButton)) : null,
|
||||
hint("click = keep · right-click or shift+click = exclude · click a point to remove it · points on another frame = a new key frame: the mask is tracked from each key frame up to the next one")]),
|
||||
h("div", { class: "mstage" }, [
|
||||
h("div", { class: "mimg", onContextmenu: e => e.preventDefault(), onPointerdown: e => {
|
||||
const box = e.currentTarget.getBoundingClientRect();
|
||||
const x = (e.clientX - box.left) / box.width, y = (e.clientY - box.top) / box.height;
|
||||
modal.points = [...modal.points, { x: +x.toFixed(4), y: +y.toFixed(4), label: (e.button === 2 || e.shiftKey) ? 0 : 1 }];
|
||||
modal.prev = null;
|
||||
edited();
|
||||
} }, [
|
||||
h("img", { draggable: false, src: api.apiURL(`/bfs/shotloop/frame?video=${encodeURIComponent(plan.video)}&fps=${plan.fps}&f=${ms.start + modal.key}&w=900`) }),
|
||||
h("img", { draggable: false, src: masked ? masked.src : api.apiURL(`/bfs/shotloop/frame?video=${encodeURIComponent(plan.video)}&fps=${plan.fps}&f=${ms.start + modal.key}&w=900`) }),
|
||||
...pointOverlays,
|
||||
])]),
|
||||
h("div", { class: "row", style: "margin-top:6px" }, [
|
||||
h("div", { class: "row", style: "margin-top:6px;align-items:center" }, [
|
||||
h("button", { title: "play the shot (with the mask after Segment)", onClick: play }, modal.playing ? "⏸" : "▶"),
|
||||
h("button", { title: "previous frame (←)", onClick: () => c.showFrame(modal.key - 1) }, "◀"),
|
||||
h("button", { title: "next frame (→)", onClick: () => c.showFrame(modal.key + 1) }, "▶|"),
|
||||
hint(`frame ${ms.start + modal.key} (${modal.key + 1}/${ms.len})`),
|
||||
h("input", { type: "range", min: 0, max: ms.len - 1, step: 1, value: modal.key, style: "flex:1",
|
||||
onInput: e => { modal.key = parseInt(e.target.value); modal.prev = null; } }),
|
||||
h("div", { class: "kslider" }, [
|
||||
h("input", { type: "range", min: 0, max: ms.len - 1, step: 1, value: modal.key, style: "width:100%",
|
||||
onInput: e => c.showFrame(parseInt(e.target.value)) }),
|
||||
marks,
|
||||
]),
|
||||
modal.prev ? h("label", { class: "row", style: "gap:4px" }, [h("input", { type: "checkbox", checked: modal.showMask,
|
||||
onChange: e => { modal.showMask = e.target.checked; } }), "mask"]) : null,
|
||||
]),
|
||||
h("div", { class: "row", style: "margin-top:4px;gap:4px;flex-wrap:wrap" }, [
|
||||
hint("key frames:"),
|
||||
...(keysOf(modal.region).length ? keysOf(modal.region).map(f => h("span", { class: ["kchip", f === modal.key && "on"] }, [
|
||||
h("button", { class: "sm", onClick: () => c.showFrame(f) }, `frame ${ms.start + f}`),
|
||||
h("button", { class: "sm dng", title: "remove this key frame's points", onClick: () => { c.dropKeyFrame(f); edited(); } }, "✕")]))
|
||||
: [hint("none yet: click on the frame to add points")]),
|
||||
]),
|
||||
h("div", { class: "row", style: "margin-top:6px" }, [
|
||||
h("input", { type: "text", value: modal.text, style: "flex:1", placeholder: "text prompt (used when there are no points)",
|
||||
onChange: e => { modal.text = e.target.value; } }),
|
||||
h("button", { onClick: () => { modal.points = []; modal.prev = null; } }, "Clear points"),
|
||||
h("button", { class: "pri", disabled: !!modal.busy || !(modal.points.length || modal.text),
|
||||
onClick: () => c.previewMask(modal.idx, { points: modal.points, key: modal.key, text: modal.text,
|
||||
preview_color: ms.color_guide ? regionColor(modal.region) : undefined }, true) }, "👁 Segment"),
|
||||
onChange: e => { modal.text = e.target.value; edited(); } }),
|
||||
h("button", { title: "remove the points of this frame", onClick: () => { modal.points = []; edited(); } }, "Clear frame"),
|
||||
h("button", { class: "pri", disabled: !!modal.busy || !(keysOf(modal.region).length || modal.text),
|
||||
onClick: () => c.previewMask(modal.idx, { ...c.pointSpec(), ...(modal.region > 1 ? { max_objects: 1 } : {}),
|
||||
preview_color: regionColor(modal.region) }, true) }, modal.prev && modal.stale ? "👁 Segment again" : "👁 Segment"),
|
||||
]),
|
||||
modal.busy ? busyBar(c, modal.busy) : null,
|
||||
modal.prev ? h("div", { class: "mstrip" }, [...modal.prev.frames.map(f => h("img", { src: f.src, title: `frame ${f.f}` })),
|
||||
hint(modal.prev.empty ? "nothing found" : `covers ${(modal.prev.coverage * 100).toFixed(1)}%`)]) : null,
|
||||
modal.prev ? hint(modal.prev.empty ? "nothing found" : `covers ${(modal.prev.coverage * 100).toFixed(1)}% of the frames`
|
||||
+ (modal.stale ? " · the points changed: Segment again to see the new mask" : " · play or scrub to check every frame; where it slips, add points on that frame"),
|
||||
"display:block;margin-top:4px") : null,
|
||||
h("div", { class: "row", style: "margin-top:8px;justify-content:flex-end" }, [
|
||||
h("button", { onClick: () => { modal.open = false; } }, "Cancel"),
|
||||
!ms.color_guide ? h("button", { title: "Apply these points to EVERY shot, each on its frame at the same relative position (a subject that stays in place)",
|
||||
!ms.color_guide ? h("button", { title: "Apply this frame's points to EVERY shot, each on its frame at the same relative position (a subject that stays in place)",
|
||||
onClick: c.savePointsAll }, "Save → all shots") : null,
|
||||
h("button", { disabled: !modal.points.length || ms.color_guide,
|
||||
title: "Track this selection with SAM 3 through the whole video (or the range), forward and backward from this frame; every shot without its own selection takes its part of the track. Also type a short text (e.g. 'woman in pink top') so the subject is found again after camera cuts.",
|
||||
title: "Track this frame's selection with SAM 3 through the whole video (or the range), forward and backward from this frame; every shot without its own selection takes its part of the track. Also type a short text (e.g. 'woman in pink top') so the subject is found again after camera cuts.",
|
||||
onClick: c.saveTrack }, "🎯 Track → whole video"),
|
||||
h("button", { class: "pri", onClick: c.savePoints }, "Save (this shot)"),
|
||||
]),
|
||||
|
||||
@@ -162,7 +162,7 @@ function editor(c, cur) {
|
||||
h("button", { disabled: !!busy.value || !(r.text || (r.points || []).length),
|
||||
onClick: () => c.previewMask(i, { text: r.text || "", points: r.points || [], key: r.key || 0, max_objects: 1,
|
||||
preview_color: pictureColor(plan, r.picture) }) }, "👁 Preview"),
|
||||
h("button", { onClick: () => c.openPoints(i, j + 2) }, (r.points || []).length ? `🎯 Points (${r.points.length})` : "🎯 Points…"),
|
||||
h("button", { onClick: () => c.openPoints(i, j + 2) }, c.keyFramesOf(r).length ? `🎯 Points (${c.keyFramesOf(r).length} key frame${c.keyFramesOf(r).length > 1 ? "s" : ""})` : "🎯 Points…"),
|
||||
h("button", { class: "dng", title: "remove this person's mark", onClick: () => c.removeMarkRegion(i)(j) }, "✕"),
|
||||
])),
|
||||
h("div", { class: "row", style: "margin-top:6px;align-items:center;gap:8px" }, [
|
||||
@@ -180,7 +180,7 @@ function editor(c, cur) {
|
||||
onInput: e => c.setMask(i, "text", e.target.value),
|
||||
onKeydown: e => { if (e.key === "Enter") { e.preventDefault(); c.setMask(i, "text", e.target.value); if (e.target.value || (cur.mask.points || []).length) c.previewMask(i); } } }),
|
||||
h("button", { title: "Pick positive / negative points on a frame of this shot (more precise than text when several people look alike)",
|
||||
onClick: () => c.openPoints(i) }, (cur.mask.points || []).length ? `🎯 Points (${cur.mask.points.length})` : "🎯 Points…"),
|
||||
onClick: () => c.openPoints(i) }, c.keyFramesOf(cur.mask).length ? `🎯 Points (${c.keyFramesOf(cur.mask).length} key frame${c.keyFramesOf(cur.mask).length > 1 ? "s" : ""})` : "🎯 Points…"),
|
||||
h("button", { class: hasMask(cur) && !mp ? "pri" : "", disabled: !!busy.value || !hasMask(cur),
|
||||
title: (cur.mask.points || []).length ? "Segment this shot (the points win over the text)" : "Segment this shot with the text and show a few frames",
|
||||
onClick: () => c.previewMask(i) }, "👁 Preview"),
|
||||
|
||||
Reference in New Issue
Block a user