Shot Planner: points on several key frames of a shot, the mask over every frame (1.116.0)

- A selection (the shot's mask and each Colored Mark person) has key frames, each with its own points: SAM 3 selects the subject on a key frame and tracks it up to the next one (before the first, backward), so points added on a later frame correct the mask from there on. Plans with one key frame (points + key) work as before; keys are saved next to them. - The points window: after Segment the mask is shown over every frame of the shot, with a play button, frame stepping (arrows), key-frame marks on the slider and a key-frame list (jump / remove). Edited points mark the preview as old until Segment again. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-18 18:19:16 +00:00
7 changed files with 223 additions and 82 deletions
@@ -1002,10 +1002,19 @@ SAM3_URL = "https://huggingface.co/Comfy-Org/sam3.1/resolve/main/checkpoints/sam
 _SAM3: dict[str, Any] = {}
 _MASK_CACHE: dict[tuple, dict] = {}
 # global mask settings (the planner's Mask card); a shot only says what to segment (text / points / key frame)
-DEFAULT_MASK = {"text": "", "points": [], "key": 0, "threshold": 0.5, "max_objects": 4, "invert": False,
+DEFAULT_MASK = {"text": "", "points": [], "key": 0, "keys": [], "threshold": 0.5, "max_objects": 4, "invert": False,
                "fill_holes": True, "temporal_expand": 2, "blockify": 0, "padding": 0.15, "expand": 16,
                "feather": 12, "paste": "box"}
-SHOT_MASK_KEYS = ("text", "points", "key")
+SHOT_MASK_KEYS = ("text", "points", "key", "keys")
+
+
+def point_keys(spec: dict) -> list[tuple[int, list]]:
+    """The shot's key frames with their points, in frame order: spec['keys'] ([{key, points}], 1.116), else the single
+    key frame of older plans (points + key)."""
+    keys = {int(k.get("key") or 0): list(k.get("points") or []) for k in spec.get("keys") or [] if k.get("points")}
+    if not keys and spec.get("points"):
+        keys = {int(spec.get("key") or 0): list(spec["points"])}
+    return sorted(keys.items())


 def _sam3_path() -> str:
@@ -1074,7 +1083,7 @@ def plan_spec(p: dict, m: dict | None, path: str = "") -> dict:


 def spec_has_mask(spec: dict) -> bool:
-    return bool(spec.get("ext") or spec.get("points") or str(spec.get("text") or "").strip())
+    return bool(spec.get("ext") or point_keys(spec) or str(spec.get("text") or "").strip())


 def ext_masks(ext: dict, analysis: dict, start: int, length: int, w: int, h: int) -> torch.Tensor:
@@ -1207,28 +1216,34 @@ def _track(model, imgs: torch.Tensor, init: torch.Tensor | None, cond, spec: dic


 def segment_frames(imgs: torch.Tensor, spec: dict) -> torch.Tensor:
-    """[N,H,W,3] -> [N,H,W] masks with SAM 3: tracked from the points on the key frame, or from the text."""
+    """[N,H,W,3] -> [N,H,W] masks with SAM 3, from the text or from the points of the key frames: each key frame's
+    points select the subject there and it is tracked from that frame up to the next key frame (before the first one,
+    backward from it), so a later key frame corrects the mask from its frame on."""
    from comfy_extras.nodes_sam3 import SAM3_Detect
    model, clip = _sam3()
    text = str(spec.get("text") or "").strip()
    cond = clip.encode_from_tokens_scheduled(clip.tokenize(text)) if text else None
-    pts = spec.get("points") or []
    N, H, W = imgs.shape[:3]
-    if not pts:
+    keys = [(max(0, min(N - 1, k)), pts) for k, pts in point_keys(spec)]
+    if not keys:
        if cond is None:
            raise ValueError("the shot's mask needs a text prompt or points")
        return _track(model, imgs, None, cond, spec)
-    key = max(0, min(N - 1, int(spec.get("key") or 0)))
-    pos = [{"x": p["x"] * W, "y": p["y"] * H} for p in pts if p.get("label", 1)]
-    neg = [{"x": p["x"] * W, "y": p["y"] * H} for p in pts if not p.get("label", 1)]
-    first = SAM3_Detect.execute(model=model, image=imgs[key:key + 1], positive_coords=json.dumps(pos),
-                                negative_coords=json.dumps(neg), threshold=float(spec["threshold"]),
-                                refine_iterations=2).args[0][:1].float()
-    fwd = _track(model, imgs[key:], first, None, spec)                     # key frame -> end
-    if key == 0:
-        return fwd
-    back = _track(model, imgs[:key + 1].flip(0), first, None, spec).flip(0)  # key frame -> start
-    return torch.cat([back[:-1], fwd], 0)
+
+    def detect(k, pts):
+        pos = [{"x": p["x"] * W, "y": p["y"] * H} for p in pts if p.get("label", 1)]
+        neg = [{"x": p["x"] * W, "y": p["y"] * H} for p in pts if not p.get("label", 1)]
+        return SAM3_Detect.execute(model=model, image=imgs[k:k + 1], positive_coords=json.dumps(pos),
+                                   negative_coords=json.dumps(neg), threshold=float(spec["threshold"]),
+                                   refine_iterations=2).args[0][:1].float()
+    out = torch.zeros(N, H, W)
+    for i, (k, pts) in enumerate(keys):
+        seed = detect(k, pts)
+        if i == 0 and k > 0:                                                  # first key frame -> start
+            out[:k + 1] = _track(model, imgs[:k + 1].flip(0), seed, None, spec).flip(0)
+        end = keys[i + 1][0] if i + 1 < len(keys) else N                      # this key frame -> the next one
+        out[k:end] = _track(model, imgs[k:end], seed, None, spec)
+    return out


 def _mask_key(path: str, analysis: dict, start: int, length: int, spec: dict) -> tuple:
@@ -1244,7 +1259,7 @@ def _mask_key(path: str, analysis: dict, start: int, length: int, spec: dict) ->
    else:
        e = None
    return (path, os.path.getmtime(path), float(analysis["fps"]), int(start), int(length),
-            json.dumps({k: spec[k] for k in ("text", "points", "key", "threshold", "max_objects")}, sort_keys=True), e)
+            json.dumps({"keys": point_keys(spec), **{k: spec[k] for k in ("text", "threshold", "max_objects")}}, sort_keys=True), e)


 def shot_mask(path: str, analysis: dict, start: int, length: int, spec: dict) -> dict:
@@ -1268,8 +1283,8 @@ def shot_mask(path: str, analysis: dict, start: int, length: int, spec: dict) ->
        frames = _read_frames(path, src[idx], (w, h), stage="Reading the shot")
        imgs = torch.from_numpy(np.stack(frames).astype(np.float32) / 255.0)
        _status(f"Segmenting {len(frames)} frames with SAM 3", force=True)
-        what = f"{len(spec.get('points') or [])} points on frame {int(spec.get('key') or 0)}" if spec.get("points") \
-            else f"'{spec.get('text', '')}'"
+        keys = point_keys(spec)
+        what = (", ".join(f"{len(p)} points on frame {k}" for k, p in keys)) if keys else f"'{spec.get('text', '')}'"
        print(f"[BFS Shot] SAM 3: segmenting {what} on {len(frames)} frames ({w}x{h}, timeline {start}-{start + length - 1})…",
              flush=True)
        t0 = time.time()
@@ -1480,13 +1495,15 @@ def uncrop(result: torch.Tensor, shot: dict, edge_hardness: float = 0.0, match_c

 def mask_preview(path: str, analysis: dict, start: int, length: int, spec: dict, count: int = 6,
                 color: str = "#ff283c", opacity: float = 0.55) -> dict:
-    """A few frames of the shot with the selected region tinted and the crop box, as data URLs."""
+    """`count` frames of the shot (0 = every frame, smaller, for the points window's player) with the selected region
+    tinted and the crop box, as data URLs."""
    import cv2
    r = shot_mask(path, analysis, start, length, spec)
    masks, (w, h), box = r["masks"], r["size"], r["box"]
    src = _timeline(analysis["n_src"], analysis["fps_src"], float(analysis["fps"]))
-    picks = sorted(set(int(round(i)) for i in np.linspace(0, length - 1, min(count, length))))
+    picks = list(range(length)) if count <= 0 else sorted(set(int(round(i)) for i in np.linspace(0, length - 1, min(count, length))))
    frames = _read_frames(path, src[np.clip(np.array(picks) + start, 0, analysis["n"] - 1)], (w, h))
+    small = 384 / max(w, h) if count <= 0 and max(w, h) > 384 else 1.0   # every frame: ~10 KB each
    try:
        tint = np.array([int(color[i:i + 2], 16) for i in (1, 3, 5)], dtype=np.float32)
        alpha = min(0.8, max(0.05, float(opacity)))
@@ -1500,7 +1517,9 @@ def mask_preview(path: str, analysis: dict, start: int, length: int, spec: dict,
        if box:
            x0, y0, x1, y1 = box_px(box, w, h)
            cv2.rectangle(img, (x0, y0), (x1 - 1, y1 - 1), (255, 210, 90), 2)
-        ok, buf = cv2.imencode(".jpg", cv2.cvtColor(img, cv2.COLOR_RGB2BGR), [cv2.IMWRITE_JPEG_QUALITY, 80])
+        if small < 1.0:
+            img = cv2.resize(img, (int(w * small) // 2 * 2, int(h * small) // 2 * 2), interpolation=cv2.INTER_AREA)
+        ok, buf = cv2.imencode(".jpg", cv2.cvtColor(img, cv2.COLOR_RGB2BGR), [cv2.IMWRITE_JPEG_QUALITY, 80 if small == 1.0 else 70])
        out.append({"f": start + i, "src": "data:image/jpeg;base64," + base64.b64encode(buf.tobytes()).decode()})
    cover = float(masks.float().mean()) if masks.numel() else 0.0
    return {"box": box, "frames": out, "coverage": cover, "empty": box is None}
@@ -1519,7 +1538,8 @@ TARGET_Q = ("Describe the main subject in this picture (a person, an animal, a c
 def target_crop(path: str, analysis: dict, start: int, length: int, spec: dict, pad: float = 0.12):
    """The selected subject (the shot's SAM 3 points / text, on its key frame) cropped from that frame, RGB uint8."""
    src = _timeline(analysis["n_src"], analysis["fps_src"], float(analysis["fps"]))
-    key = max(0, min(length - 1, int(spec.get("key") or length // 2)))
+    keys = point_keys(spec)
+    key = max(0, min(length - 1, keys[0][0] if keys else int(spec.get("key") or length // 2)))
    fr = _read_frames(path, src[np.clip(np.array([start + key]), 0, analysis["n"] - 1)], None)[0]
    if spec.get("ext"):
        sm = shot_mask(path, analysis, start, length, spec)["masks"]
@@ -3965,8 +3985,8 @@ def color_marks(plan: dict, m: dict) -> list[dict]:
                   "key": m.get("color_guide_key2")}] if m.get("color_guide_text2") or m.get("color_guide_points2") else [])
    marks = [{"picture": max(1, int(m.get("color_guide_picture") or 1))}]
    marks += [{"picture": max(1, int(r.get("picture") or 2)), "text": str(r.get("text") or "").strip(),
-               "points": list(r.get("points") or []), "key": int(r.get("key") or 0)}
-              for r in extra if isinstance(r, dict) and (str(r.get("text") or "").strip() or r.get("points"))]
+               "points": list(r.get("points") or []), "key": int(r.get("key") or 0), "keys": list(r.get("keys") or [])}
+              for r in extra if isinstance(r, dict) and (str(r.get("text") or "").strip() or r.get("points") or r.get("keys"))]
    for x in marks:
        x["color"] = picture_color(plan, x["picture"])
    return marks
@@ -3985,6 +4005,7 @@ def extra_mark_masks(shot: dict, log: str = "") -> list[tuple]:
    out = []
    for x in extra:
        spec = dict(DEFAULT_MASK, text=x.get("text") or "", points=x.get("points") or [], key=x.get("key") or 0,
+                    keys=x.get("keys") or [],
                    max_objects=1)
        out.append((shot_mask(ms["path"], ms["analysis"], ms["start"], ms["length"], spec)["masks"], x))
    return out
@@ -1,7 +1,7 @@
 [project]
 name = "bfsnodes"
 description = "Custom ComfyUI nodes for reference-guided video: BFS Sampler (guide-driven IC-LoRAs, auto crop, mask inpainting, temporal chunking), LTX Multiple Controls / Identity Transfer / Edit Anything, Head-Swap and Anime2Real conditioning, MiniMax-H3 latent guides, RoPE layouts and loaders, LoRA Surgery, and Shot Planner / Shot Loop (tabbed panel): split long videos into shots at camera cuts, face-tracked cast, SAM 3 mask per shot with full frame, paste, mask, crop and crop + mask generation modes, optional translucent Colored Mark overlay for manually selected regions, optional Qwen3-VL to suggest settings, describe references ({details}) and the person to replace ({target}), continuity between shots, H3 conditioning with an optional duet and prompt writer, join with the original timing, audio and a side-by-side comparison. H3 Duet: pin a source clip beside the video, training-free."
-version = "1.115.1"
+version = "1.116.0"
 license = {file = "LICENSE"}
 dependencies = [
    "torch",
@@ -1088,6 +1088,52 @@ class MultiRefTest(unittest.TestCase):
        self.assertNotIn("ref_image", SL.BFSShotPlanner.INPUT_TYPES()["optional"])


+class KeyFrameSegmentTest(unittest.TestCase):
+    """Points on several frames of a shot: each key frame starts the track up to the next one."""
+
+    def setUp(self):
+        import json as _json
+
+        class Detect:   # the mask of a key frame carries its first point's x (in pixels), to see who filled a frame
+            @staticmethod
+            def execute(model, image, positive_coords, **kw):
+                return types.SimpleNamespace(args=(torch.full((1, 4, 4), _json.loads(positive_coords)[0]["x"]),))
+        mod = types.ModuleType("comfy_extras.nodes_sam3"); mod.SAM3_Detect = Detect
+        sys.modules.setdefault("comfy_extras", types.ModuleType("comfy_extras"))
+        self._old = (sys.modules.get("comfy_extras.nodes_sam3"), SL._sam3, SL._track)
+        sys.modules["comfy_extras.nodes_sam3"] = mod
+        SL._sam3 = lambda: (None, None)
+        self.tracks = []
+
+        def track(model, imgs, seed, cond, spec):
+            self.tracks.append((imgs.shape[0], float(seed.flatten()[0])))
+            return seed.expand(imgs.shape[0], -1, -1).clone()
+        SL._track = track
+
+    def tearDown(self):
+        mod, SL._sam3, SL._track = self._old
+        if mod is None:
+            sys.modules.pop("comfy_extras.nodes_sam3", None)
+        else:
+            sys.modules["comfy_extras.nodes_sam3"] = mod
+
+    def test_each_key_frame_corrects_the_mask_from_its_frame_on(self):
+        pt = lambda x: [{"x": x, "y": 0.5, "label": 1}]
+        spec = dict(SL.DEFAULT_MASK, keys=[{"key": 6, "points": pt(0.75)}, {"key": 2, "points": pt(0.25)}])
+        out = SL.segment_frames(torch.zeros(10, 4, 4, 3), spec)
+        self.assertEqual([float(out[f, 0, 0]) for f in range(10)], [1.0] * 6 + [3.0] * 4)   # frames 0-5 from key 2, 6-9 from key 6
+        self.assertEqual(self.tracks, [(3, 1.0), (4, 1.0), (4, 3.0)])     # back 2->0, forward 2->5, forward 6->9
+        self.assertTrue(SL.spec_has_mask(dict(SL.DEFAULT_MASK, keys=[{"key": 3, "points": pt(0.5)}])))
+        self.assertFalse(SL.spec_has_mask(dict(SL.DEFAULT_MASK, keys=[{"key": 3, "points": []}])))
+
+    def test_a_plan_with_one_key_frame_works_as_before(self):
+        spec = dict(SL.DEFAULT_MASK, points=[{"x": 0.5, "y": 0.5, "label": 1}], key=4)
+        self.assertEqual(SL.point_keys(spec), [(4, spec["points"])])
+        out = SL.segment_frames(torch.zeros(8, 4, 4, 3), spec)
+        self.assertEqual(self.tracks, [(5, 2.0), (4, 2.0)])
+        self.assertEqual(float(out.sum()), 2.0 * 8 * 16)
+
+
 class DescribeShortTest(unittest.TestCase):
    def test_short_details_fit_inside_the_prompt(self):
        self.assertEqual(SL.tidy_details('"The green goblin in a torn leather vest."\nextra', {"describe_preset": "short"}),
@@ -70,7 +70,8 @@ function Panel(io) {
  const maskPrev = ref({});      // "start-end" -> {frames, box, coverage, empty} from /bfs/shotloop/mask
  const showMasks = ref(true);   // overlay the mask preview on the shot cards
  const vlmSug = ref({});        // "start-end" -> {segment, shot, people, recommend, reason} from the VLM
-  const modal = reactive({ open: false, idx: -1, key: 0, points: [], text: "", region: 1, groups: {}, prev: null, busy: "" });
+  const modal = reactive({ open: false, idx: -1, key: 0, points: [], text: "", region: 1, groups: {}, prev: null, stale: false,
+                           showMask: true, playing: 0, busy: "" });
  const segPeople = ref({});     // "start-end" -> {people, main} from /bfs/shotloop/plan
  const targetCrop = ref({});    // "start-end" -> data URL of the subject the description was made from
  const tab = ref(store.get("bfs.shotloop.tab", "shots"));
@@ -352,15 +353,17 @@ const meta = i => { while (plan.segs.length <= i) plan.segs.push({}); return pla
    } catch (e) { error.value = String(e.message || e); }
    busy.value = "";
  }
+  // intoModal: every frame of the shot, for the points window's player (count 0)
  async function previewMask(i, spec, intoModal = false) {
    const s = segs.value[i]; if (!s) return;
    const label = "Segmenting with SAM 3… (the first time downloads/loads the model)";
    if (intoModal) { modal.busy = label; busySince.value = Date.now(); Object.assign(status, { stage: "", done: 0, total: 0 }); } else busy.value = label;
    error.value = "";
    try {
-      const r = await api.fetchApi("/bfs/shotloop/mask", { method: "POST", body: JSON.stringify({ plan: { ...plan }, index: i, mask: spec || null }) });
+      const body = { plan: { ...plan }, index: i, mask: spec || null, ...(intoModal ? { count: 0 } : {}) };
+      const r = await api.fetchApi("/bfs/shotloop/mask", { method: "POST", body: JSON.stringify(body) });
      const j = await r.json(); if (j.error) throw new Error(j.error);
-      if (intoModal) modal.prev = j; else maskPrev.value = { ...maskPrev.value, [segKey(s)]: j };
+      if (intoModal) { modal.prev = j; modal.stale = false; modal.showMask = true; } else maskPrev.value = { ...maskPrev.value, [segKey(s)]: j };
    } catch (e) { error.value = String(e.message || e); }
    if (intoModal) modal.busy = ""; else busy.value = "";
  }
@@ -371,29 +374,52 @@ const meta = i => { while (plan.segs.length <= i) plan.segs.push({}); return pla
    if (s && maskPrev.value[segKey(s)]) { const mp = { ...maskPrev.value }; delete mp[segKey(s)]; maskPrev.value = mp; }
  };
  const setMaskCfg = (k, v) => { plan.mask_cfg = { ...plan.mask_cfg, [k]: v }; save(); maskPrev.value = {}; };
-  // Colored Mark regions in the points window: 1 = the shot's own selection (its mask), 2.. = its extra regions
+  // The points window. A selection (region 1 = the shot's own mask, 2.. = its extra Colored Mark regions) has key
+  // frames, each with its own points: SAM 3 starts from a key frame and tracks up to the next one, so points added on a
+  // later frame correct the mask from there on. modal.key is the frame shown, modal.points its points (being edited).
+  const keyFramesOf = m => (Array.isArray(m?.keys) && m.keys.length ? m.keys.map(k => ({ key: +k.key || 0, points: [...(k.points || [])] }))
+    : (m?.points || []).length ? [{ key: +m.key || 0, points: [...m.points] }] : []);
+  const pointGroup = () => modal.groups[modal.region];
+  const keepFrame = () => {     // the frame being edited back into its selection's key frames
+    const g = pointGroup(); if (!g) return;
+    g.keys = g.keys.filter(k => k.key !== modal.key);
+    if (modal.points.length) g.keys = [...g.keys, { key: modal.key, points: [...modal.points] }].sort((a, b) => a.key - b.key);
+    g.text = modal.text;
+  };
+  const showFrame = f => {
+    keepFrame();
+    modal.key = Math.max(0, Math.min((segs.value[modal.idx]?.len || 1) - 1, f));
+    modal.points = [...(pointGroup()?.keys.find(k => k.key === modal.key)?.points || [])];
+  };
+  const dropKeyFrame = f => {
+    const g = pointGroup(); g.keys = g.keys.filter(k => k.key !== f);
+    if (f === modal.key) modal.points = [];
+    modal.stale = !!modal.prev;
+  };
  const openPoints = (i, region = 1) => {
    const s = segs.value[i]; if (!s) return;
-    const g1 = { key: s.mask.key ?? Math.floor(s.len / 2), points: [...(s.mask.points || [])], text: s.mask.text || "" };
-    const groups = { 1: g1 };
-    if (s.color_guide) s.regions.forEach((r, j) => { groups[j + 2] = { key: r.key ?? g1.key, points: [...(r.points || [])], text: r.text || "" }; });
-    const g = groups[region] || g1;
-    Object.assign(modal, { open: true, idx: i, region: groups[region] ? region : 1, groups, key: g.key, points: [...g.points],
-      text: g.text, prev: null, busy: "" });
+    const groups = { 1: { keys: keyFramesOf(s.mask), text: s.mask.text || "" } };
+    if (s.color_guide) s.regions.forEach((r, j) => { groups[j + 2] = { keys: keyFramesOf(r), text: r.text || "" }; });
+    const r0 = groups[region] ? region : 1, key = groups[r0].keys[0]?.key ?? Math.floor(s.len / 2);
+    Object.assign(modal, { open: true, idx: i, region: r0, groups, key, points: [...(groups[r0].keys[0]?.points || [])],
+      text: groups[r0].text, prev: null, stale: false, showMask: true, busy: "" });
  };
  const switchPointRegion = region => {
-    modal.groups[modal.region] = { key: modal.key, points: [...modal.points], text: modal.text };
-    const next = modal.groups[region] || { key: modal.key, points: [], text: "" };
-    modal.region = region; modal.points = [...next.points]; modal.text = next.text; modal.prev = null;
+    keepFrame();
+    modal.region = region; modal.text = pointGroup()?.text || "";
+    modal.points = [...(pointGroup()?.keys.find(k => k.key === modal.key)?.points || [])]; modal.prev = null;
  };
+  // the selection being edited as a mask spec (what Segment sends)
+  const pointSpec = () => { keepFrame(); const g = pointGroup(); return { keys: g.keys, points: g.keys[0]?.points || [], key: g.keys[0]?.key ?? modal.key, text: g.text }; };
  const commitPointGroups = i => {
-    modal.groups[modal.region] = { key: modal.key, points: [...modal.points], text: modal.text };
+    keepFrame();
    const m = meta(i), g1 = modal.groups[1];
-    m.mask = { ...(m.mask || {}), points: [...g1.points], key: g1.key, text: g1.text };
+    // keys is what 1.116 reads; points + key (the first key frame) for older versions
+    m.mask = { ...(m.mask || {}), keys: g1.keys, points: g1.keys[0]?.points || [], key: g1.keys[0]?.key ?? 0, text: g1.text };
    if (segs.value[i]?.color_guide) {
      m.color_guide_regions = markRegions(m).map((r, j) => {
        const g = modal.groups[j + 2];
-        return g ? { ...r, points: [...g.points], key: g.key, text: g.text } : r;
+        return g ? { ...r, keys: g.keys, points: g.keys[0]?.points || [], key: g.keys[0]?.key ?? 0, text: g.text } : r;
      });
    }
  };
@@ -421,7 +447,11 @@ const meta = i => { while (plan.segs.length <= i) plan.segs.push({}); return pla
  };
  const savePoints = () => {
    const i = modal.idx; commitPointGroups(i); save();
-    if (modal.prev) maskPrev.value = { ...maskPrev.value, [segKey(segs.value[modal.idx])]: modal.prev };
+    if (modal.prev && !modal.stale) {   // the card's strip shows 6 of the window's frames
+      const fr = modal.prev.frames, pick = [...new Set(Array.from({ length: Math.min(6, fr.length) }, (_, k) => Math.round(k * (fr.length - 1) / Math.max(1, Math.min(6, fr.length) - 1))))];
+      maskPrev.value = { ...maskPrev.value, [segKey(segs.value[modal.idx])]: { ...modal.prev, frames: pick.map(k => fr[k]) } };
+    }
+    clearInterval(modal.playing); modal.playing = 0;
    modal.open = false;
  };
  const targets = (scope, from) => segs.value.map((_, i) => i).filter(i => i !== from && (scope === "all" || i > from));
@@ -431,7 +461,8 @@ const meta = i => { while (plan.segs.length <= i) plan.segs.push({}); return pla
    const src = segs.value[fromIdx]; const rel = src ? key / Math.max(1, src.len - 1) : 0.5;
    [fromIdx, ...targets(scope, fromIdx)].forEach(i => {
      const x = segs.value[i], m = meta(i);
-      m.mask = { ...(m.mask || {}), points: points.map(p => ({ ...p })), key: Math.round(rel * Math.max(0, x.len - 1)), text: text ?? (m.mask || {}).text };
+      const key = Math.round(rel * Math.max(0, x.len - 1));
+      m.mask = { ...(m.mask || {}), keys: [{ key, points: points.map(p => ({ ...p })) }], points: points.map(p => ({ ...p })), key, text: text ?? (m.mask || {}).text };
    });
    maskPrev.value = {}; save();
  };
@@ -444,13 +475,13 @@ const meta = i => { while (plan.segs.length <= i) plan.segs.push({}); return pla
  // part of that track, so the subject is found wherever it moves (copied points can fall on the background)
  const saveTrack = () => {
    const s0 = segs.value[modal.idx]; if (!s0 || !modal.points.length) return;
-    const own = segs.value.filter((x, i) => i !== modal.idx && (x.mask.text || (x.mask.points || []).length)).length;
+    const own = segs.value.filter((x, i) => i !== modal.idx && (x.mask.text || (x.mask.points || []).length || (x.mask.keys || []).length)).length;
    if (own && !window.confirm(`${own} other shot(s) have their own mask text or points, which would win over the track. Clear them so every shot uses the track?`)) {
      // keep them: those shots keep their own selection
    } else {
-      segs.value.forEach((_, i) => { const m = meta(i); if (m.mask) { m.mask = { ...m.mask, points: [], text: "" }; } });
+      segs.value.forEach((_, i) => { const m = meta(i); if (m.mask) { m.mask = { ...m.mask, points: [], keys: [], text: "" }; } });
    }
-    const m = meta(modal.idx); m.mask = { ...(m.mask || {}), points: [], text: "" };
+    const m = meta(modal.idx); m.mask = { ...(m.mask || {}), points: [], keys: [], text: "" };
    plan.track = { points: modal.points.map(p => ({ ...p })), frame: s0.start + modal.key, text: modal.text || "",
                   threshold: plan.mask_cfg?.threshold ?? 0.5 };
    maskPrev.value = {}; save(); modal.open = false;
@@ -873,7 +904,7 @@ const meta = i => { while (plan.segs.length <= i) plan.segs.push({}); return pla
    refreshFiles, analyze, autoSplit, analyzeContent, findPeople, setCast, describeTarget, previewMask, setMask, setMaskCfg,
    openPoints, switchPointRegion, savePoints, savePointsAll, saveTrack, clearTrack, pointsTo, clearMask, copyFrom, analyseVLM, setVlmCfg,
    applySug, applyAllSug, setCastOpt, setGlobalMaskVideo, setFilter, pickFile, progress, setVideo, splitAt, mergeNext, removeCut, select, drag,
-    frameAt, thumbFor, playShot, playAll, stop, seek, setRef, putRefs, uploadRef, dropRefs, pictures, describePictures, setPictureDetail, setPictureCfg,
+    frameAt, thumbFor, playShot, playAll, stop, seek, setRef, putRefs, uploadRef, dropRefs, showFrame, dropKeyFrame, pointSpec, keyFramesOf, pictures, describePictures, setPictureDetail, setPictureCfg,
    setMarkPicture, addMarkRegion, setMarkRegion, removeMarkRegion, setPictureColor,
    autoFor, writtenOf, setAuto, setWritten, writePrompts,
  };
@@ -209,6 +209,11 @@ const CSS = `
 .bsl .pt.pos{background:#3ccf6b}.bsl .pt.neg{background:#e8455a}
 .bsl .pt.cg.pos{background:var(--point-color,#00ff00)}
 .bsl .pt.other{opacity:.5}
+.bsl .kslider{position:relative;flex:1;min-width:160px;padding-bottom:8px}
+.bsl .kmarks{position:absolute;left:8px;right:8px;bottom:0;height:8px}
+.bsl .kmark{position:absolute;width:8px;height:8px;margin-left:-4px;border-radius:50%;background:var(--mark,#3ccf6b);opacity:.55;cursor:pointer;border:1px solid #000}
+.bsl .kmark.on{opacity:1}
+.bsl .kchip{display:inline-flex;gap:1px}.bsl .kchip.on button:first-child{border-color:var(--acc)}
 .bsl .mreg{display:flex;gap:6px;align-items:center;flex-wrap:wrap;margin-top:6px}
 `;

@@ -15,68 +15,106 @@ export function busyBar(c, msg) {
 }

 export function pointsModal(c) {
-  const { modal, segs, plan, api, setMeta } = c;
+  const { modal, segs, plan, api } = c;
  const ms = segs.value[modal.idx];
  if (!modal.open || !ms) return null;
  // Colored Mark: region 1 = the shot's own selection, 2.. = its extra regions; each one is a picture, in its color
  const regionPicture = r => (r === 1 ? ms.color_guide_picture : ms.regions[r - 2]?.picture) || 1;
-  const regionColor = r => pictureColor(plan, regionPicture(r));
+  const regionColor = r => (ms.color_guide ? pictureColor(plan, regionPicture(r)) : "#3ccf6b");
  const regionsN = ms.color_guide ? 1 + ms.regions.length : 1;
-  const pointOverlays = ms.color_guide
-    ? Array.from({ length: regionsN }, (_, q) => q + 1).flatMap(region => (region === modal.region ? modal.points : modal.groups[region]?.points || []).map((p, k) => h("div", {
-      class: ["pt", "cg", region !== modal.region && "other", p.label ? "pos" : "neg"],
-      style: `left:${p.x * 100}%;top:${p.y * 100}%;--point-color:${regionColor(region)}`,
-      title: `Region ${region} · Picture ${regionPicture(region)}${region === modal.region ? " · click to remove" : ""}`,
-      onPointerdown: e => { if (region !== modal.region) return; e.stopPropagation(); modal.points = modal.points.filter((_, j) => j !== k); modal.prev = null; },
-    })))
-    : modal.points.map((p, k) => h("div", { class: ["pt", p.label ? "pos" : "neg"], style: `left:${p.x * 100}%;top:${p.y * 100}%`,
-      title: "click to remove", onPointerdown: e => { e.stopPropagation(); modal.points = modal.points.filter((_, j) => j !== k); modal.prev = null; } }));
+  const regions = Array.from({ length: regionsN }, (_, q) => q + 1);
+  // the points of a region on the frame shown (the one being edited are in modal.points)
+  const pointsAt = r => (r === modal.region ? modal.points : modal.groups[r]?.keys.find(k => k.key === modal.key)?.points || []);
+  const keysOf = r => {
+    const ks = new Set((modal.groups[r]?.keys || []).map(k => k.key));
+    if (r === modal.region) { if (modal.points.length) ks.add(modal.key); else ks.delete(modal.key); }
+    return [...ks].sort((a, b) => a - b);
+  };
+  const edited = () => { modal.stale = !!modal.prev; };
+  const pointOverlays = regions.flatMap(r => pointsAt(r).map((p, k) => h("div", {
+    class: ["pt", ms.color_guide && "cg", r !== modal.region && "other", p.label ? "pos" : "neg"],
+    style: `left:${p.x * 100}%;top:${p.y * 100}%;--point-color:${regionColor(r)}`,
+    title: r === modal.region ? "click to remove" : `Picture ${regionPicture(r)}`,
+    onPointerdown: e => { if (r !== modal.region) return; e.stopPropagation(); modal.points = modal.points.filter((_, j) => j !== k); edited(); },
+  })));
  const regionButton = r => {
    const k = regionPicture(r), name = ms.names[k - 1];
    return h("button", { class: modal.region === r ? "pri" : "", onClick: () => c.switchPointRegion(r),
      title: `Region ${r}: becomes <Picture ${k}> (${markName(regionColor(r))})` }, [
      name ? h("img", { class: "mthumb", src: c.viewUrl(name), style: `--mark:${regionColor(r)}` }) : null,
-      ` Picture ${k} · ${markName(regionColor(r))} (${(r === modal.region ? modal.points : modal.groups[r]?.points || []).length})`]);
+      ` Picture ${k} · ${markName(regionColor(r))} (${keysOf(r).length} key frame${keysOf(r).length === 1 ? "" : "s"})`]);
  };
+  // after Segment: the mask over every frame of the shot (modal.prev.frames, one per frame)
+  const masked = modal.prev && modal.showMask ? modal.prev.frames.find(f => f.f === ms.start + modal.key) : null;
+  const play = () => {
+    if (modal.playing) { clearInterval(modal.playing); modal.playing = 0; return; }
+    modal.playing = setInterval(() => {
+      if (modal.key >= ms.len - 1 || !modal.open) { clearInterval(modal.playing); modal.playing = 0; return; }
+      c.showFrame(modal.key + 1);
+    }, 1000 / Math.min(24, plan.fps || 24));
+  };
+  const marks = h("div", { class: "kmarks" }, regions.flatMap(r => keysOf(r).map(f => h("span", {
+    class: ["kmark", r === modal.region && "on"], style: `left:${(f / Math.max(1, ms.len - 1)) * 100}%;--mark:${regionColor(r)}`,
+    title: `key frame ${ms.start + f}`, onClick: () => c.showFrame(f) }))));
  return h(Teleport, { to: "body" }, h("div", { class: "bsl", style: "background:none;padding:0" }, h("div", { class: "mmodal",
-    onKeydown: e => e.stopPropagation(), onPointerdown: e => { if (e.target === e.currentTarget) modal.open = false; } }, [
-    h("div", { class: "mbox" }, [
+    onKeydown: e => {
+      e.stopPropagation();
+      if (e.target.tagName === "INPUT" && e.target.type === "text") return;
+      if (e.key === "ArrowLeft") c.showFrame(modal.key - 1);
+      if (e.key === "ArrowRight") c.showFrame(modal.key + 1);
+    }, onPointerdown: e => { if (e.target === e.currentTarget) modal.open = false; } }, [
+    h("div", { class: "mbox", tabindex: 0 }, [
      h("div", { class: "row", style: "margin-bottom:6px" }, [h("span", { class: "sect" }, `Shot #${modal.idx + 1} · points`),
-        ms.color_guide ? h("div", { class: "row" }, [hint("One point set per marked person; switch to edit each. All use the frame selected below."),
-          ...Array.from({ length: regionsN }, (_, q) => regionButton(q + 1))]) : null,
-        hint("click = keep · right-click or shift+click = exclude · click a point to remove it")]),
+        ms.color_guide ? h("div", { class: "row" }, regions.map(regionButton)) : null,
+        hint("click = keep · right-click or shift+click = exclude · click a point to remove it · points on another frame = a new key frame: the mask is tracked from each key frame up to the next one")]),
      h("div", { class: "mstage" }, [
      h("div", { class: "mimg", onContextmenu: e => e.preventDefault(), onPointerdown: e => {
        const box = e.currentTarget.getBoundingClientRect();
        const x = (e.clientX - box.left) / box.width, y = (e.clientY - box.top) / box.height;
        modal.points = [...modal.points, { x: +x.toFixed(4), y: +y.toFixed(4), label: (e.button === 2 || e.shiftKey) ? 0 : 1 }];
-        modal.prev = null;
+        edited();
      } }, [
-        h("img", { draggable: false, src: api.apiURL(`/bfs/shotloop/frame?video=${encodeURIComponent(plan.video)}&fps=${plan.fps}&f=${ms.start + modal.key}&w=900`) }),
+        h("img", { draggable: false, src: masked ? masked.src : api.apiURL(`/bfs/shotloop/frame?video=${encodeURIComponent(plan.video)}&fps=${plan.fps}&f=${ms.start + modal.key}&w=900`) }),
        ...pointOverlays,
      ])]),
-      h("div", { class: "row", style: "margin-top:6px" }, [
+      h("div", { class: "row", style: "margin-top:6px;align-items:center" }, [
+        h("button", { title: "play the shot (with the mask after Segment)", onClick: play }, modal.playing ? "⏸" : "▶"),
+        h("button", { title: "previous frame (←)", onClick: () => c.showFrame(modal.key - 1) }, "◀"),
+        h("button", { title: "next frame (→)", onClick: () => c.showFrame(modal.key + 1) }, "▶|"),
        hint(`frame ${ms.start + modal.key} (${modal.key + 1}/${ms.len})`),
-        h("input", { type: "range", min: 0, max: ms.len - 1, step: 1, value: modal.key, style: "flex:1",
-          onInput: e => { modal.key = parseInt(e.target.value); modal.prev = null; } }),
+        h("div", { class: "kslider" }, [
+          h("input", { type: "range", min: 0, max: ms.len - 1, step: 1, value: modal.key, style: "width:100%",
+            onInput: e => c.showFrame(parseInt(e.target.value)) }),
+          marks,
+        ]),
+        modal.prev ? h("label", { class: "row", style: "gap:4px" }, [h("input", { type: "checkbox", checked: modal.showMask,
+          onChange: e => { modal.showMask = e.target.checked; } }), "mask"]) : null,
+      ]),
+      h("div", { class: "row", style: "margin-top:4px;gap:4px;flex-wrap:wrap" }, [
+        hint("key frames:"),
+        ...(keysOf(modal.region).length ? keysOf(modal.region).map(f => h("span", { class: ["kchip", f === modal.key && "on"] }, [
+          h("button", { class: "sm", onClick: () => c.showFrame(f) }, `frame ${ms.start + f}`),
+          h("button", { class: "sm dng", title: "remove this key frame's points", onClick: () => { c.dropKeyFrame(f); edited(); } }, "✕")]))
+          : [hint("none yet: click on the frame to add points")]),
      ]),
      h("div", { class: "row", style: "margin-top:6px" }, [
        h("input", { type: "text", value: modal.text, style: "flex:1", placeholder: "text prompt (used when there are no points)",
-          onChange: e => { modal.text = e.target.value; } }),
-        h("button", { onClick: () => { modal.points = []; modal.prev = null; } }, "Clear points"),
-        h("button", { class: "pri", disabled: !!modal.busy || !(modal.points.length || modal.text),
-          onClick: () => c.previewMask(modal.idx, { points: modal.points, key: modal.key, text: modal.text,
-            preview_color: ms.color_guide ? regionColor(modal.region) : undefined }, true) }, "👁 Segment"),
+          onChange: e => { modal.text = e.target.value; edited(); } }),
+        h("button", { title: "remove the points of this frame", onClick: () => { modal.points = []; edited(); } }, "Clear frame"),
+        h("button", { class: "pri", disabled: !!modal.busy || !(keysOf(modal.region).length || modal.text),
+          onClick: () => c.previewMask(modal.idx, { ...c.pointSpec(), ...(modal.region > 1 ? { max_objects: 1 } : {}),
+            preview_color: regionColor(modal.region) }, true) }, modal.prev && modal.stale ? "👁 Segment again" : "👁 Segment"),
      ]),
      modal.busy ? busyBar(c, modal.busy) : null,
-      modal.prev ? h("div", { class: "mstrip" }, [...modal.prev.frames.map(f => h("img", { src: f.src, title: `frame ${f.f}` })),
-        hint(modal.prev.empty ? "nothing found" : `covers ${(modal.prev.coverage * 100).toFixed(1)}%`)]) : null,
+      modal.prev ? hint(modal.prev.empty ? "nothing found" : `covers ${(modal.prev.coverage * 100).toFixed(1)}% of the frames`
+        + (modal.stale ? " · the points changed: Segment again to see the new mask" : " · play or scrub to check every frame; where it slips, add points on that frame"),
+        "display:block;margin-top:4px") : null,
      h("div", { class: "row", style: "margin-top:8px;justify-content:flex-end" }, [
        h("button", { onClick: () => { modal.open = false; } }, "Cancel"),
-        !ms.color_guide ? h("button", { title: "Apply these points to EVERY shot, each on its frame at the same relative position (a subject that stays in place)",
+        !ms.color_guide ? h("button", { title: "Apply this frame's points to EVERY shot, each on its frame at the same relative position (a subject that stays in place)",
          onClick: c.savePointsAll }, "Save → all shots") : null,
        h("button", { disabled: !modal.points.length || ms.color_guide,
-          title: "Track this selection with SAM 3 through the whole video (or the range), forward and backward from this frame; every shot without its own selection takes its part of the track. Also type a short text (e.g. 'woman in pink top') so the subject is found again after camera cuts.",
+          title: "Track this frame's selection with SAM 3 through the whole video (or the range), forward and backward from this frame; every shot without its own selection takes its part of the track. Also type a short text (e.g. 'woman in pink top') so the subject is found again after camera cuts.",
          onClick: c.saveTrack }, "🎯 Track → whole video"),
        h("button", { class: "pri", onClick: c.savePoints }, "Save (this shot)"),
      ]),
@@ -162,7 +162,7 @@ function editor(c, cur) {
        h("button", { disabled: !!busy.value || !(r.text || (r.points || []).length),
          onClick: () => c.previewMask(i, { text: r.text || "", points: r.points || [], key: r.key || 0, max_objects: 1,
            preview_color: pictureColor(plan, r.picture) }) }, "👁 Preview"),
-        h("button", { onClick: () => c.openPoints(i, j + 2) }, (r.points || []).length ? `🎯 Points (${r.points.length})` : "🎯 Points…"),
+        h("button", { onClick: () => c.openPoints(i, j + 2) }, c.keyFramesOf(r).length ? `🎯 Points (${c.keyFramesOf(r).length} key frame${c.keyFramesOf(r).length > 1 ? "s" : ""})` : "🎯 Points…"),
        h("button", { class: "dng", title: "remove this person's mark", onClick: () => c.removeMarkRegion(i)(j) }, "✕"),
      ])),
      h("div", { class: "row", style: "margin-top:6px;align-items:center;gap:8px" }, [
@@ -180,7 +180,7 @@ function editor(c, cur) {
        onInput: e => c.setMask(i, "text", e.target.value),
        onKeydown: e => { if (e.key === "Enter") { e.preventDefault(); c.setMask(i, "text", e.target.value); if (e.target.value || (cur.mask.points || []).length) c.previewMask(i); } } }),
      h("button", { title: "Pick positive / negative points on a frame of this shot (more precise than text when several people look alike)",
-        onClick: () => c.openPoints(i) }, (cur.mask.points || []).length ? `🎯 Points (${cur.mask.points.length})` : "🎯 Points…"),
+        onClick: () => c.openPoints(i) }, c.keyFramesOf(cur.mask).length ? `🎯 Points (${c.keyFramesOf(cur.mask).length} key frame${c.keyFramesOf(cur.mask).length > 1 ? "s" : ""})` : "🎯 Points…"),
      h("button", { class: hasMask(cur) && !mp ? "pri" : "", disabled: !!busy.value || !hasMask(cur),
        title: (cur.mask.points || []).length ? "Segment this shot (the points win over the text)" : "Segment this shot with the text and show a few frames",
        onClick: () => c.previewMask(i) }, "👁 Preview"),