Compare commits
16
Commits
+53
-29
@@ -713,7 +713,11 @@ if _io is not None:
|
||||
_io.Autogrow.Input("ref_audios", optional=True, template=_io.Autogrow.TemplatePrefix(
|
||||
input=_io.Audio.Input("ref_audio", tooltip="<Audio n> reference"), prefix="ref_audio_", min=0, max=3)),
|
||||
_io.Combo.Input("source_blur", options=_blur_modes(), default="off", optional=True, tooltip=
|
||||
"Blur the source person in what the model sees (the panel / the guide), keeping the mouth sharp (SAM 3.1): it reads the pose, motion and lips but has no face to copy. The output is never blurred. Works with or instead of panel_static."),
|
||||
BLUR_TT + " Works with or instead of panel_static."),
|
||||
_io.Combo.Input("source_blur_segmenter", options=_blur_lists().SEGMENTERS, default="SAM 3.1",
|
||||
optional=True, tooltip=SEGMENTER_TT),
|
||||
_io.Combo.Input("source_blur_effect", options=_blur_lists().EFFECTS, default="blur", optional=True,
|
||||
tooltip=EFFECT_TT),
|
||||
_io.Combo.Input("panel_static", options=PANEL_STATIC, default="off", optional=True, tooltip=
|
||||
"TSC's fix for swaps in the same world: cover the face (or the whole person) of the pinned clip in TV "
|
||||
"static, every frame (SAM 3), so the generated half cannot copy it and takes the identity from the "
|
||||
@@ -739,12 +743,13 @@ if _io is not None:
|
||||
panel_noise, hold, rope_mode, rope_gap, ref_image_size, steps, sampler_name, scheduler, seed,
|
||||
decode_canvas, audio_vae=None, panel=None, guide=None, guide_frame_idx=0, ref_images=None,
|
||||
ref_videos=None, ref_video_audios=None, ref_audios=None, panel_static="off", setting=None, vlm=None, source_blur="off",
|
||||
vlm_max_tokens=1024):
|
||||
vlm_max_tokens=1024, source_blur_segmenter="SAM 3.1", source_blur_effect="blur"):
|
||||
if panel is None and guide is None:
|
||||
raise ValueError("BFS H3 Duet needs a panel, a guide, or both")
|
||||
raw_panel = panel
|
||||
panel = static_over_panel(_blur(panel, source_blur, 1.0, "[BFS H3 Duet] panel"), panel_static)
|
||||
guide = _blur(guide, source_blur, 1.0, "[BFS H3 Duet] guide")
|
||||
panel = static_over_panel(_blur(panel, source_blur, 1.0, "[BFS H3 Duet] panel", source_blur_segmenter,
|
||||
source_blur_effect), panel_static)
|
||||
guide = _blur(guide, source_blur, 1.0, "[BFS H3 Duet] guide", source_blur_segmenter, source_blur_effect)
|
||||
if length <= 0:
|
||||
src = guide if guide is not None else panel
|
||||
length = src.shape[0] if src.shape[0] >= 5 else 124
|
||||
@@ -778,22 +783,32 @@ def _shot_loop():
|
||||
PANEL_STATIC = ["off", "face", "person"]
|
||||
|
||||
|
||||
def _blur_modes():
|
||||
def _blur_lists():
|
||||
try:
|
||||
from .bfs_source_blur import SOURCE_BLUR
|
||||
from . import bfs_source_blur as B
|
||||
except ImportError:
|
||||
from bfs_source_blur import SOURCE_BLUR
|
||||
return SOURCE_BLUR
|
||||
import bfs_source_blur as B
|
||||
return B
|
||||
|
||||
|
||||
def _blur(frames, mode, strength=1.0, log=""):
|
||||
def _blur_modes():
|
||||
return _blur_lists().SOURCE_BLUR
|
||||
|
||||
|
||||
BLUR_TT = ("Blur the source person in what the model sees (the panel / the guide): '<region> (<what stays sharp>)'. "
|
||||
"keep mouth: lips, teeth and tongue stay sharp; keep lips + tongue: the teeth are blurred too; mouth too: "
|
||||
"nothing stays sharp. masked area: the shot's own mask (elsewhere: the whole person). It reads the pose and "
|
||||
"motion but has no face to copy. The output is never blurred.")
|
||||
SEGMENTER_TT = ("What finds the face / head / hands / person and the mouth for source_blur. SAM 3.1: text prompts. "
|
||||
"Sapiens2: body-part classes in one pass, finer lips and teeth (downloads facebook/sapiens2-seg-0.4b, "
|
||||
"1.6 GB, to models/sapiens2 the first time). 'keep lips + tongue' always uses Sapiens2 for the lips.")
|
||||
EFFECT_TT = "blur: no detail left. invert colours: keeps the shapes and the expression but not the skin tone or colours."
|
||||
|
||||
|
||||
def _blur(frames, mode, strength=1.0, log="", segmenter="SAM 3.1", effect="blur", area=None):
|
||||
if frames is None or not mode or mode == "off":
|
||||
return frames
|
||||
try:
|
||||
from .bfs_source_blur import blur_source
|
||||
except ImportError:
|
||||
from bfs_source_blur import blur_source
|
||||
return blur_source(frames, mode, float(strength), log)
|
||||
return _blur_lists().blur_source(frames, mode, float(strength), log, segmenter, effect, area)
|
||||
|
||||
|
||||
def _small(frames: torch.Tensor, side: int = 640) -> torch.Tensor:
|
||||
@@ -909,7 +924,11 @@ if _io is not None:
|
||||
_io.Autogrow.Input("ref_images", optional=True, template=_io.Autogrow.TemplatePrefix(
|
||||
input=_io.Image.Input("ref_image", tooltip="<Picture n>, in order"), prefix="ref_image_", min=0, max=9)),
|
||||
_io.Combo.Input("source_blur", options=_blur_modes(), default="off", optional=True, tooltip=
|
||||
"Blur the source person in what the model sees (the panel / the guide), keeping the mouth sharp (SAM 3.1): it reads the pose, motion and lips but has no face to copy. The output is never blurred. Works with or instead of panel_static."),
|
||||
BLUR_TT + " Works with or instead of panel_static."),
|
||||
_io.Combo.Input("source_blur_segmenter", options=_blur_lists().SEGMENTERS, default="SAM 3.1",
|
||||
optional=True, tooltip=SEGMENTER_TT),
|
||||
_io.Combo.Input("source_blur_effect", options=_blur_lists().EFFECTS, default="blur", optional=True,
|
||||
tooltip=EFFECT_TT),
|
||||
_io.Combo.Input("panel_static", options=PANEL_STATIC, default="off", optional=True, tooltip=
|
||||
"TSC's fix for swaps in the same world: cover the face (or the whole person) of the pinned clip in TV "
|
||||
"static, every frame (SAM 3), so the generated half cannot copy it and takes the identity from the "
|
||||
@@ -927,12 +946,14 @@ if _io is not None:
|
||||
def execute(cls, clip, vae, panel, task, instruction, prompt, width, height, length, position, size, panel_noise,
|
||||
rope_mode, fit, gap, hold, rope_gap, ref_image_size, vlm_max_tokens, vlm=None, model=None,
|
||||
audio_vae=None, guide=None, ref_images=None, keep_mask=None, keep_video=None, panel_static="off",
|
||||
setting=None, source_blur="off"):
|
||||
setting=None, source_blur="off", source_blur_segmenter="SAM 3.1", source_blur_effect="blur"):
|
||||
from comfy_extras.nodes_minimax_h3 import MiniMaxH3ReferenceToVideo
|
||||
refs = [v for v in (ref_images or {}).values() if v is not None]
|
||||
raw_panel = panel
|
||||
panel = static_over_panel(_blur(panel, source_blur, 1.0, "[BFS H3 Duet Conditioning] panel"), panel_static)
|
||||
guide = _blur(guide, source_blur, 1.0, "[BFS H3 Duet Conditioning] guide")
|
||||
panel = static_over_panel(_blur(panel, source_blur, 1.0, "[BFS H3 Duet Conditioning] panel",
|
||||
source_blur_segmenter, source_blur_effect), panel_static)
|
||||
guide = _blur(guide, source_blur, 1.0, "[BFS H3 Duet Conditioning] guide", source_blur_segmenter,
|
||||
source_blur_effect)
|
||||
if length <= 0:
|
||||
length = panel.shape[0] if panel.shape[0] >= 5 else 124
|
||||
text = _shot_loop().duet_prompt(vlm, prompt, raw_panel, refs, task, instruction, int(vlm_max_tokens),
|
||||
@@ -996,8 +1017,10 @@ class BFSShotH3Duet:
|
||||
"optional": {"audio_vae": ("VAE",),
|
||||
"rope_mode": (ROPE_MODES, {"default": "canvas", "advanced": True}),
|
||||
"rope_gap": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 256.0, "step": 1.0, "advanced": True}),
|
||||
"source_blur": (_blur_modes(), {"default": "off", "tooltip": "Blur the source person in what the model sees (the panel / the guide), keeping the mouth sharp (SAM 3.1): it reads the pose, motion and lips but has no face to copy. The output is never blurred."}),
|
||||
"source_blur": (_blur_modes(), {"default": "off", "tooltip": BLUR_TT}),
|
||||
"source_blur_strength": ("FLOAT", {"default": 1.0, "min": 0.1, "max": 1.0, "step": 0.05}),
|
||||
"source_blur_segmenter": (_blur_lists().SEGMENTERS, {"default": "SAM 3.1", "tooltip": SEGMENTER_TT}),
|
||||
"source_blur_effect": (_blur_lists().EFFECTS, {"default": "blur", "tooltip": EFFECT_TT}),
|
||||
"vlm": ("CLIP", {"tooltip": "Optional VLM (Qwen3-VL CLIPLoader): fills {details} in the shot's "
|
||||
"prompt from its references when the planner left it."})},
|
||||
}
|
||||
@@ -1013,12 +1036,10 @@ class BFSShotH3Duet:
|
||||
|
||||
def render(self, shot, model, clip, vae, mode, task, instruction, use_ref_2, position, size, fit, gap,
|
||||
panel_noise, ref_image_size, steps, sampler_name, scheduler, seed, decode_canvas, audio_vae=None,
|
||||
rope_mode="canvas", rope_gap=0.0, source_blur="off", source_blur_strength=1.0, vlm=None):
|
||||
imgs = {}
|
||||
if shot.get("ref") is not None:
|
||||
imgs["ref_image_1"] = shot["ref"]
|
||||
if use_ref_2 and shot.get("ref2") is not None:
|
||||
imgs[f"ref_image_{len(imgs) + 1}"] = shot["ref2"]
|
||||
rope_mode="canvas", rope_gap=0.0, source_blur="off", source_blur_strength=1.0, vlm=None,
|
||||
source_blur_segmenter="SAM 3.1", source_blur_effect="blur"):
|
||||
own = _shot_loop().shot_refs(shot)
|
||||
imgs = {f"ref_image_{k + 1}": r for k, r in enumerate(own[:_shot_loop().MAX_REFS if use_ref_2 else 1])}
|
||||
duet, guided = mode != self.MODES[1], mode != self.MODES[0]
|
||||
try:
|
||||
from .bfs_shot_loop import chain_image, remember_result, shot_history
|
||||
@@ -1028,12 +1049,15 @@ class BFSShotH3Duet:
|
||||
if prev is not None and shot.get("chain") == "reference":
|
||||
imgs[f"ref_image_{len(imgs) + 1}"] = prev
|
||||
text = _resolve_prompt(shot.get("prompt", ""), task, instruction, len(imgs), 0)
|
||||
if "{details}" in text:
|
||||
text = _shot_loop().fill_details_now(text, vlm, [shot.get("ref"), shot.get("ref2")],
|
||||
f"[BFS Shot H3 Duet] shot {int(shot.get('index', 0)) + 1}")
|
||||
if "{details" in text:
|
||||
text = _shot_loop().fill_details_now(text, vlm, own, f"[BFS Shot H3 Duet] shot {int(shot.get('index', 0)) + 1}",
|
||||
shot.get("ref_names"))
|
||||
hist, hsrc = shot_history(shot) if shot_history else (None, None)
|
||||
area = None
|
||||
if source_blur and source_blur.startswith("masked area") and (shot.get("mask_src") or (shot.get("crop") or {}).get("crop_mask") is not None):
|
||||
area = _shot_loop().generation_mask(shot)
|
||||
seen = _blur(shot["frames"], source_blur, source_blur_strength,
|
||||
f"[BFS Shot H3 Duet] shot {int(shot.get('index', 0)) + 1}")
|
||||
f"[BFS Shot H3 Duet] shot {int(shot.get('index', 0)) + 1}", source_blur_segmenter, source_blur_effect, area)
|
||||
out = h3_render(model, clip, vae, audio_vae, text, {"ref_images": imgs}, shot["width"], shot["height"],
|
||||
shot["gen_length"], steps, sampler_name, scheduler, seed + int(shot.get("index", 0)),
|
||||
panel=seen if duet else None, guide=seen if guided else None,
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Sapiens2 body-part segmentation (Meta, facebook/sapiens2-seg-*): one class per pixel out of 29 (28 parts +
|
||||
background), lips, teeth and tongue separate from the face. Used by the source blur for parts SAM 3.1 can't
|
||||
find reliably from a text prompt (lips, tongue). It marks every person in the frame (no instances).
|
||||
|
||||
The weights (0.4B, 1.6 GB) go to models/sapiens2/sapiens2-seg-0.4b, downloaded from Hugging Face the first time.
|
||||
Sapiens2 License: https://github.com/facebookresearch/sapiens2/blob/main/LICENSE.md
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
import torch
|
||||
|
||||
REPO = "facebook/sapiens2-seg-0.4b"
|
||||
FILES = ("config.json", "preprocessor_config.json", "model.safetensors")
|
||||
CLASSES = ("Background", "Apparel", "Eyeglass", "Face_Neck", "Hair", "Left_Foot", "Left_Hand", "Left_Lower_Arm",
|
||||
"Left_Lower_Leg", "Left_Shoe", "Left_Sock", "Left_Upper_Arm", "Left_Upper_Leg", "Lower_Clothing",
|
||||
"Right_Foot", "Right_Hand", "Right_Lower_Arm", "Right_Lower_Leg", "Right_Shoe", "Right_Sock",
|
||||
"Right_Upper_Arm", "Right_Upper_Leg", "Torso", "Upper_Clothing", "Lower_Lip", "Upper_Lip", "Lower_Teeth",
|
||||
"Upper_Teeth", "Tongue")
|
||||
ID = {n: i for i, n in enumerate(CLASSES)}
|
||||
PIXELS = 1024 * 768 # the size it was trained at; other shapes keep this area and their aspect ratio
|
||||
_MEAN = (0.485, 0.456, 0.406)
|
||||
_STD = (0.229, 0.224, 0.225)
|
||||
_MODEL: dict = {}
|
||||
|
||||
|
||||
def model_dir() -> str:
|
||||
try:
|
||||
import folder_paths
|
||||
base = folder_paths.models_dir
|
||||
except Exception:
|
||||
base = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "..", "models")
|
||||
return os.path.join(base, "sapiens2", REPO.split("/")[-1])
|
||||
|
||||
|
||||
def _load():
|
||||
if "m" in _MODEL:
|
||||
return _MODEL["m"]
|
||||
d = model_dir()
|
||||
if not all(os.path.exists(os.path.join(d, f)) for f in FILES):
|
||||
print(f"[BFS Sapiens2] downloading {REPO} (1.6 GB) to {d}", flush=True)
|
||||
from huggingface_hub import hf_hub_download
|
||||
for f in FILES:
|
||||
hf_hub_download(REPO, f, local_dir=d)
|
||||
from transformers import Sapiens2ForSemanticSegmentation
|
||||
_MODEL["m"] = Sapiens2ForSemanticSegmentation.from_pretrained(d, dtype=torch.float32).eval()
|
||||
return _MODEL["m"]
|
||||
|
||||
|
||||
def _device():
|
||||
try:
|
||||
import comfy.model_management as mm
|
||||
return mm.get_torch_device()
|
||||
except Exception:
|
||||
return torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
|
||||
def input_size(h: int, w: int) -> tuple[int, int]:
|
||||
"""(H, W) with the trained area, the frame's aspect ratio, multiples of the 16 px patch."""
|
||||
s = (PIXELS / (h * w)) ** 0.5
|
||||
return max(16, round(h * s / 16) * 16), max(16, round(w * s / 16) * 16)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def segment(frames: torch.Tensor, out_size: tuple[int, int] | None = None, batch: int = 4) -> torch.Tensor:
|
||||
"""frames [N,H,W,3] 0-1 -> class ids [N,h,w] uint8 at `out_size` (default the frames' size)."""
|
||||
m = _load()
|
||||
dev = _device()
|
||||
half = dev.type == "cuda"
|
||||
m.to(dev, dtype=torch.float16 if half else torch.float32)
|
||||
N, H, W = frames.shape[:3]
|
||||
oh, ow = out_size or (H, W)
|
||||
ih, iw = input_size(H, W)
|
||||
mean = torch.tensor(_MEAN, device=dev).view(1, 3, 1, 1)
|
||||
std = torch.tensor(_STD, device=dev).view(1, 3, 1, 1)
|
||||
out = torch.empty(N, oh, ow, dtype=torch.uint8)
|
||||
try:
|
||||
for a in range(0, N, batch):
|
||||
x = frames[a:a + batch, ..., :3].to(dev).float().movedim(-1, 1)
|
||||
x = torch.nn.functional.interpolate(x, size=(ih, iw), mode="bilinear", align_corners=False, antialias=True)
|
||||
x = ((x - mean) / std).to(torch.float16 if half else torch.float32)
|
||||
lg = m(pixel_values=x).logits.float()
|
||||
lg = torch.nn.functional.interpolate(lg, size=(oh, ow), mode="bilinear", align_corners=False)
|
||||
out[a:a + batch] = lg.argmax(1).to(torch.uint8).cpu()
|
||||
del x, lg
|
||||
finally:
|
||||
m.to("cpu") # 0.4B: back to RAM, the VRAM is the video model's
|
||||
if half:
|
||||
torch.cuda.empty_cache()
|
||||
return out
|
||||
|
||||
|
||||
def parts(classes: torch.Tensor, names) -> torch.Tensor:
|
||||
"""[N,h,w] float mask of the pixels whose class is one of `names`."""
|
||||
ids = torch.tensor([ID[n] for n in names], dtype=classes.dtype)
|
||||
return torch.isin(classes, ids).float()
|
||||
+955
-477
File diff suppressed because it is too large
Load Diff
+100
-27
@@ -1,17 +1,41 @@
|
||||
"""Optional blur of the source person in what the model SEES of the source (aligned guide, native reference video,
|
||||
duet panel), keeping the mouth sharp: the model still reads the pose, the motion and the lips, but has no face to copy,
|
||||
so the identity comes from the reference. JalenBrunson's trick for reliable ref2v swaps (Sapiens2 there); here the
|
||||
masks come from SAM 3.1, already used by the planner. The kept area (inpaint) and the output are never blurred.
|
||||
duet panel): the model still reads the pose, the motion and (if kept sharp) the lips, but has no face to copy, so the
|
||||
identity comes from the reference. JalenBrunson's trick for reliable ref2v swaps. The kept area (inpaint) and the
|
||||
output are never blurred.
|
||||
|
||||
Modes are "<region> (<what stays sharp>)":
|
||||
region face / head + hands / person: found by the segmenter, every person in the frame;
|
||||
masked area: the shot's own mask (the person being replaced, with its expand), no extra segmentation
|
||||
sharp keep mouth: lips, teeth and tongue; keep lips + tongue: the teeth are blurred too (always Sapiens2);
|
||||
mouth too: nothing stays sharp
|
||||
Segmenter: SAM 3.1 (text prompts, already used by the planner) or Sapiens2 (body-part classes, one pass, finer lips).
|
||||
Effect: blur, or invert the colours (keeps the shapes and the expression, loses skin tone and colours).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
|
||||
SOURCE_BLUR = ["off", "face (keep mouth)", "head + hands (keep mouth)", "person (keep mouth)"]
|
||||
_PARTS = {SOURCE_BLUR[1]: ("face",), SOURCE_BLUR[2]: ("head", "hand"), SOURCE_BLUR[3]: ("person",)}
|
||||
_REGIONS = ("face", "head + hands", "person", "masked area")
|
||||
_SHARP = ("keep mouth", "keep lips + tongue", "mouth too")
|
||||
SOURCE_BLUR = ["off"] + [f"{r} ({k})" for k in _SHARP for r in _REGIONS]
|
||||
SEGMENTERS = ["SAM 3.1", "Sapiens2"]
|
||||
EFFECTS = ["blur", "invert colours"]
|
||||
_SAM_PARTS = {"face": ("face",), "head + hands": ("head", "hand"), "person": ("person",)}
|
||||
_SAP_FACE = ("Face_Neck", "Eyeglass", "Lower_Lip", "Upper_Lip", "Lower_Teeth", "Upper_Teeth", "Tongue")
|
||||
_SAP_PARTS = {"face": _SAP_FACE, "head + hands": _SAP_FACE + ("Hair", "Left_Hand", "Right_Hand")}
|
||||
_SAP_SHARP = {"keep mouth": ("Lower_Lip", "Upper_Lip", "Lower_Teeth", "Upper_Teeth", "Tongue"),
|
||||
"keep lips + tongue": ("Lower_Lip", "Upper_Lip", "Tongue")}
|
||||
_CACHE: dict = {}
|
||||
|
||||
|
||||
def parse(mode: str) -> tuple[str, str] | None:
|
||||
"""'face (keep mouth)' -> ('face', 'keep mouth'); None for off / unknown."""
|
||||
if not mode or "(" not in mode:
|
||||
return None
|
||||
region, sharp = mode.rstrip(")").split(" (", 1)
|
||||
return (region, sharp) if region in _REGIONS and sharp in _SHARP else None
|
||||
|
||||
|
||||
def _sl():
|
||||
try:
|
||||
from . import bfs_shot_loop as SL
|
||||
@@ -20,6 +44,13 @@ def _sl():
|
||||
return SL
|
||||
|
||||
|
||||
def _sap():
|
||||
try:
|
||||
from . import bfs_sapiens as SP
|
||||
except ImportError:
|
||||
import bfs_sapiens as SP
|
||||
return SP
|
||||
|
||||
def _small(frames: torch.Tensor, side: int = 640) -> torch.Tensor:
|
||||
H, W = frames.shape[1:3]
|
||||
s = side / max(H, W)
|
||||
@@ -36,51 +67,93 @@ def _grow(m: torch.Tensor, px: int) -> torch.Tensor:
|
||||
return torch.nn.functional.max_pool2d(m[:, None], 2 * px + 1, stride=1, padding=px)[:, 0]
|
||||
|
||||
|
||||
def blur_mask(frames: torch.Tensor, mode: str, log: str = "") -> torch.Tensor | None:
|
||||
"""[N,h,w] float mask (working size) of what to blur: the parts for `mode` minus the (slightly grown) mouth."""
|
||||
parts = _PARTS.get(mode)
|
||||
if not parts:
|
||||
def blur_mask(frames: torch.Tensor, mode: str, log: str = "", segmenter: str = "SAM 3.1",
|
||||
area: torch.Tensor | None = None) -> torch.Tensor | None:
|
||||
"""[N,h,w] float mask (working size) of what to blur: the region for `mode` minus what stays sharp (grown a
|
||||
little). `area` [N,H,W]: the shot's own mask, for the 'masked area' region."""
|
||||
pr = parse(mode)
|
||||
if not pr:
|
||||
return None
|
||||
region, sharp = pr
|
||||
if region == "masked area" and area is None:
|
||||
if log:
|
||||
print(f"{log}: source blur 'masked area' needs a mask on the shot (SAM 3 text or points): blurring the "
|
||||
"whole person instead", flush=True)
|
||||
region = "person"
|
||||
SL = _sl()
|
||||
small = _small(frames)
|
||||
key = (mode, tuple(small.shape), SL._image_key(frames[:1]), SL._image_key(frames[-1:]),
|
||||
SL._image_key(frames[frames.shape[0] // 2:frames.shape[0] // 2 + 1]))
|
||||
h, w = small.shape[1:3]
|
||||
sap_region = region != "masked area" and segmenter == "Sapiens2"
|
||||
sap_sharp = sharp == "keep lips + tongue" or (sharp == "keep mouth" and segmenter == "Sapiens2")
|
||||
key = (region, sharp, segmenter, tuple(small.shape), SL._image_key(frames[:1]), SL._image_key(frames[-1:]),
|
||||
SL._image_key(frames[frames.shape[0] // 2:frames.shape[0] // 2 + 1]),
|
||||
None if area is None else (tuple(area.shape), float(area.float().sum())))
|
||||
if key in _CACHE:
|
||||
return _CACHE[key]
|
||||
m = torch.zeros(small.shape[:3])
|
||||
for p in parts:
|
||||
m = torch.maximum(m, (SL.segment_frames(small, dict(SL.DEFAULT_MASK, text=p, max_objects=8, threshold=0.4)) > 0.5).float())
|
||||
mouth = (SL.segment_frames(small, dict(SL.DEFAULT_MASK, text="mouth", max_objects=8, threshold=0.3)) > 0.5).float()
|
||||
keep = _grow(mouth, max(1, int(min(small.shape[1:3]) * 0.012)))
|
||||
out = (_grow(m, max(1, int(min(small.shape[1:3]) * 0.015))) * (1 - keep)).clamp(0, 1)
|
||||
cls = _sap().segment(small) if sap_region or sap_sharp else None
|
||||
if region == "masked area":
|
||||
m = torch.nn.functional.interpolate(area[:, None].float(), size=(h, w), mode="nearest")[:, 0].clamp(0, 1)
|
||||
m_grow = 0 # already grown by the mask's own expand
|
||||
elif sap_region:
|
||||
m = (cls > 0).float() if region == "person" else _sap().parts(cls, _SAP_PARTS[region])
|
||||
m_grow = max(1, int(min(h, w) * 0.01))
|
||||
else:
|
||||
m = torch.zeros(small.shape[:3])
|
||||
for p in _SAM_PARTS[region]:
|
||||
m = torch.maximum(m, (SL.segment_frames(small, dict(SL.DEFAULT_MASK, text=p, max_objects=8, threshold=0.4)) > 0.5).float())
|
||||
m_grow = max(1, int(min(h, w) * 0.015))
|
||||
if sharp == "mouth too":
|
||||
keep, found = torch.zeros_like(m), None
|
||||
elif sap_sharp:
|
||||
keep = _sap().parts(cls, _SAP_SHARP[sharp])
|
||||
keep = _grow(keep, max(1, int(min(h, w) * 0.005)))
|
||||
found = float(keep.sum()) > 0
|
||||
else:
|
||||
mouth = (SL.segment_frames(small, dict(SL.DEFAULT_MASK, text="mouth", max_objects=8, threshold=0.3)) > 0.5).float()
|
||||
keep = _grow(mouth, max(1, int(min(h, w) * 0.012)))
|
||||
found = float(mouth.sum()) > 0
|
||||
out = (_grow(m, m_grow) * (1 - keep)).clamp(0, 1)
|
||||
if log:
|
||||
print(f"{log}: source blur '{mode}': {float(m.mean()) * 100:.1f}% of the frame, mouth kept "
|
||||
f"({'found' if float(mouth.sum()) > 0 else 'not found: the whole part is blurred'})", flush=True)
|
||||
kept = "nothing kept sharp" if found is None else \
|
||||
f"{sharp[5:]} kept ({'found' if found else 'not found: the whole region is blurred'})"
|
||||
used = "shot mask" if region == "masked area" else "Sapiens2" if sap_region else "SAM 3.1"
|
||||
print(f"{log}: source blur '{region}' ({used}{' + Sapiens2 lips' if sap_sharp and not sap_region else ''}): "
|
||||
f"{float(m.mean()) * 100:.1f}% of the frame, {kept}", flush=True)
|
||||
if len(_CACHE) > 8:
|
||||
_CACHE.pop(next(iter(_CACHE)))
|
||||
_CACHE[key] = out
|
||||
return out
|
||||
|
||||
|
||||
def blur_source(frames: torch.Tensor | None, mode: str, strength: float = 1.0, log: str = "") -> torch.Tensor | None:
|
||||
"""The frames with the source person blurred per `mode` (mouth kept). `strength` 0-1 sets how coarse the blur is."""
|
||||
def _soft(m: torch.Tensor, H: int, W: int, N: int) -> torch.Tensor:
|
||||
"""The working-size mask at the frame size with a soft edge, one per frame ([N,H,W,1])."""
|
||||
mm = torch.nn.functional.interpolate(m[:, None], size=(H, W), mode="bilinear", align_corners=False)
|
||||
mm = torch.nn.functional.avg_pool2d(mm, 9, stride=1, padding=4)[:, 0, ..., None].clamp(0, 1)
|
||||
if mm.shape[0] != N: # one mask per frame; a shorter list holds its last one
|
||||
mm = mm[[min(i, mm.shape[0] - 1) for i in range(N)]]
|
||||
return mm
|
||||
|
||||
|
||||
def blur_source(frames: torch.Tensor | None, mode: str, strength: float = 1.0, log: str = "",
|
||||
segmenter: str = "SAM 3.1", effect: str = "blur", area: torch.Tensor | None = None) -> torch.Tensor | None:
|
||||
"""The frames with the source person blurred (or colour-inverted) per `mode`. `strength` 0-1 sets how coarse the
|
||||
blur is; `area`: the shot's mask for the 'masked area' modes."""
|
||||
if frames is None or not mode or mode == "off" or strength <= 0:
|
||||
return frames
|
||||
m = blur_mask(frames, mode, log)
|
||||
m = blur_mask(frames, mode, log, segmenter, area)
|
||||
if m is None or float(m.sum()) == 0:
|
||||
return frames
|
||||
x = frames[..., :3].float()
|
||||
N, H, W = x.shape[:3]
|
||||
if effect == "invert colours":
|
||||
mm = _soft(m, H, W, N)
|
||||
return x * (1 - mm) + (1 - x) * mm
|
||||
# a heavy blur: down to ~1/12-1/40 of the size and back, then a soft pass; blocky detail never survives it
|
||||
f = max(4, int(min(H, W) * (0.025 + 0.06 * float(strength))))
|
||||
low = torch.nn.functional.interpolate(x.movedim(-1, 1), size=(max(2, H // f), max(2, W // f)), mode="area")
|
||||
blur = torch.nn.functional.interpolate(low, size=(H, W), mode="bilinear", align_corners=False)
|
||||
blur = torch.nn.functional.avg_pool2d(blur, 5, stride=1, padding=2).movedim(1, -1)
|
||||
mm = torch.nn.functional.interpolate(m[:, None], size=(H, W), mode="bilinear", align_corners=False)
|
||||
mm = torch.nn.functional.avg_pool2d(mm, 9, stride=1, padding=4)[:, 0, ..., None].clamp(0, 1)
|
||||
if mm.shape[0] != N: # one mask per frame; a shorter list holds its last one
|
||||
idx = [min(i, mm.shape[0] - 1) for i in range(N)]
|
||||
mm = mm[idx]
|
||||
mm = _soft(m, H, W, N)
|
||||
out = x * (1 - mm) + blur * mm
|
||||
del low, blur, mm
|
||||
return out
|
||||
|
||||
+196
-1
@@ -7,8 +7,11 @@ gets three temporary MP4s (guide, target, difference) and the per-frame differen
|
||||
"""
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import uuid
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
@@ -51,6 +54,32 @@ def psnr(mse: float) -> float:
|
||||
return float("inf") if mse <= 0 else 10 * math.log10(1.0 / mse)
|
||||
|
||||
|
||||
MAX_REFS = 12
|
||||
|
||||
|
||||
def ref_pictures(reference) -> list:
|
||||
"""The reference pictures, one [H,W,3] each: from a list of IMAGEs of any sizes (e.g. the planner's or BFS Shot
|
||||
Unpack's refs) and / or IMAGE batches."""
|
||||
out = []
|
||||
for item in reference if isinstance(reference, list) else [reference]:
|
||||
if item is not None:
|
||||
out += [item[i] for i in range(int(item.shape[0]))]
|
||||
return out[:MAX_REFS]
|
||||
|
||||
|
||||
def save_refs(reference, out_dir: str, height: int) -> list:
|
||||
"""The reference pictures as PNGs no taller than the player (no upscale) -> file names."""
|
||||
from PIL import Image
|
||||
names = []
|
||||
for i, pic in enumerate(ref_pictures(reference)):
|
||||
im = Image.fromarray((pic[..., :3].float().clamp(0, 1).cpu().numpy() * 255).round().astype(np.uint8))
|
||||
if im.height > height:
|
||||
im = im.resize((max(1, round(im.width * height / im.height)), height), Image.LANCZOS)
|
||||
names.append(f"ref_{i}.png")
|
||||
im.save(os.path.join(out_dir, names[-1]))
|
||||
return names
|
||||
|
||||
|
||||
class BFSVideoCompare:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
@@ -72,11 +101,18 @@ class BFSVideoCompare:
|
||||
"preview_height": ("INT", {"default": 720, "min": 0, "max": 4096, "step": 16, "tooltip":
|
||||
"Height of the player's videos (0 = the compared size). Only the preview: "
|
||||
"the difference is measured before it."}),
|
||||
"reference": ("IMAGE", {"tooltip": "Optional reference pictures (e.g. the people swapped in), shown in a "
|
||||
"strip below the videos, numbered as <Picture n>, and in what ⬇ save / ● rec "
|
||||
"download. Takes the refs list of BFS Shot Planner / Unpack (pictures of any "
|
||||
"size) or a batch; up to 12."}),
|
||||
"label_guide": ("STRING", {"default": "guide"}),
|
||||
"label_target": ("STRING", {"default": "target"}),
|
||||
},
|
||||
}
|
||||
|
||||
# lists in: the reference can be the refs list of the planner / Unpack (pictures of different sizes); the other
|
||||
# inputs take their first item
|
||||
INPUT_IS_LIST = True
|
||||
RETURN_TYPES = ("IMAGE", "STRING", "FLOAT")
|
||||
RETURN_NAMES = ("difference", "report", "mean_difference")
|
||||
OUTPUT_TOOLTIPS = ("The difference map, one frame per compared frame (black = equal, white = very different).",
|
||||
@@ -89,7 +125,10 @@ class BFSVideoCompare:
|
||||
"where they differ, pixel by pixel.")
|
||||
|
||||
def compare(self, guide, target, fps=24.0, audio=None, gain=2.0, size="guide", preview_height=720,
|
||||
label_guide="guide", label_target="target"):
|
||||
label_guide="guide", label_target="target", reference=None):
|
||||
one = lambda v: v[0] if isinstance(v, list) and v else v
|
||||
guide, target, fps, audio, gain, size, preview_height, label_guide, label_target = map(
|
||||
one, (guide, target, fps, audio, gain, size, preview_height, label_guide, label_target))
|
||||
notes = []
|
||||
n = min(guide.shape[0], target.shape[0])
|
||||
if guide.shape[0] != target.shape[0]:
|
||||
@@ -136,11 +175,167 @@ class BFSVideoCompare:
|
||||
print(f"[BFS Video Compare] {report}", flush=True)
|
||||
sub = os.path.relpath(out_dir, folder_paths.get_temp_directory())
|
||||
files = {k: {"filename": f, "subfolder": sub, "type": "temp"} for k, f in names.items()}
|
||||
refs = [{"filename": f, "subfolder": sub, "type": "temp"} for f in save_refs(reference, out_dir, PH)]
|
||||
ui = {"bfs_compare": [{**files, "fps": float(fps), "frames": n, "width": PW, "height": PH,
|
||||
"labels": [label_guide, label_target], "mae": [round(float(x), 2) for x in mae],
|
||||
"refs": refs,
|
||||
"report": report}]}
|
||||
return {"ui": ui, "result": (diff_map, report, mean)}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- save / record: the browser's file -> a shareable one
|
||||
|
||||
EXPORT_FORMATS = { # what the player's format list offers; "webm" never reaches the server (the browser's own file)
|
||||
"mp4": "MP4 (H.264)",
|
||||
"small": "MP4 small (under 10 MB)",
|
||||
"gif": "GIF (under 10 MB, no sound)",
|
||||
}
|
||||
LIMIT_BYTES = int(9.5 * 1024 * 1024) # under the 10 MB many chat apps accept for free
|
||||
_ID = re.compile(r"^[0-9a-f]{8,32}$")
|
||||
|
||||
|
||||
def _export_dir() -> str:
|
||||
d = os.path.join(folder_paths.get_temp_directory(), "bfs_compare", "exports")
|
||||
os.makedirs(d, exist_ok=True)
|
||||
return d
|
||||
|
||||
|
||||
def write_part(rid: str, offset: int, data: bytes) -> int:
|
||||
"""One piece of a recording uploaded by the player (pieces keep big recordings under the upload limit)."""
|
||||
if not _ID.match(rid or "") or offset < 0:
|
||||
raise ValueError("bad recording id / offset")
|
||||
path = os.path.join(_export_dir(), f"{rid}.rec")
|
||||
with open(path, "r+b" if os.path.exists(path) else "wb") as f:
|
||||
f.seek(offset)
|
||||
f.write(data)
|
||||
return len(data)
|
||||
|
||||
|
||||
def _run(cmd: list) -> None:
|
||||
r = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if r.returncode != 0:
|
||||
raise RuntimeError(f"ffmpeg failed: {r.stderr.strip()[-600:]}")
|
||||
|
||||
|
||||
def encode(src: str, fmt: str, seconds: float, out_base: str) -> str:
|
||||
"""The recording `src` (WebM / fragmented MP4 from MediaRecorder, which many apps won't play or preview) as
|
||||
`fmt` -> the written path. Both MP4s are H.264 + AAC, yuv420p, faststart. mp4: CRF 18. small: the bitrate set so
|
||||
the file stays under 9.5 MB (smaller frame when it gets low). gif: palette per video, 15 fps, made smaller until
|
||||
it fits 9.5 MB."""
|
||||
if fmt not in EXPORT_FORMATS:
|
||||
raise ValueError(f"unknown format {fmt!r}")
|
||||
exe = VIO.ffmpeg_bin()
|
||||
if exe is None:
|
||||
return _encode_av(src, fmt, out_base)
|
||||
seconds = max(0.5, float(seconds or 0))
|
||||
even = "scale=trunc(iw/2)*2:trunc(ih/2)*2"
|
||||
if fmt == "gif":
|
||||
out = out_base + ".gif"
|
||||
for width, fps in ((720, 15), (540, 15), (480, 12), (360, 12), (270, 10)):
|
||||
vf = f"fps={fps},scale='min({width},iw)':-2:flags=lanczos,split[a][b];[a]palettegen=stats_mode=diff[p];[b][p]paletteuse=dither=bayer:bayer_scale=4"
|
||||
_run([exe, "-y", "-v", "error", "-i", src, "-vf", vf, "-loop", "0", out])
|
||||
if os.path.getsize(out) <= LIMIT_BYTES:
|
||||
break
|
||||
return out
|
||||
out = out_base + ".mp4"
|
||||
if fmt == "mp4":
|
||||
_run([exe, "-y", "-v", "error", "-i", src, "-vf", even, "-c:v", "libx264", "-preset", "medium", "-crf", "18",
|
||||
"-pix_fmt", "yuv420p", "-c:a", "aac", "-b:a", "192k", "-movflags", "+faststart", out])
|
||||
return out
|
||||
audio_k = 128
|
||||
for shrink in (1.0, 0.92, 0.8): # a VBR encode can overshoot a little: a second, lower try
|
||||
kbps = int(min(8000, max(250, LIMIT_BYTES * 8 / 1000 / seconds * shrink - audio_k)))
|
||||
vf = even if kbps >= 1500 else "scale=-2:'min(720,ih)':flags=lanczos," + even if kbps >= 700 else \
|
||||
"scale=-2:'min(480,ih)':flags=lanczos," + even
|
||||
_run([exe, "-y", "-v", "error", "-i", src, "-vf", vf, "-c:v", "libx264", "-preset", "medium",
|
||||
"-b:v", f"{kbps}k", "-maxrate", f"{int(kbps * 1.4)}k", "-bufsize", f"{kbps * 2}k", "-pix_fmt", "yuv420p",
|
||||
"-r", "30", "-c:a", "aac", "-b:a", f"{audio_k}k", "-ac", "2", "-movflags", "+faststart", out])
|
||||
if os.path.getsize(out) <= LIMIT_BYTES:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def _encode_av(src: str, fmt: str, out_base: str) -> str:
|
||||
"""No ffmpeg program: decode with PyAV and write an MP4 (no sound) or a GIF (PIL); 'small' at most 720p."""
|
||||
import av
|
||||
from PIL import Image
|
||||
frames, fps = [], 30.0
|
||||
with av.open(src) as c:
|
||||
st = c.streams.video[0]
|
||||
fps = float(st.average_rate or st.guessed_rate or 30)
|
||||
for f in c.decode(st):
|
||||
frames.append(f.to_ndarray(format="rgb24"))
|
||||
if not frames:
|
||||
raise RuntimeError("the recording has no frames")
|
||||
h, w = frames[0].shape[:2]
|
||||
if fmt == "gif":
|
||||
out = out_base + ".gif"
|
||||
step = max(1, round(fps / 12))
|
||||
for side in (540, 400, 300, 220):
|
||||
k = min(1.0, side / max(h, w))
|
||||
ims = [Image.fromarray(x).resize((max(2, int(w * k)), max(2, int(h * k))), Image.LANCZOS) for x in frames[::step]]
|
||||
ims[0].save(out, save_all=True, append_images=ims[1:], duration=int(1000 * step / fps), loop=0, optimize=True)
|
||||
if os.path.getsize(out) <= LIMIT_BYTES:
|
||||
break
|
||||
return out
|
||||
out = out_base + ".mp4"
|
||||
k = 1.0 if fmt == "mp4" else min(1.0, 720 / h)
|
||||
W2, H2 = int(w * k) // 2 * 2, int(h * k) // 2 * 2
|
||||
wr = VIO.Mp4Writer(out, W2, H2, fps, None, crf=18 if fmt == "mp4" else 27, who="BFS Video Compare")
|
||||
try:
|
||||
for a in range(0, len(frames), 16):
|
||||
x = torch.from_numpy(np.stack(frames[a:a + 16])).float() / 255
|
||||
wr.write(_scale(x, H2, W2))
|
||||
finally:
|
||||
wr.close()
|
||||
print("[BFS Video Compare] no ffmpeg program: saved without sound", flush=True)
|
||||
return out
|
||||
|
||||
|
||||
def finish_export(rid: str, fmt: str, seconds: float, name: str) -> dict:
|
||||
src = os.path.join(_export_dir(), f"{rid}.rec")
|
||||
if not _ID.match(rid or "") or not os.path.exists(src):
|
||||
raise ValueError("recording not found")
|
||||
stem = re.sub(r"[^\w.-]+", "_", os.path.splitext(name or "bfs_compare")[0])[:80] or "bfs_compare"
|
||||
try:
|
||||
out = encode(src, fmt, seconds, os.path.join(_export_dir(), f"{stem}_{rid[:6]}"))
|
||||
finally:
|
||||
os.remove(src)
|
||||
sub = os.path.relpath(os.path.dirname(out), folder_paths.get_temp_directory()).replace(os.sep, "/")
|
||||
mb = os.path.getsize(out) / 1024 / 1024
|
||||
print(f"[BFS Video Compare] saved {os.path.basename(out)} ({EXPORT_FORMATS[fmt]}, {mb:.1f} MB)", flush=True)
|
||||
return {"filename": os.path.basename(out), "subfolder": sub, "type": "temp", "mb": round(mb, 2)}
|
||||
|
||||
|
||||
try:
|
||||
import asyncio
|
||||
|
||||
from aiohttp import web
|
||||
from server import PromptServer
|
||||
|
||||
@PromptServer.instance.routes.post("/bfs/compare/part")
|
||||
async def _bfs_compare_part(request):
|
||||
try:
|
||||
data = await request.read()
|
||||
n = await asyncio.get_running_loop().run_in_executor(
|
||||
None, lambda: write_part(request.query.get("id", ""), int(request.query.get("offset", "-1")), data))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return web.json_response({"error": f"{type(exc).__name__}: {exc}"}, status=400)
|
||||
return web.json_response({"written": n})
|
||||
|
||||
@PromptServer.instance.routes.post("/bfs/compare/export")
|
||||
async def _bfs_compare_export(request):
|
||||
try:
|
||||
b = await request.json()
|
||||
r = await asyncio.get_running_loop().run_in_executor(
|
||||
None, lambda: finish_export(b.get("id", ""), b.get("format", ""), float(b.get("seconds") or 0), b.get("name", "")))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return web.json_response({"error": f"{type(exc).__name__}: {exc}"}, status=400)
|
||||
return web.json_response(r)
|
||||
except Exception as _exc: # noqa: BLE001 - outside ComfyUI (tests)
|
||||
if "PromptServer" not in repr(_exc) and "server" not in repr(_exc):
|
||||
print(f"[BFS Video Compare] export routes not registered: {_exc!r}", flush=True)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {"BFSVideoCompare": BFSVideoCompare}
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {"BFSVideoCompare": "BFS Video Compare"}
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
[project]
|
||||
name = "bfsnodes"
|
||||
description = "Custom ComfyUI nodes for reference-guided video: BFS Sampler (guide-driven IC-LoRAs, auto crop, mask inpainting, temporal chunking), LTX Multiple Controls / Identity Transfer / Edit Anything, Head-Swap and Anime2Real conditioning, MiniMax-H3 latent guides, RoPE layouts and loaders, LoRA Surgery, and Shot Planner / Shot Loop (tabbed panel): split long videos into shots at camera cuts, face-tracked cast, SAM 3 mask per shot with full frame, paste, mask, crop and crop + mask generation modes, optional translucent Colored Mark overlay for manually selected regions, optional Qwen3-VL to suggest settings, describe references ({details}) and the person to replace ({target}), continuity between shots, H3 conditioning with an optional duet and prompt writer, join with the original timing, audio and a side-by-side comparison. H3 Duet: pin a source clip beside the video, training-free."
|
||||
version = "1.107.0"
|
||||
version = "1.116.0"
|
||||
license = {file = "LICENSE"}
|
||||
dependencies = [
|
||||
"torch",
|
||||
|
||||
+398
-58
@@ -174,6 +174,43 @@ class JoinTest(unittest.TestCase):
|
||||
self.assertEqual(video.shape[0], 45)
|
||||
self.assertAlmostEqual(float(video[0, 0, 0, 0]), 0.1, places=2)
|
||||
|
||||
def test_run_again_with_every_shot_stored_generates_nothing(self):
|
||||
join = SL.BFSShotJoin()
|
||||
shots = [_shot(0, 0, 20, count=2, queue=True, run_id="t2"), _shot(1, 20, 45, cut=True, count=2, queue=True, run_id="t2")]
|
||||
join.join([self._frames(22, 0.1)], [shots[0]], [0])
|
||||
join.join([self._frames(39, 0.9)], [shots[1]], [0])
|
||||
again = dict(shots[1], assemble_only=True) # what the planner hands out when nothing is left to run
|
||||
self.assertEqual(join.check_lazy_status(images=None, shots=[again]), []) # the sampler is not pulled
|
||||
self.assertEqual(join.check_lazy_status(images=None, shots=[shots[1]]), ["images"])
|
||||
video = SL.BFSShotJoin().join(shots=[again], crossfade=[0])[0]
|
||||
self.assertEqual(video.shape[0], 45)
|
||||
self.assertAlmostEqual(float(video[0, 0, 0, 0]), 0.1, places=2)
|
||||
self.assertAlmostEqual(float(video[-1, 0, 0, 0]), 0.9, places=2)
|
||||
|
||||
def test_comparison_turned_on_later_reads_the_originals_from_the_source(self):
|
||||
import cv2
|
||||
import numpy as np
|
||||
src = os.path.join(_TMP, "cmp_src.mp4")
|
||||
vw = cv2.VideoWriter(src, cv2.VideoWriter_fourcc(*"mp4v"), 24, (32, 32))
|
||||
for _ in range(50):
|
||||
vw.write(np.full((32, 32, 3), 200, np.uint8))
|
||||
vw.release()
|
||||
tl = {"path": src, "fps": 24.0, "width": 32, "height": 32, "fill": "original", "audio": None,
|
||||
"segs": [{"start": 0, "end": 20, "run": True, "cut_before": False, "run_index": 0},
|
||||
{"start": 20, "end": 45, "run": True, "cut_before": True, "run_index": 1}],
|
||||
"src": list(range(50)), "n": 50}
|
||||
shots = [_shot(0, 0, 20, count=2, queue=True, run_id="t3"), _shot(1, 20, 45, cut=True, count=2, queue=True, run_id="t3")]
|
||||
join = SL.BFSShotJoin()
|
||||
frames = lambda n, v: torch.full((n, 32, 32, 3), float(v))
|
||||
join.join([frames(22, 0.1)], [shots[0]], [0], timeline=[tl]) # stored with the comparison off
|
||||
join.join([frames(39, 0.9)], [shots[1]], [0], timeline=[tl])
|
||||
out = SL.BFSShotJoin().join(shots=[dict(shots[1], assemble_only=True)], crossfade=[0], timeline=[tl],
|
||||
comparison=[True], comparison_height=[32])
|
||||
comp, guide = out[3], out[5]
|
||||
self.assertEqual(guide.shape[0], 45) # the guide output is there again
|
||||
self.assertAlmostEqual(float(guide[10].mean()), 200 / 255, places=1) # and it is the source video
|
||||
self.assertGreater((comp - 200 / 255).abs().lt(0.05).float().mean().item(), 0.05) # original column shown
|
||||
|
||||
|
||||
class FilterTest(unittest.TestCase):
|
||||
STATS = {"persons": 1, "person_area": 0.2, "person_frames": 6, "faces": 1, "brightness": 0.4, "motion": 0.02, "sampled": 6}
|
||||
@@ -385,29 +422,41 @@ class RepackTest(unittest.TestCase):
|
||||
shot = dict(_shot(0, 0, 20, gen=22), width=8, height=8, frames=torch.zeros(22, 8, 8, 3),
|
||||
ref=torch.zeros(1, 4, 4, 3), ref2=None, prompt="old", audio=None)
|
||||
ref = torch.ones(1, 6, 6, 3)
|
||||
out = SL.BFSShotRepack().repack(shot, ref_image=ref)[0]
|
||||
out = SL.BFSShotRepack().repack([shot], refs=[ref])[0][0]
|
||||
self.assertTrue(torch.equal(out["ref"], ref))
|
||||
self.assertEqual(out["prompt"], "old")
|
||||
self.assertIs(out["frames"], shot["frames"])
|
||||
out = SL.BFSShotRepack().repack(shot, guide_frames=torch.ones(30, 16, 16, 3), prompt="new")[0]
|
||||
out = SL.BFSShotRepack().repack([shot], guide_frames=[torch.ones(30, 16, 16, 3)], prompt=["new"])[0][0]
|
||||
self.assertEqual(tuple(out["frames"].shape), (22, 8, 8, 3))
|
||||
self.assertEqual(out["prompt"], "new")
|
||||
self.assertEqual(out["length"], 20)
|
||||
|
||||
def test_unpack_lists_the_pictures_and_repack_puts_them_back_per_shot(self):
|
||||
a = [torch.full((1, 4, 4, 3), k / 10) for k in range(3)]
|
||||
base = dict(_shot(0, 0, 20, gen=22), width=8, height=8, frames=torch.zeros(22, 8, 8, 3), prompt="",
|
||||
audio=None)
|
||||
s1, s2, s3 = dict(base, refs=a[:2]), dict(base, index=1, refs=[]), dict(base, index=2, refs=a[2:])
|
||||
flat = []
|
||||
for s in (s1, s2, s3): # what ComfyUI hands the next node: the lists of every Unpack call, in order
|
||||
out = SL.BFSShotUnpack().unpack(s)
|
||||
flat += out[1]
|
||||
self.assertEqual(out[2], len(s["refs"]))
|
||||
edited = [x + 1 for x in flat] # e.g. a background remover, once per picture
|
||||
s1n, s2n, s3n = SL.BFSShotRepack().repack([s1, s2, s3], refs=edited)[0]
|
||||
self.assertEqual([len(s1n["refs"]), len(s2n["refs"]), len(s3n["refs"])], [2, 0, 1])
|
||||
self.assertTrue(torch.equal(s3n["refs"][0], a[2] + 1) and torch.equal(s1n["ref2"], a[1] + 1))
|
||||
|
||||
class ConditioningWriterTest(unittest.TestCase):
|
||||
"""The optional prompt writer of BFS Shot H3 Conditioning (template path, no models)."""
|
||||
|
||||
class ConditioningPromptTest(unittest.TestCase):
|
||||
"""BFS Shot H3 Conditioning passes the planner's prompt and adds the setting picture (no models)."""
|
||||
|
||||
def setUp(self):
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
import bfs_h3_side_panel as SP
|
||||
self.SP, self.seen = SP, {}
|
||||
seen = self.seen
|
||||
self.seen = seen = {}
|
||||
|
||||
class R2V:
|
||||
@staticmethod
|
||||
def execute(**kw):
|
||||
seen["prompt"] = kw["prompt"]
|
||||
seen.update(kw)
|
||||
return types.SimpleNamespace(args=("pos", "lat"))
|
||||
|
||||
mod = types.ModuleType("comfy_extras.nodes_minimax_h3")
|
||||
@@ -415,13 +464,8 @@ class ConditioningWriterTest(unittest.TestCase):
|
||||
self._old = {k: sys.modules.get(k) for k in ("comfy_extras", "comfy_extras.nodes_minimax_h3")}
|
||||
sys.modules["comfy_extras"] = types.ModuleType("comfy_extras")
|
||||
sys.modules["comfy_extras.nodes_minimax_h3"] = mod
|
||||
self._apply = SP.BFSH3SidePanel.apply
|
||||
SP.BFSH3SidePanel.apply = lambda self, pos, lat, *a, **k: (pos, lat, {"position": "left"}, None, None)
|
||||
self._patch = SP.patch_model_rope
|
||||
SP.patch_model_rope = lambda m, info, gap: "patched"
|
||||
|
||||
def tearDown(self):
|
||||
self.SP.BFSH3SidePanel.apply, self.SP.patch_model_rope = self._apply, self._patch
|
||||
for k, v in self._old.items():
|
||||
if v is None:
|
||||
sys.modules.pop(k, None)
|
||||
@@ -429,44 +473,23 @@ class ConditioningWriterTest(unittest.TestCase):
|
||||
sys.modules[k] = v
|
||||
|
||||
def _shot(self):
|
||||
return dict(prompt="my own {layout} prompt", width=448, height=800, gen_length=22,
|
||||
frames=torch.zeros(22, 8, 8, 3), ref=torch.zeros(1, 4, 4, 3), ref2=None, audio=None)
|
||||
return dict(prompt="subject_definitions: <Subject 1> is the man in <Picture 1>.", width=448, height=800,
|
||||
gen_length=22, frames=torch.full((22, 8, 8, 3), 0.5), ref=torch.zeros(1, 4, 4, 3), ref2=None,
|
||||
audio=None)
|
||||
|
||||
def _run(self, **kw):
|
||||
c = SL.BFSShotH3Conditioning()
|
||||
return c.condition(self._shot(), "clip", "vae", "none", True, "none", "match", **kw)
|
||||
|
||||
def test_planner_prompt_is_default_and_layout_filled(self):
|
||||
out = self._run()
|
||||
self.assertEqual(out[3], "my own prompt")
|
||||
out = self._run(duet="canvas")
|
||||
self.assertIn("the left half shows the original performance", out[3])
|
||||
def test_the_planner_prompt_goes_as_it_is_and_the_model_passes_through(self):
|
||||
out = SL.BFSShotH3Conditioning().condition(self._shot(), "clip", "vae", "none", "none", "match", model="m")
|
||||
self.assertEqual(out[3], self._shot()["prompt"])
|
||||
self.assertEqual(self.seen["prompt"], out[3])
|
||||
|
||||
def test_task_template_follows_the_duet_mode(self):
|
||||
out = self._run(duet="canvas", task="character swap")
|
||||
self.assertIn("split screen", out[3])
|
||||
self.assertIn("<Picture 1>", out[3])
|
||||
out = self._run(duet="shifted RoPE", model="m", task="character swap")
|
||||
self.assertNotIn("split screen", out[3])
|
||||
self.assertEqual(out[2], "patched")
|
||||
self.assertEqual(out[2], "m")
|
||||
self.assertNotIn("duet", SL.BFSShotH3Conditioning.INPUT_TYPES()["optional"])
|
||||
|
||||
def test_setting_picture_is_the_last_reference_with_static(self):
|
||||
seen = {}
|
||||
orig = sys.modules["comfy_extras.nodes_minimax_h3"].MiniMaxH3ReferenceToVideo.execute
|
||||
|
||||
def capture(**kw):
|
||||
seen.update(kw)
|
||||
return orig(**kw)
|
||||
sys.modules["comfy_extras.nodes_minimax_h3"].MiniMaxH3ReferenceToVideo.execute = staticmethod(capture)
|
||||
shot = self._shot()
|
||||
shot["frames"] = torch.full((22, 8, 8, 3), 0.5)
|
||||
mask = torch.zeros(22, 8, 8)
|
||||
mask[:, 2:4, 2:4] = 1
|
||||
out = SL.BFSShotH3Conditioning().condition(shot, "clip", "vae", "none", True, "none", "match", duet="canvas",
|
||||
task="character swap", setting_ref="on (generation size)",
|
||||
setting_mask=mask)
|
||||
refs = list(seen["ref_images"].values())
|
||||
out = SL.BFSShotH3Conditioning().condition(self._shot(), "clip", "vae", "none", "none", "match",
|
||||
setting_ref="on (generation size)", setting_mask=mask)
|
||||
refs = list(self.seen["ref_images"].values())
|
||||
self.assertEqual(len(refs), 2)
|
||||
pic = refs[-1][0]
|
||||
self.assertTrue(((pic[3, 3] == 0) | (pic[3, 3] == 1)).all()) # static inside the (grown) mask
|
||||
@@ -474,10 +497,6 @@ class ConditioningWriterTest(unittest.TestCase):
|
||||
self.assertIn("<Picture 2> shows the setting", out[3])
|
||||
self.assertEqual(SL.add_setting("a {setting} b", 3, True), "a <Picture 3> b")
|
||||
|
||||
def test_task_needs_duet(self):
|
||||
with self.assertRaises(ValueError):
|
||||
self._run(task="style", instruction="anime")
|
||||
|
||||
|
||||
class SimpleConditioningTest(unittest.TestCase):
|
||||
def test_exposes_only_the_common_controls(self):
|
||||
@@ -501,13 +520,13 @@ class SimpleConditioningTest(unittest.TestCase):
|
||||
SL.BFSShotH3Conditioning.condition = fake
|
||||
try:
|
||||
out = SL.BFSShotH3ConditioningSimple().condition(
|
||||
"shot", "clip", "vae", SL.BFSShotH3ConditioningSimple.GUIDE_MODES[0], True, "match",
|
||||
"shot", "clip", "vae", SL.BFSShotH3ConditioningSimple.GUIDE_MODES[0], "match",
|
||||
audio_vae="audio vae", with_audio=True)
|
||||
finally:
|
||||
SL.BFSShotH3Conditioning.condition = original
|
||||
|
||||
self.assertEqual(out, ("positive", "latent", "prompt"))
|
||||
self.assertEqual(seen["args"][5], "none")
|
||||
self.assertEqual(seen["args"][4], "none")
|
||||
self.assertTrue(seen["kwargs"]["with_audio"])
|
||||
self.assertEqual(seen["kwargs"]["inpaint"], SL.INPAINT_MODES[0])
|
||||
|
||||
@@ -796,7 +815,7 @@ class ConditionFlowTest(unittest.TestCase):
|
||||
orig = SL.shot_mask
|
||||
SL.shot_mask = lambda *a: {"masks": m.to(torch.uint8)}
|
||||
try:
|
||||
return SL.BFSShotH3Conditioning().condition(shot, None, VAE(), SL.BFSShotH3Conditioning.GUIDE_MODES[0], False,
|
||||
return SL.BFSShotH3Conditioning().condition(shot, None, VAE(), SL.BFSShotH3Conditioning.GUIDE_MODES[0],
|
||||
"none", "match", **kw)
|
||||
finally:
|
||||
SL.shot_mask = orig
|
||||
@@ -824,6 +843,41 @@ class ConditionFlowTest(unittest.TestCase):
|
||||
self.assertEqual(len(self.calls["guides"]), 1)
|
||||
self.assertAlmostEqual(float(self.calls["guides"][0][0, 2, 2, 0]), 0.5) # it is the masked one
|
||||
|
||||
def test_passes_every_reference_in_order(self):
|
||||
refs = [torch.full((1, 32, 32, 3), k / 12) for k in range(12)]
|
||||
shot, _ = self._shot(inpaint=False)
|
||||
shot.pop("mask_src")
|
||||
shot.update(refs=refs, ref=refs[0], ref2=refs[1], ref_names=[f"r{k}.png" for k in range(12)],
|
||||
prompt="<Picture 12> is the {details@Picture 12}.")
|
||||
out = SL.BFSShotH3Conditioning().condition(shot, None, None, SL.BFSShotH3Conditioning.GUIDE_MODES[3],
|
||||
"none", "match", inpaint="off")
|
||||
kw = self.calls["r2v"][0]
|
||||
self.assertEqual(list(kw["ref_images"]), [f"ref_image_{k}" for k in range(12)])
|
||||
self.assertIs(kw["ref_images"]["ref_image_11"], refs[11])
|
||||
self.assertEqual(out[3], "<Picture 12> is the character.") # no VLM and no saved description
|
||||
self.assertNotIn("use_ref_2", SL.BFSShotH3Conditioning.INPUT_TYPES()["required"])
|
||||
self.assertNotIn("use_ref_2", SL.BFSShotH3ConditioningSimple.INPUT_TYPES()["required"])
|
||||
|
||||
def test_every_marked_person_in_their_picture_color(self):
|
||||
refs = [torch.zeros(1, 32, 32, 3) for _ in range(3)]
|
||||
shot, m = self._shot(inpaint=False)
|
||||
marks = [{"picture": 1, "color": "#00ff00"}, {"picture": 3, "color": "#ff2020", "text": "the man", "points": [], "key": 0}]
|
||||
shot.update(refs=refs, color_guide=True, color_guide_marks=marks, color_guide_color="#00ff00", color_guide_opacity=0.5)
|
||||
second = torch.zeros(22, 64, 64); second[:, 0:10, 0:10] = 1
|
||||
orig = SL.shot_mask
|
||||
SL.shot_mask = lambda path, a, start, length, spec: {"masks": second if spec.get("text") == "the man" else m.to(torch.uint8)}
|
||||
try:
|
||||
SL.BFSShotH3Conditioning().condition(shot, None, None, SL.BFSShotH3Conditioning.GUIDE_MODES[0],
|
||||
"none", "match", inpaint="off")
|
||||
guide = self.calls["guides"][0]
|
||||
self.assertGreater(float(guide[0, 5, 5, 0]), 0.75) # the second person, in red
|
||||
self.assertGreater(float(guide[0, 30, 30, 1]), 0.75) # the selection, in green
|
||||
with self.assertRaisesRegex(ValueError, "Picture 3"):
|
||||
SL.BFSShotH3Conditioning().condition(dict(shot, refs=refs[:2]), None, None,
|
||||
SL.BFSShotH3Conditioning.GUIDE_MODES[0], "none", "match", inpaint="off")
|
||||
finally:
|
||||
SL.shot_mask = orig
|
||||
|
||||
def test_ignored_outside_mask_only(self):
|
||||
shot, m = self._shot(inpaint=False)
|
||||
self.run_cond(shot, m, inpaint=SL.INPAINT_MODES[0], mask_guide=SL.MASK_GUIDES[3])
|
||||
@@ -834,6 +888,252 @@ class ConditionFlowTest(unittest.TestCase):
|
||||
self.assertNotIn("ref_videos", self.calls["r2v"][0])
|
||||
|
||||
|
||||
class MultiRefTest(unittest.TestCase):
|
||||
"""Up to 12 reference pictures per shot (plans before 1.113 had ref / ref2)."""
|
||||
|
||||
def test_old_plans_become_a_list_and_keep_ref_and_ref2(self):
|
||||
p = SL._load_plan('{"global_ref": "a.png", "global_ref2": "b.png"}')
|
||||
self.assertEqual(p["global_refs"], ["a.png", "b.png"])
|
||||
p = SL._load_plan('{"global_refs": ["a.png", "", "c.png", ""], "global_ref": "stale.png"}')
|
||||
self.assertEqual(p["global_refs"], ["a.png", "", "c.png"])
|
||||
self.assertEqual((p["global_ref"], p["global_ref2"]), ("a.png", ""))
|
||||
self.assertEqual(len(SL.ref_list([f"{k}.png" for k in range(20)])), SL.MAX_REFS)
|
||||
|
||||
def test_a_shot_slot_falls_back_to_the_global_one_and_gaps_close(self):
|
||||
self.assertEqual(SL.shot_ref_names(["", "x.png"], ["a.png", "b.png", "c.png"]), ["a.png", "x.png", "c.png"])
|
||||
self.assertEqual(SL.shot_ref_names(["", "", "z.png"], ["a.png"]), ["a.png", "z.png"])
|
||||
|
||||
def test_signature_of_a_two_reference_shot_is_unchanged(self):
|
||||
seg = {"start": 0, "end": 10, "ref": "a.png", "ref2": "b.png", "prompt": ""}
|
||||
plan = dict(SL.DEFAULT_PLAN, global_ref="g.png")
|
||||
old = SL.shot_signature(plan, seg, 0, {})
|
||||
new = SL.shot_signature(dict(plan, global_refs=["g.png"]), dict(seg, refs=["a.png", "b.png"]), 0, {})
|
||||
self.assertEqual(old, new)
|
||||
three = SL.shot_signature(dict(plan, global_refs=["g.png"]), dict(seg, refs=["a.png", "b.png", "c.png"]), 0, {})
|
||||
self.assertNotEqual(old, three)
|
||||
|
||||
def test_picture_details(self):
|
||||
names, imgs = ["a.png", "b.png"], [torch.zeros(1, 4, 4, 3)] * 2
|
||||
text = "<Subject 1> is the {details@Picture 2} shown in <Picture 2>, {details@Picture1}."
|
||||
out = SL.fill_picture_details(text, names, imgs, {"b.png": "the tall man"})
|
||||
self.assertEqual(out, "<Subject 1> is the tall man shown in <Picture 2>, character.")
|
||||
kept = SL.fill_picture_details(text, names, imgs, {"b.png": "tall man"}, keep_missing=True)
|
||||
self.assertIn("{details@Picture1}", kept)
|
||||
self.assertEqual(SL.fill_picture_details("{details@Picture 5}", names, imgs, {}), "character")
|
||||
self.assertEqual(SL.prompt_slot(text, "{details@Picture 2}"),
|
||||
"<Subject 1> is the ___ shown in <Picture 2>, the subject of <Picture 1>.")
|
||||
self.assertEqual(SL.picture_marker(text, 1), "{details@Picture1}")
|
||||
self.assertEqual(SL.expand_two_ref_prompt(text), text) # the prompt already says what <Picture 2> is
|
||||
|
||||
def test_details_and_details2_are_pictures_1_and_2(self):
|
||||
self.assertEqual(SL.picture_markers("{details} and {details2}"), "{details@Picture 1} and {details@Picture 2}")
|
||||
old = {"ref_details": {"a.png|b.png": {"details": "goblin", "details2": "robot"}, "c.png|": "cat"},
|
||||
"picture_details": {"b.png": "tall robot"}}
|
||||
self.assertEqual(SL.saved_picture_details(old), {"a.png": "goblin", "b.png": "tall robot", "c.png": "cat"})
|
||||
|
||||
def test_a_picture_can_have_its_own_preset(self):
|
||||
cfg = SL.vlm_cfg({"describe_preset": "short"})
|
||||
plan = {"picture_cfg": {"a.png": {"preset": "custom", "custom": "Describe the outfit only."}}}
|
||||
self.assertEqual(SL.picture_cfg(cfg, plan, "a.png")["describe_preset"], "custom")
|
||||
self.assertIn("outfit only", SL.describe_instruction(SL.picture_cfg(cfg, plan, "a.png")))
|
||||
self.assertIs(SL.picture_cfg(cfg, plan, "b.png"), cfg)
|
||||
|
||||
def test_colored_mark_regions_follow_the_pictures(self):
|
||||
legacy = {"global_color_guide_ref1": "#168cff", "global_color_guide_ref2": "#9b30ff"}
|
||||
marks = SL.color_marks(legacy, {"color_guide_text2": "the man", "color_guide_key2": 4})
|
||||
self.assertEqual([(x["picture"], x["color"]) for x in marks], [(1, "#168cff"), (2, "#9b30ff")])
|
||||
self.assertEqual(marks[1]["text"], "the man")
|
||||
plan = {"picture_colors": ["#00ff00"]}
|
||||
marks = SL.color_marks(plan, {"color_guide_picture": 2, "color_guide_regions": [
|
||||
{"picture": 5, "text": "the dog"}, {"picture": 3}]}) # a region with neither text nor points is ignored
|
||||
self.assertEqual([(x["picture"], x["color"]) for x in marks], [(2, "#9b30ff"), (5, "#ffe600")])
|
||||
text = SL.color_guide_prompt("A scene.", marks)
|
||||
self.assertIn("marked in purple becomes the character from <Picture 2>", text)
|
||||
self.assertIn("marked in yellow becomes the character from <Picture 5>", text)
|
||||
one = SL.color_guide_prompt("", marks[:1])
|
||||
self.assertIn("Replace only the person marked in purple with the character from <Picture 2>", one)
|
||||
|
||||
def test_auto_prompt_asks_with_the_official_format(self):
|
||||
seen = {}
|
||||
|
||||
def fake(clip, prompt, images, max_tokens, *a, **k):
|
||||
seen.update(prompt=prompt, n=images.shape[0], tokens=max_tokens)
|
||||
return "```\n**subject_definitions:**\n<Subject 1> is the robot in <Picture 2>.\nsummary:\n[video editing] x```"
|
||||
old, SL.vlm_generate = SL.vlm_generate, fake
|
||||
try:
|
||||
seg = {"color_guide": True, "color_guide_marks": [{"picture": 2, "color": "#9b30ff"}], "mask": {"text": "the man"},
|
||||
"inpaint": True}
|
||||
frame, pic = torch.zeros(1, 8, 8, 3), torch.zeros(1, 4, 4, 3)
|
||||
out = SL.write_h3_prompt(None, [frame] * 3, [0, 0.5, 1.0], [pic, pic], ["", "a silver robot"],
|
||||
"swap the man for the robot", seg, "video", 1.5)
|
||||
finally:
|
||||
SL.vlm_generate = old
|
||||
self.assertEqual(seen["n"], 5) # 3 frames + 2 pictures
|
||||
for want in ("swap the man for the robot", "<Picture 2>: image 5, described as: a silver robot",
|
||||
"tinted purple becomes the subject from <Picture 2>", "Only the selected subject is generated again",
|
||||
"edited version of <Video 1>", "retention_analysis:", "non_diegetic_music:", "00:00.500",
|
||||
"Never write a timestamp inside a shot", "is replaced by <Subject N>"):
|
||||
self.assertIn(want, seen["prompt"])
|
||||
self.assertTrue(out.startswith("subject_definitions:\n<Subject 1>")) # fences and bold removed
|
||||
self.assertIn("marked in purple", out) # the mapping the writer left out is added
|
||||
seg2 = dict(seg, prompt="")
|
||||
self.assertEqual(SL.auto_prompt_for("x", ["a.png"], seg2), SL.auto_prompt_for("x", ["a.png"], dict(seg2)))
|
||||
self.assertNotEqual(SL.auto_prompt_for("x", ["a.png"], seg2), SL.auto_prompt_for("y", ["a.png"], seg2))
|
||||
self.assertEqual(SL.auto_frame_count(24, 24.0), 3)
|
||||
self.assertEqual(SL.auto_frame_count(240, 24.0), 8)
|
||||
|
||||
def test_a_template_request_keeps_its_trigger_and_wording(self):
|
||||
seen = {}
|
||||
|
||||
def fake(clip, prompt, images, max_tokens, *a, **k):
|
||||
seen["prompt"] = prompt
|
||||
return "subject_definitions: <Subject 1> is the woman in Picture 1. detailed_description: [Shot 1] She walks."
|
||||
old, SL.vlm_generate = SL.vlm_generate, fake
|
||||
template = ("body_swap: subject_definitions: <Subject 1> is the {details@Picture 1} shown in <Picture 1>. "
|
||||
"summary: [video generation + reference] Replace the guide subject. detailed_description: [Shot 1] x")
|
||||
try:
|
||||
out = SL.write_h3_prompt(None, [torch.zeros(1, 8, 8, 3)], [0.0], [torch.zeros(1, 4, 4, 3)], [""], template,
|
||||
{}, "guide")
|
||||
finally:
|
||||
SL.vlm_generate = old
|
||||
self.assertIn("prompt TEMPLATE", seen["prompt"])
|
||||
self.assertIn("'body_swap'", seen["prompt"])
|
||||
self.assertIn("stands for the description of <Picture n>", seen["prompt"])
|
||||
self.assertTrue(out.startswith("body_swap: subject_definitions:")) # the trigger the writer dropped
|
||||
self.assertIn("the woman in <Picture 1>", out) # brackets put back
|
||||
self.assertEqual(SL.request_trigger("make it rain: now"), "")
|
||||
self.assertEqual(SL.request_trigger("head_swap: subject_definitions: x"), "head_swap")
|
||||
self.assertFalse(SL.is_template("replace the bottle with the can"))
|
||||
|
||||
def test_key_frames_cover_every_piece_and_the_cuts_reach_the_writer(self):
|
||||
self.assertEqual(SL.key_frames(0, 100, [], 4), [12, 38, 62, 88])
|
||||
frames = SL.key_frames(0, 300, [43, 86, 129, 172, 215, 258], 3) # a montage: 7 pieces, one frame each at least
|
||||
self.assertEqual(len(frames), 7)
|
||||
self.assertTrue(all(any(a <= f < b for f in frames) for a, b in zip([0, 43, 86, 129, 172, 215, 258], [43, 86, 129, 172, 215, 258, 300])))
|
||||
seen = {}
|
||||
|
||||
def fake(clip, prompt, images, max_tokens, *a, **k):
|
||||
seen["prompt"] = prompt
|
||||
return ("subject_definitions:\n<Subject 1> is x.\ndetailed_description:\n[Shot 1] At 00:00.000, a woman stands. "
|
||||
"At 00:01.792, she walks.")
|
||||
old, SL.vlm_generate = SL.vlm_generate, fake
|
||||
try:
|
||||
one = SL.write_h3_prompt(None, [torch.zeros(1, 8, 8, 3)], [0.0], [], [], "Troque a mulher pela <Picture 1>", {})
|
||||
two = SL.write_h3_prompt(None, [torch.zeros(1, 8, 8, 3)], [0.0], [], [], "swap", {}, cuts=[1.792],
|
||||
template="body_swap: subject_definitions: a. summary: b. detailed_description: [Shot 1] c")
|
||||
finally:
|
||||
SL.vlm_generate = old
|
||||
self.assertIn("[Shot 1] A woman stands. She walks.", one) # no timestamps for one continuous shot
|
||||
self.assertIn('"[Shot 2] At 00:01.792, the shot cuts to ..."', seen["prompt"])
|
||||
self.assertIn("The user's request: swap", seen["prompt"])
|
||||
self.assertIn("The prompt template:\nbody_swap:", seen["prompt"])
|
||||
self.assertTrue(two.startswith("body_swap: "))
|
||||
self.assertIn("[Shot 1] A woman stands. At 00:01.792", two)
|
||||
|
||||
def test_a_looping_answer_is_cleaned_and_the_labels_are_tied(self):
|
||||
loop = " The goblin looks at the cat, and the cat looks at the goblin. The goblin then looks up." * 6
|
||||
raw = ("subject_definitions:\n<Subject 1> is the gaunt, emaciated goblin with pale skin in <Picture 1>.\n"
|
||||
"<Subject 2> is the stout goblin with red skin in <Picture 2>.\nsummary:\n[video editing] x\n"
|
||||
"retention_analysis:\n<Subject 1>: fully_preserved - x\ndetailed_description:\n[Shot 1] A goblin types."
|
||||
+ loop + " The gaunt goblin")
|
||||
self.assertIn("sentences are repeated in a loop", SL.h3_prompt_problems(raw, 2))
|
||||
t = SL.tidy_h3_prompt(raw, {}, "", False, 2)
|
||||
self.assertEqual(t.count("The goblin then looks up."), 1)
|
||||
self.assertNotIn("The gaunt goblin\n", t) # the sentence cut in the middle
|
||||
self.assertIn("overall_soundscape:", t)
|
||||
self.assertIn("non_diegetic_music:\nN/A", t)
|
||||
named = SL.name_subjects(t)
|
||||
self.assertIn("[Shot 1] <Subject 1> is the gaunt. <Subject 2> is the stout goblin with red skin.", named)
|
||||
self.assertEqual(SL.h3_prompt_problems(named, 2), [])
|
||||
|
||||
def test_caption_style_asks_for_the_training_captions_shape(self):
|
||||
seen = {}
|
||||
|
||||
def fake(clip, prompt, images, max_tokens, *a, **k):
|
||||
seen.update(prompt=prompt, tokens=max_tokens)
|
||||
return ('"Replace the dog in the video with the gaunt goblin from Picture 1. Replace the cat in the video with '
|
||||
'the stout goblin from <Picture 2>. Keep the same background and location as in the video, and maintain '
|
||||
'the motion, camera angle, and timing."')
|
||||
old, SL.vlm_generate = SL.vlm_generate, fake
|
||||
try:
|
||||
out = SL.write_h3_prompt(None, [torch.zeros(1, 8, 8, 3)], [0.0], [torch.zeros(1, 4, 4, 3)] * 2, ["", ""],
|
||||
"Troque o cão pelo <Picture 1> e o gato pelo <Picture 2>", {}, style="captions")
|
||||
finally:
|
||||
SL.vlm_generate = old
|
||||
self.assertIn("short style of a video-edit request", seen["prompt"])
|
||||
self.assertNotIn("full-reference format", seen["prompt"])
|
||||
self.assertEqual(seen["tokens"], 512)
|
||||
self.assertTrue(out.startswith("Replace the dog in the video with the gaunt goblin from <Picture 1>."))
|
||||
self.assertEqual(SL.caption_problems(out, 2), [])
|
||||
self.assertTrue(SL.caption_problems("subject_definitions: <Subject 1> is x", 1))
|
||||
|
||||
def test_a_written_prompt_is_part_of_the_shot_signature_only_with_auto_prompt_on(self):
|
||||
seg = {"start": 0, "end": 10, "ref": "", "ref2": "", "prompt": "make it rain"}
|
||||
plan = dict(SL.DEFAULT_PLAN, auto_prompts={"0-10": {"text": "A"}})
|
||||
base = SL.shot_signature(plan, seg, 0, {})
|
||||
self.assertEqual(base, SL.shot_signature(dict(SL.DEFAULT_PLAN), seg, 0, {}))
|
||||
on = dict(plan, auto_prompt={"enabled": True})
|
||||
self.assertNotEqual(SL.shot_signature(on, seg, 0, {}),
|
||||
SL.shot_signature(dict(on, auto_prompts={"0-10": {"text": "B"}}), seg, 0, {}))
|
||||
|
||||
def test_repack_replaces_the_first_two_and_keeps_the_rest(self):
|
||||
refs = [torch.zeros(1, 4, 4, 3) for _ in range(3)]
|
||||
new = torch.ones(1, 4, 4, 3)
|
||||
out = SL.BFSShotRepack().repack([dict(_shot(0, 0, 20), refs=list(refs), ref=refs[0], ref2=refs[1])],
|
||||
ref_image_2=[new])[0][0]
|
||||
self.assertTrue(torch.equal(out["refs"][1], new) and torch.equal(out["ref2"], new))
|
||||
self.assertIs(out["refs"][2], refs[2])
|
||||
|
||||
def test_the_planner_has_no_reference_inputs(self):
|
||||
self.assertNotIn("ref_image", SL.BFSShotPlanner.INPUT_TYPES()["optional"])
|
||||
|
||||
|
||||
class KeyFrameSegmentTest(unittest.TestCase):
|
||||
"""Points on several frames of a shot: each key frame starts the track up to the next one."""
|
||||
|
||||
def setUp(self):
|
||||
import json as _json
|
||||
|
||||
class Detect: # the mask of a key frame carries its first point's x (in pixels), to see who filled a frame
|
||||
@staticmethod
|
||||
def execute(model, image, positive_coords, **kw):
|
||||
return types.SimpleNamespace(args=(torch.full((1, 4, 4), _json.loads(positive_coords)[0]["x"]),))
|
||||
mod = types.ModuleType("comfy_extras.nodes_sam3"); mod.SAM3_Detect = Detect
|
||||
sys.modules.setdefault("comfy_extras", types.ModuleType("comfy_extras"))
|
||||
self._old = (sys.modules.get("comfy_extras.nodes_sam3"), SL._sam3, SL._track)
|
||||
sys.modules["comfy_extras.nodes_sam3"] = mod
|
||||
SL._sam3 = lambda: (None, None)
|
||||
self.tracks = []
|
||||
|
||||
def track(model, imgs, seed, cond, spec):
|
||||
self.tracks.append((imgs.shape[0], float(seed.flatten()[0])))
|
||||
return seed.expand(imgs.shape[0], -1, -1).clone()
|
||||
SL._track = track
|
||||
|
||||
def tearDown(self):
|
||||
mod, SL._sam3, SL._track = self._old
|
||||
if mod is None:
|
||||
sys.modules.pop("comfy_extras.nodes_sam3", None)
|
||||
else:
|
||||
sys.modules["comfy_extras.nodes_sam3"] = mod
|
||||
|
||||
def test_each_key_frame_corrects_the_mask_from_its_frame_on(self):
|
||||
pt = lambda x: [{"x": x, "y": 0.5, "label": 1}]
|
||||
spec = dict(SL.DEFAULT_MASK, keys=[{"key": 6, "points": pt(0.75)}, {"key": 2, "points": pt(0.25)}])
|
||||
out = SL.segment_frames(torch.zeros(10, 4, 4, 3), spec)
|
||||
self.assertEqual([float(out[f, 0, 0]) for f in range(10)], [1.0] * 6 + [3.0] * 4) # frames 0-5 from key 2, 6-9 from key 6
|
||||
self.assertEqual(self.tracks, [(3, 1.0), (4, 1.0), (4, 3.0)]) # back 2->0, forward 2->5, forward 6->9
|
||||
self.assertTrue(SL.spec_has_mask(dict(SL.DEFAULT_MASK, keys=[{"key": 3, "points": pt(0.5)}])))
|
||||
self.assertFalse(SL.spec_has_mask(dict(SL.DEFAULT_MASK, keys=[{"key": 3, "points": []}])))
|
||||
|
||||
def test_a_plan_with_one_key_frame_works_as_before(self):
|
||||
spec = dict(SL.DEFAULT_MASK, points=[{"x": 0.5, "y": 0.5, "label": 1}], key=4)
|
||||
self.assertEqual(SL.point_keys(spec), [(4, spec["points"])])
|
||||
out = SL.segment_frames(torch.zeros(8, 4, 4, 3), spec)
|
||||
self.assertEqual(self.tracks, [(5, 2.0), (4, 2.0)])
|
||||
self.assertEqual(float(out.sum()), 2.0 * 8 * 16)
|
||||
|
||||
|
||||
class DescribeShortTest(unittest.TestCase):
|
||||
def test_short_details_fit_inside_the_prompt(self):
|
||||
self.assertEqual(SL.tidy_details('"The green goblin in a torn leather vest."\nextra', {"describe_preset": "short"}),
|
||||
@@ -1127,17 +1427,57 @@ class ShotStoreTest(unittest.TestCase):
|
||||
with patch.object(SL, "_next_output_path", lambda prefix: os.path.join(d, prefix + "_00001.mp4")):
|
||||
out = SL.BFSShotJoin().join(imgs, shots, [0], audio=[audio], save_to=[SL.SAVE_TO[1]],
|
||||
filename_prefix=["test"])
|
||||
res = out["result"]
|
||||
res = out
|
||||
self.assertEqual(res[0].shape[0], 45) # the output still carries the video (read back)
|
||||
self.assertAlmostEqual(float(res[0][30].mean()), 0.8, places=1)
|
||||
self.assertTrue(res[4].endswith("test_00001.mp4"))
|
||||
self.assertEqual(out["ui"]["images"][0]["filename"], "test_00001.mp4") # shown on the node
|
||||
self.assertIsInstance(out, tuple) # no preview on the node: plain outputs
|
||||
cap = cv2.VideoCapture(res[4]); n = 0
|
||||
while cap.read()[0]:
|
||||
n += 1
|
||||
self.assertEqual(n, 45)
|
||||
|
||||
|
||||
class RenderFingerprintTest(unittest.TestCase):
|
||||
"""Changing what renders the shots (LoRA, seed...) must make the stored shots stale; the join's settings must not."""
|
||||
|
||||
@staticmethod
|
||||
def graph():
|
||||
return {
|
||||
"1": {"class_type": "BFSShotPlanner", "inputs": {"plan": "{\"run\": \"auto\"}"}},
|
||||
"2": {"class_type": "BFSShotUnpack", "inputs": {"shot": ["1", 0]}},
|
||||
"3": {"class_type": "BFSShotH3ConditioningSimple", "inputs": {"shot": ["2", 10], "clip": ["4", 0], "guide_mode": "aligned guide"}},
|
||||
"4": {"class_type": "CLIPLoader", "inputs": {"clip_name": "qwen3vl.safetensors", "type": "minimax"}},
|
||||
"5": {"class_type": "UNETLoader", "inputs": {"unet_name": "h3.safetensors", "weight_dtype": "default"}},
|
||||
"6": {"class_type": "LoraLoaderModelOnly", "inputs": {"model": ["5", 0], "lora_name": "swap_v1.safetensors", "strength_model": 1.0}},
|
||||
"7": {"class_type": "KSampler", "inputs": {"model": ["6", 0], "positive": ["3", 0], "seed": 42, "steps": 8}},
|
||||
"8": {"class_type": "VAEDecode", "inputs": {"samples": ["7", 0], "vae": ["9", 0]}},
|
||||
"9": {"class_type": "VAELoader", "inputs": {"vae_name": "vae.safetensors"}},
|
||||
"10": {"class_type": "BFSShotJoin", "inputs": {"images": ["8", 0], "shots": ["2", 10], "crossfade": 4}},
|
||||
"11": {"class_type": "BFSVideoCompare", "inputs": {"guide": ["10", 5], "target": ["10", 0], "gain": 2.0}},
|
||||
}
|
||||
|
||||
def test_render_changes_count_and_join_settings_do_not(self):
|
||||
base = SL.render_fingerprint(self.graph(), "1")
|
||||
self.assertTrue(base)
|
||||
self.assertEqual(SL.render_fingerprint(self.graph(), "1"), base)
|
||||
for node, key, value, changes in (("6", "lora_name", "swap_v2.safetensors", True), ("6", "strength_model", 0.8, True),
|
||||
("7", "seed", 43, True), ("3", "guide_mode", "reference video", True),
|
||||
("10", "crossfade", 8, False), ("1", "plan", "{}", False), ("11", "gain", 4.0, False)):
|
||||
g = self.graph()
|
||||
g[node]["inputs"][key] = value
|
||||
self.assertEqual(SL.render_fingerprint(g, "1") != base, changes, f"{node}.{key}")
|
||||
g = self.graph()
|
||||
del g["10"]
|
||||
self.assertEqual(SL.render_fingerprint(g, "1"), "")
|
||||
|
||||
def test_the_fingerprint_is_part_of_the_shot_signature(self):
|
||||
seg = {"start": 0, "end": 20, "prompt": "x"}
|
||||
a = SL.shot_signature({}, seg, 0, {"render": "aaa"})
|
||||
self.assertNotEqual(a, SL.shot_signature({}, seg, 0, {"render": "bbb"}))
|
||||
self.assertEqual(SL.BFSShotPlanner.INPUT_TYPES()["hidden"], {"graph": "PROMPT", "node_id": "UNIQUE_ID"})
|
||||
|
||||
|
||||
class SecondPassSizeTest(unittest.TestCase):
|
||||
def _upscaler(self, w, h, mode, val, align=32):
|
||||
# Minimax H3 Latent Upscaler (3D), from a latent of w/16 x h/16 cells
|
||||
@@ -1170,7 +1510,7 @@ class JoinWithoutFfmpegTest(unittest.TestCase):
|
||||
patch.object(SL, "_ram_available", lambda: 1000), \
|
||||
patch.object(SL, "_next_output_path", lambda prefix: os.path.join(d, prefix + "_00001.mp4")):
|
||||
out = SL.BFSShotJoin().join(imgs, shots, [0], audio=[audio], comparison=[True])
|
||||
res = out["result"]
|
||||
res = out
|
||||
self.assertTrue(os.path.getsize(res[4]) > 0 and os.path.exists(res[4][:-4] + "_comparison.mp4"))
|
||||
self.assertEqual((res[0].shape[0], res[3].shape[0], res[5].shape[0]), (45, 45, 45))
|
||||
self.assertAlmostEqual(float(res[0][30].mean()), 0.8, delta=0.05)
|
||||
@@ -1186,7 +1526,7 @@ class JoinFileMetadataTest(unittest.TestCase):
|
||||
out = SL.BFSShotJoin().join([torch.rand(22, 32, 32, 3)], [_shot(0, 0, 20, gen=22)], [0], save_to=[SL.SAVE_TO[1]],
|
||||
filename_prefix=["m"], prompt=[pr], extra_pnginfo=[{"workflow": wf}])
|
||||
tags = json.loads(subprocess.run(["ffprobe", "-v", "error", "-show_entries", "format_tags", "-of", "json",
|
||||
out["result"][4]], capture_output=True).stdout)["format"]["tags"]
|
||||
out[4]], capture_output=True).stdout)["format"]["tags"]
|
||||
self.assertEqual((json.loads(tags["workflow"]), json.loads(tags["prompt"])), (wf, pr))
|
||||
|
||||
|
||||
@@ -1198,7 +1538,7 @@ class JoinRamGuardTest(unittest.TestCase):
|
||||
with patch.object(SL, "_ram_available", lambda: 1000), \
|
||||
patch.object(SL, "_next_output_path", lambda prefix: os.path.join(d, prefix + "_00001.mp4")):
|
||||
out = SL.BFSShotJoin().join(imgs, shots, [0], comparison=[True])
|
||||
res = out["result"]
|
||||
res = out
|
||||
self.assertTrue(os.path.exists(res[4]) and os.path.exists(res[4][:-4] + "_comparison.mp4"))
|
||||
self.assertEqual(res[0].shape[0], 45) # never an empty output node
|
||||
self.assertTrue(res[0].shape[1] < 32 and res[3].shape[0] == 45) # scaled down to the (tiny) free RAM
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
"""Source blur modes: '<region> (<what stays sharp>)', SAM 3.1 or Sapiens2, blur or inverted colours.
|
||||
The segmenters are replaced by fixed masks, so this checks which pixels end up blurred, not the models."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import types
|
||||
import unittest
|
||||
|
||||
import torch
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.path.insert(0, ROOT)
|
||||
# only what the blur uses of the shot loop: the frame key and SAM 3 (a fixed 'mouth' / part mask here)
|
||||
_SL = types.ModuleType("bfs_shot_loop")
|
||||
_SL._image_key = lambda t: (tuple(t.shape), float(t.float().sum()))
|
||||
_SL.DEFAULT_MASK = {}
|
||||
_SL.SAM_CALLS = []
|
||||
|
||||
|
||||
def _sam(frames, spec):
|
||||
_SL.SAM_CALLS.append(spec["text"])
|
||||
m = torch.zeros(frames.shape[:3])
|
||||
if spec["text"] == "mouth":
|
||||
m[:, 22:25, 28:36] = 1
|
||||
else:
|
||||
m[:, 10:30, 20:44] = 1
|
||||
return m
|
||||
|
||||
|
||||
_SL.segment_frames = _sam
|
||||
sys.modules.setdefault("bfs_shot_loop", _SL)
|
||||
import bfs_sapiens as SP # noqa: E402
|
||||
import bfs_source_blur as B # noqa: E402
|
||||
|
||||
H, W = 48, 64
|
||||
FACE = (slice(10, 30), slice(20, 44))
|
||||
LIPS = (slice(22, 24), slice(28, 36))
|
||||
TEETH = (slice(24, 25), slice(28, 36))
|
||||
|
||||
|
||||
def fake_classes(frames, out_size=None, batch=4):
|
||||
c = torch.zeros(frames.shape[0], H, W, dtype=torch.uint8)
|
||||
c[:, 5:45, 10:54] = SP.ID["Upper_Clothing"]
|
||||
c[(slice(None),) + FACE] = SP.ID["Face_Neck"]
|
||||
c[(slice(None),) + LIPS] = SP.ID["Upper_Lip"]
|
||||
c[(slice(None),) + TEETH] = SP.ID["Upper_Teeth"]
|
||||
return c
|
||||
|
||||
|
||||
class SourceBlurTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
B._CACHE.clear()
|
||||
self._seg = SP.segment
|
||||
SP.segment = fake_classes
|
||||
self.frames = torch.rand(3, H, W, 3)
|
||||
|
||||
def tearDown(self):
|
||||
SP.segment = self._seg
|
||||
|
||||
def mask(self, mode, **kw):
|
||||
return B.blur_mask(self.frames, mode, segmenter=kw.pop("segmenter", "Sapiens2"), **kw)
|
||||
|
||||
def test_saved_workflows_keep_their_values(self):
|
||||
for old in ("off", "face (keep mouth)", "head + hands (keep mouth)", "person (keep mouth)"):
|
||||
self.assertIn(old, B.SOURCE_BLUR)
|
||||
self.assertEqual(B.parse("masked area (mouth too)"), ("masked area", "mouth too"))
|
||||
self.assertIsNone(B.parse("off"))
|
||||
|
||||
def test_keep_lips_and_tongue_blurs_the_teeth(self):
|
||||
m = self.mask("face (keep lips + tongue)")
|
||||
self.assertEqual(float(m[0, 15, 25]), 1.0) # face
|
||||
self.assertEqual(float(m[0, 23, 32]), 0.0) # lips stay sharp
|
||||
self.assertEqual(float(m[0, 24, 32]), 0.0) # teeth are inside the lips' small grow here...
|
||||
big = fake_classes(self.frames)
|
||||
big[:, 26:29, 28:36] = SP.ID["Lower_Teeth"] # ...so check teeth away from the lips
|
||||
SP.segment = lambda *a, **k: big
|
||||
B._CACHE.clear()
|
||||
m = self.mask("face (keep lips + tongue)")
|
||||
self.assertEqual(float(m[0, 28, 32]), 1.0)
|
||||
m = self.mask("face (keep mouth)")
|
||||
self.assertEqual(float(m[0, 28, 32]), 0.0) # keep mouth keeps the teeth too
|
||||
|
||||
def test_mouth_too_blurs_everything_in_the_region(self):
|
||||
m = self.mask("face (mouth too)")
|
||||
self.assertEqual(float(m[0, 23, 32]), 1.0)
|
||||
self.assertEqual(float(m[0, 2, 2]), 0.0) # background untouched
|
||||
|
||||
def test_masked_area_uses_the_shot_mask_without_segmenting(self):
|
||||
SP.segment = lambda *a, **k: self.fail("masked area + mouth too must not segment")
|
||||
area = torch.zeros(3, H, W)
|
||||
area[:, 0:10, 0:10] = 1
|
||||
m = self.mask("masked area (mouth too)", area=area)
|
||||
self.assertEqual(float(m[0, 5, 5]), 1.0)
|
||||
self.assertEqual(float(m[0, 20, 30]), 0.0)
|
||||
|
||||
def test_masked_area_without_a_mask_blurs_the_whole_person(self):
|
||||
m = self.mask("masked area (mouth too)")
|
||||
self.assertEqual(float(m[0, 40, 15]), 1.0) # clothing = person
|
||||
|
||||
def test_invert_colours_inside_the_mask_only(self):
|
||||
x = torch.full((3, H, W, 3), 0.2)
|
||||
out = B.blur_source(x, "face (mouth too)", 1.0, segmenter="Sapiens2", effect="invert colours")
|
||||
self.assertAlmostEqual(float(out[0, 18, 30, 0]), 0.8, places=4)
|
||||
self.assertAlmostEqual(float(out[0, 2, 2, 0]), 0.2, places=4)
|
||||
|
||||
def test_sam_keeps_the_old_behaviour(self):
|
||||
SP.segment = lambda *a, **k: self.fail("SAM 3.1 + keep mouth must not run Sapiens2")
|
||||
_SL.SAM_CALLS.clear()
|
||||
m = self.mask("face (keep mouth)", segmenter="SAM 3.1")
|
||||
self.assertEqual(_SL.SAM_CALLS, ["face", "mouth"])
|
||||
self.assertEqual(float(m[0, 15, 25]), 1.0)
|
||||
self.assertEqual(float(m[0, 23, 32]), 0.0)
|
||||
|
||||
|
||||
class SapiensSizeTest(unittest.TestCase):
|
||||
def test_input_keeps_the_area_and_the_aspect(self):
|
||||
for h, w in ((1080, 1920), (1920, 1080), (512, 512)):
|
||||
ih, iw = SP.input_size(h, w)
|
||||
self.assertEqual((ih % 16, iw % 16), (0, 0))
|
||||
self.assertAlmostEqual(iw / ih, w / h, delta=0.03) # never stretched to 1024x768
|
||||
self.assertAlmostEqual(ih * iw / SP.PIXELS, 1.0, delta=0.04)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -65,6 +65,68 @@ class CompareNodeTest(unittest.TestCase):
|
||||
n += 1
|
||||
self.assertEqual(n, 18, k)
|
||||
|
||||
def test_reference_pictures_go_to_the_player_no_taller_than_it(self):
|
||||
from PIL import Image
|
||||
guide, target = torch.rand(4, 32, 48, 3), torch.rand(4, 32, 48, 3)
|
||||
ref = torch.rand(6, 80, 40, 3) # taller than the preview
|
||||
ui = VC.BFSVideoCompare().compare(guide, target, preview_height=32, reference=ref)["ui"]["bfs_compare"][0]
|
||||
self.assertEqual([r["filename"] for r in ui["refs"]], [f"ref_{k}.png" for k in range(6)])
|
||||
im = Image.open(os.path.join(_TMP, ui["refs"][0]["subfolder"], "ref_0.png"))
|
||||
self.assertEqual(im.size, (16, 32))
|
||||
# a refs list as the planner / Unpack give it: pictures of different sizes, as ComfyUI hands a list input
|
||||
refs = [torch.rand(1, 64, 32, 3), torch.rand(1, 20, 60, 3)] + [torch.rand(1, 8, 8, 3)] * 12
|
||||
ui = VC.BFSVideoCompare().compare([guide], [target], preview_height=[32], reference=refs)["ui"]["bfs_compare"][0]
|
||||
self.assertEqual(len(ui["refs"]), 12) # at most 12
|
||||
sizes = [Image.open(os.path.join(_TMP, r["subfolder"], r["filename"])).size for r in ui["refs"][:2]]
|
||||
self.assertEqual(sizes, [(16, 32), (60, 20)])
|
||||
none = VC.BFSVideoCompare().compare(guide, target, preview_height=32)["ui"]["bfs_compare"][0]
|
||||
self.assertEqual(none["refs"], [])
|
||||
|
||||
|
||||
@unittest.skipUnless(VC.VIO.ffmpeg_bin(), "needs an ffmpeg program")
|
||||
class ExportTest(unittest.TestCase):
|
||||
"""The player's recording (WebM from MediaRecorder) converted for sharing."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
import subprocess
|
||||
cls.src = os.path.join(_TMP, "rec_src.webm")
|
||||
# 20 s of noise at 720p with sound: far above 10 MB at the browser's bitrate
|
||||
subprocess.run([VC.VIO.ffmpeg_bin(), "-y", "-v", "error", "-f", "lavfi", "-i", "nullsrc=s=1280x720:r=30:d=20,geq=random(1)*255:128:128",
|
||||
"-f", "lavfi", "-i", "sine=frequency=440:duration=20", "-c:v", "libvpx", "-b:v", "16M",
|
||||
"-deadline", "realtime", "-cpu-used", "8", "-c:a", "libopus", "-shortest", cls.src], check=True)
|
||||
|
||||
def probe(self, path):
|
||||
import json
|
||||
import subprocess
|
||||
r = subprocess.run([VC.VIO.ffmpeg_bin().replace("ffmpeg", "ffprobe"), "-v", "error", "-show_streams", "-of", "json", path],
|
||||
capture_output=True, text=True)
|
||||
return {st["codec_type"]: st for st in json.loads(r.stdout)["streams"]}
|
||||
|
||||
def test_small_mp4_is_h264_aac_faststart_under_the_limit(self):
|
||||
self.assertGreater(os.path.getsize(self.src), VC.LIMIT_BYTES)
|
||||
out = VC.encode(self.src, "small", 20.0, os.path.join(_TMP, "exp_small"))
|
||||
self.assertLessEqual(os.path.getsize(out), VC.LIMIT_BYTES)
|
||||
st = self.probe(out)
|
||||
self.assertEqual((st["video"]["codec_name"], st["video"]["pix_fmt"], st["audio"]["codec_name"]), ("h264", "yuv420p", "aac"))
|
||||
head = open(out, "rb").read(256 * 1024)
|
||||
self.assertLess(head.find(b"moov"), head.find(b"mdat") if b"mdat" in head else 1 << 30) # faststart: moov first
|
||||
|
||||
def test_gif_fits_the_limit(self):
|
||||
out = VC.encode(self.src, "gif", 20.0, os.path.join(_TMP, "exp_gif"))
|
||||
self.assertTrue(out.endswith(".gif"))
|
||||
self.assertLessEqual(os.path.getsize(out), VC.LIMIT_BYTES)
|
||||
|
||||
def test_bad_ids_and_formats_are_refused(self):
|
||||
with self.assertRaises(ValueError):
|
||||
VC.write_part("../../etc", 0, b"x")
|
||||
with self.assertRaises(ValueError):
|
||||
VC.encode(self.src, "avi", 1.0, os.path.join(_TMP, "x"))
|
||||
rid = "abcdef0123"
|
||||
VC.write_part(rid, 0, b"12345")
|
||||
VC.write_part(rid, 5, b"678")
|
||||
self.assertEqual(open(os.path.join(_TMP, "bfs_compare", "exports", rid + ".rec"), "rb").read(), b"12345678")
|
||||
|
||||
|
||||
class NoFfmpegTest(unittest.TestCase):
|
||||
"""Windows portable installs often have no ffmpeg program: everything must go through PyAV."""
|
||||
|
||||
+307
-69
@@ -11,8 +11,10 @@
|
||||
import { app } from "../../scripts/app.js";
|
||||
import { api } from "../../scripts/api.js";
|
||||
import { showGuide } from "./bfs_md_view.js";
|
||||
import { wheelToCanvas } from "./bfs_wheel.js";
|
||||
import { createApp, ref, reactive, computed, watch, nextTick, onMounted, onBeforeUnmount } from "./vendor/vue.esm-browser.prod.mjs";
|
||||
import { h, rawH, t, uiLanguage, setUiLanguage, DEFAULTS, MASK_DEFAULTS, VLM_DEFAULTS, snapUp, snapDown, segKey, hasMask, pill } from "./shotloop/common.js";
|
||||
import { h, rawH, t, uiLanguage, setUiLanguage, DEFAULTS, MASK_DEFAULTS, VLM_DEFAULTS, snapUp, snapDown, segKey, hasMask, pill,
|
||||
MAX_REFS, refList, shotRefNames, MARK_PALETTE, pictureColor, markRegions, AUTO_DEFAULTS } from "./shotloop/common.js";
|
||||
import { styles } from "./shotloop/styles.js";
|
||||
// history lengths H3's VAE encodes whole (17k+5); old plans saved 17/34/51, read as the next valid one up
|
||||
const historyLen = n => (n = Number(n) || 22) <= 5 ? 5 : 5 + Math.ceil((n - 5) / 17) * 17;
|
||||
@@ -68,7 +70,8 @@ function Panel(io) {
|
||||
const maskPrev = ref({}); // "start-end" -> {frames, box, coverage, empty} from /bfs/shotloop/mask
|
||||
const showMasks = ref(true); // overlay the mask preview on the shot cards
|
||||
const vlmSug = ref({}); // "start-end" -> {segment, shot, people, recommend, reason} from the VLM
|
||||
const modal = reactive({ open: false, idx: -1, key: 0, points: [], text: "", region: 1, groups: {}, prev: null, busy: "" });
|
||||
const modal = reactive({ open: false, idx: -1, key: 0, points: [], text: "", region: 1, groups: {}, prev: null, stale: false,
|
||||
showMask: true, playing: 0, busy: "" });
|
||||
const segPeople = ref({}); // "start-end" -> {people, main} from /bfs/shotloop/plan
|
||||
const targetCrop = ref({}); // "start-end" -> data URL of the subject the description was made from
|
||||
const tab = ref(store.get("bfs.shotloop.tab", "shots"));
|
||||
@@ -79,6 +82,18 @@ function Panel(io) {
|
||||
target: false, chain: false, color_guide: false, scope: "all" }));
|
||||
watch(copyOpts, v => store.set("bfs.shotloop.copy", { ...v }), { deep: true });
|
||||
|
||||
// one description per picture: picture_details, plus the ones plans before 1.113 saved per reference set in
|
||||
// ref_details ("a|b" -> {details, details2}, or one text for a); same rule as bfs_shot_loop.saved_picture_details
|
||||
const pictureDetailsOf = p => {
|
||||
const out = {};
|
||||
Object.entries(p.ref_details || {}).forEach(([key, v]) => {
|
||||
const [a, b] = key.split("|");
|
||||
const pairs = v && typeof v === "object" ? [[a, v.details || v.legacy], [b, v.details2]] : [[a, v]];
|
||||
pairs.forEach(([name, text]) => { if (name && text && !(name in out)) out[name] = String(text); });
|
||||
});
|
||||
Object.entries(p.picture_details || {}).forEach(([name, text]) => { if (text) out[name] = text; });
|
||||
return out;
|
||||
};
|
||||
const load = () => {
|
||||
let p = {};
|
||||
try { p = JSON.parse(io.getPlan() || "{}"); } catch { p = {}; }
|
||||
@@ -88,14 +103,34 @@ function Panel(io) {
|
||||
plan.mask_cfg = { ...MASK_DEFAULTS, ...(p.mask_cfg || {}) };
|
||||
plan.vlm_cfg = { ...VLM_DEFAULTS, ...(p.vlm_cfg || {}) };
|
||||
plan.ref_details = { ...(p.ref_details || {}) };
|
||||
plan.picture_details = { ...pictureDetailsOf(p) };
|
||||
plan.picture_cfg = { ...(p.picture_cfg || {}) };
|
||||
plan.auto_prompt = { ...AUTO_DEFAULTS, ...(p.auto_prompt || {}) };
|
||||
plan.auto_prompts = { ...(p.auto_prompts || {}) };
|
||||
if (!Array.isArray(plan.bounds)) plan.bounds = [];
|
||||
if (!Array.isArray(plan.segs)) plan.segs = [];
|
||||
// plans before 1.113 had two references (ref / ref2): now a list of up to 12 pictures
|
||||
plan.global_refs = refList(p.global_refs, p.global_ref, p.global_ref2);
|
||||
plan.segs.forEach(m => { if (m) m.refs = refList(m.refs, m.ref, m.ref2); });
|
||||
// Colored Mark: a color per picture (was: two marker colors), regions tied to a picture (was: region 2)
|
||||
if (!Array.isArray(p.picture_colors)) plan.picture_colors = [p.global_color_guide_ref1 || MARK_PALETTE[0][0], p.global_color_guide_ref2 || MARK_PALETTE[1][0]];
|
||||
plan.segs.forEach(m => {
|
||||
if (!m) return;
|
||||
m.color_guide_regions = markRegions(m);
|
||||
["color_guide_text2", "color_guide_points2", "color_guide_key2", "color_guide_color", "color_guide_color2"].forEach(k => delete m[k]);
|
||||
});
|
||||
Object.values(plan.cast).forEach(person => { if (person && typeof person === "object") { delete person.ref; delete person.ref2; delete person.guide_color; } });
|
||||
plan.segs.forEach(meta => { if (meta) { delete meta.color_guide_person; delete meta.color_guide_person2; } });
|
||||
};
|
||||
// any change of the plan (a new video, a prompt, a setting...) is a new loop run on the server: show its progress
|
||||
let progTimer = 0;
|
||||
const save = () => { io.setPlan(JSON.stringify(plan)); clearTimeout(progTimer); progTimer = setTimeout(() => progress(), 400); };
|
||||
// ref / ref2 (and global_ref / global_ref2) keep pictures 1 and 2 for older readers of the plan
|
||||
const syncRefs = () => {
|
||||
plan.global_refs = refList(plan.global_refs); [plan.global_ref = "", plan.global_ref2 = ""] = plan.global_refs;
|
||||
plan.global_color_guide_ref1 = pictureColor(plan, 1); plan.global_color_guide_ref2 = pictureColor(plan, 2);
|
||||
plan.segs.forEach(m => { if (m) { m.refs = refList(m.refs); [m.ref = "", m.ref2 = ""] = m.refs; } });
|
||||
};
|
||||
const save = () => { syncRefs(); io.setPlan(JSON.stringify(plan)); clearTimeout(progTimer); progTimer = setTimeout(() => progress(), 400); };
|
||||
|
||||
const n = computed(() => (an.value ? an.value.n : 0)); // the whole video on the timeline
|
||||
// the part the plan works on [lo, hi): start / end (0 = the end), then "max total seconds" counted from the start
|
||||
@@ -120,15 +155,14 @@ function Panel(io) {
|
||||
if (b[i + 1] <= b[i]) continue;
|
||||
const m = plan.segs[out.length] || {};
|
||||
out.push({ start: b[i], end: b[i + 1], len: b[i + 1] - b[i], gen: snapUp(b[i + 1] - b[i], plan.grid),
|
||||
cut: cs.has(b[i]), enabled: m.enabled !== false, ref: m.ref || "", ref2: m.ref2 || "", prompt: m.prompt || "",
|
||||
cut: cs.has(b[i]), enabled: m.enabled !== false, refs: refList(m.refs, m.ref, m.ref2), prompt: m.prompt || "",
|
||||
force: m.force || "auto", chain: m.chain || "head", chainFrame: m.chain_frame || "first", chainHistory: historyLen(m.chain_history),
|
||||
color_guide: !!m.color_guide, color_guide_color: m.color_guide_color || "",
|
||||
color_guide_opacity: m.color_guide_opacity ?? 0.35,
|
||||
color_guide_text2: m.color_guide_text2 || "", color_guide_color2: m.color_guide_color2 || "",
|
||||
color_guide_points2: m.color_guide_points2 || [], color_guide_key2: m.color_guide_key2,
|
||||
color_guide: !!m.color_guide, color_guide_picture: +m.color_guide_picture || 1,
|
||||
color_guide_opacity: m.color_guide_opacity ?? 0.35, regions: markRegions(m),
|
||||
crop: !!m.crop, inpaint: !!m.inpaint, paste: !!m.paste, pasteText: m.paste_text || "", strength: m.strength ?? 1, mask: m.mask || {}, target: m.target || "",
|
||||
extMask: !!plan.mask_video || !!(plan.track?.points?.length), trackMask: !!(plan.track?.points?.length) });
|
||||
}
|
||||
out.forEach(s => { s.names = shotRefNames(s.refs, plan.global_refs); [s.ref = "", s.ref2 = ""] = s.refs; });
|
||||
if (plan.max_parts > 0) out = out.slice(0, plan.max_parts);
|
||||
return out;
|
||||
});
|
||||
@@ -172,12 +206,16 @@ const modeOf = s => (MODES.find(m => m[2] === !!s.crop && m[3] === !!s.inpaint &
|
||||
const modeName = s => MODES.find(m => m[0] === modeOf(s))[1];
|
||||
|
||||
const meta = i => { while (plan.segs.length <= i) plan.segs.push({}); return plan.segs[i]; };
|
||||
// the reference a shot uses: its own or the global one
|
||||
const refOf = (s, k) => s[k] || plan["global_" + k] || "";
|
||||
// the pictures a shot uses: "ref" = <Picture 1>, "ref2" = <Picture 2> (its own or the global ones)
|
||||
const refOf = (s, k) => s.names[k === "ref2" ? 1 : 0] || "";
|
||||
const promptOf = s => s.prompt || plan.global_prompt || "";
|
||||
const detailsOf = s => {
|
||||
const value = plan.ref_details?.[`${refOf(s, "ref")}|${refOf(s, "ref2")}`] || "";
|
||||
return typeof value === "object" ? Object.values(value).some(Boolean) : !!value;
|
||||
// the pictures a prompt describes ({details} = 1, {details2} = 2, {details@Picture n}) all have a description
|
||||
const detailsOf = (s, prompt) => {
|
||||
const want = new Set();
|
||||
if (prompt.includes("{details}")) want.add(1);
|
||||
if (prompt.includes("{details2}")) want.add(2);
|
||||
for (const m of prompt.matchAll(/\{details@Picture\s*(\d+)\}/g)) want.add(+m[1]);
|
||||
return [...want].every(k => !s.names[k - 1] || plan.picture_details?.[s.names[k - 1]]);
|
||||
};
|
||||
|
||||
// things worth fixing before a run, per shot (only shots that run)
|
||||
@@ -221,11 +259,11 @@ const meta = i => { while (plan.segs.length <= i) plan.segs.push({}); return pla
|
||||
if ((s.crop || s.inpaint || s.paste) && !hasMask(s)) add("warn", `${modeName(s)} needs a mask (text, points or a mask video; or the planner's mask input)`);
|
||||
if (hasMask(s) && maskPrev.value[segKey(s)]?.empty) add("warn", "the mask preview found nothing");
|
||||
if (/\{target\}/.test(p) && !s.target) add("warn", "the prompt uses {target} but the shot has no target description");
|
||||
if (/\{details2?\}/.test(p) && !detailsOf(s) && !plan.vlm_cfg?.enabled) add("info", "the prompt uses {details} but its references have no description yet");
|
||||
if (/\{details/.test(p) && !detailsOf(s, p) && !plan.vlm_cfg?.enabled) add("info", "the prompt uses {details} but its pictures have no description yet");
|
||||
if (/\{details2?\}/.test(p) && (plan.vlm_cfg?.describe_preset || "short") !== "short")
|
||||
add("warn", "{details} sits inside a sentence of the prompt: use the 'short' describe preset (Prompts & refs), the other presets write a paragraph");
|
||||
if (!p) add("info", "no prompt (fine when the node's prompt input is connected)");
|
||||
if (!refOf(s, "ref")) add("info", "no reference (fine when the node's ref_image input is connected)");
|
||||
if (!refOf(s, "ref")) add("info", "no reference picture");
|
||||
if (vram.max_tokens && shotTokens(s, i) > 1.3 * vram.max_tokens)
|
||||
add("warn", "much bigger than any shot that ran on this GPU (over 1.3×): it may run out of memory. Lower Megapixels or Max seconds / shot, use head 5, or set Between shots = free memory (Run tab)");
|
||||
if (i > 0 && plan.run === "auto" && (s.chain === "head" || s.chain === "history") && !s.cut)
|
||||
@@ -257,7 +295,9 @@ const meta = i => { while (plan.segs.length <= i) plan.segs.push({}); return pla
|
||||
if (!plan.video) return;
|
||||
busy.value = "Detecting cuts…"; error.value = "";
|
||||
try {
|
||||
const body = { plan: { ...plan, bounds: apply ? [] : plan.bounds } };
|
||||
// every detected boundary is kept (Max shots only picks how many shots run): asked for without the limit, the
|
||||
// last kept shot would otherwise have no end and run to the end of the range
|
||||
const body = { plan: { ...plan, bounds: apply ? [] : plan.bounds, max_parts: apply ? 0 : plan.max_parts } };
|
||||
const r = await api.fetchApi("/bfs/shotloop/plan", { method: "POST", body: JSON.stringify(body) });
|
||||
const j = await r.json();
|
||||
if (j.error) throw new Error(j.error);
|
||||
@@ -266,12 +306,11 @@ const meta = i => { while (plan.segs.length <= i) plan.segs.push({}); return pla
|
||||
.map(s => [`${s.start}-${s.end}`, { people: s.people, main: s.main }]));
|
||||
if (apply) {
|
||||
plan.bounds = j.segs.slice(1).map(s => s.start);
|
||||
const keep = plan.segs; plan.segs = j.segs.map((_, i) => ({ enabled: true, ref: keep[i]?.ref || "", ref2: keep[i]?.ref2 || "", prompt: keep[i]?.prompt || "",
|
||||
const keep = plan.segs; plan.segs = j.segs.map((_, i) => ({ enabled: true, refs: refList(keep[i]?.refs), prompt: keep[i]?.prompt || "",
|
||||
chain: keep[i]?.chain || "head", chain_frame: keep[i]?.chain_frame || "first", chain_history: historyLen(keep[i]?.chain_history),
|
||||
crop: !!keep[i]?.crop, inpaint: !!keep[i]?.inpaint, paste: !!keep[i]?.paste, paste_text: keep[i]?.paste_text || "", strength: keep[i]?.strength ?? 1, mask: keep[i]?.mask || {}, target: keep[i]?.target || "",
|
||||
color_guide: !!keep[i]?.color_guide, color_guide_color: keep[i]?.color_guide_color || "", color_guide_opacity: keep[i]?.color_guide_opacity ?? 0.35,
|
||||
color_guide_text2: keep[i]?.color_guide_text2 || "", color_guide_color2: keep[i]?.color_guide_color2 || "",
|
||||
color_guide_points2: keep[i]?.color_guide_points2 || [], color_guide_key2: keep[i]?.color_guide_key2 ?? 0 }));
|
||||
color_guide: !!keep[i]?.color_guide, color_guide_picture: keep[i]?.color_guide_picture || 1, color_guide_opacity: keep[i]?.color_guide_opacity ?? 0.35,
|
||||
color_guide_regions: (keep[i]?.color_guide_regions || []).map(r => ({ ...r })) }));
|
||||
sel.value = 0; save();
|
||||
}
|
||||
} catch (e) { error.value = String(e.message || e); }
|
||||
@@ -314,15 +353,17 @@ const meta = i => { while (plan.segs.length <= i) plan.segs.push({}); return pla
|
||||
} catch (e) { error.value = String(e.message || e); }
|
||||
busy.value = "";
|
||||
}
|
||||
// intoModal: every frame of the shot, for the points window's player (count 0)
|
||||
async function previewMask(i, spec, intoModal = false) {
|
||||
const s = segs.value[i]; if (!s) return;
|
||||
const label = "Segmenting with SAM 3… (the first time downloads/loads the model)";
|
||||
if (intoModal) { modal.busy = label; busySince.value = Date.now(); Object.assign(status, { stage: "", done: 0, total: 0 }); } else busy.value = label;
|
||||
error.value = "";
|
||||
try {
|
||||
const r = await api.fetchApi("/bfs/shotloop/mask", { method: "POST", body: JSON.stringify({ plan: { ...plan }, index: i, mask: spec || null }) });
|
||||
const body = { plan: { ...plan }, index: i, mask: spec || null, ...(intoModal ? { count: 0 } : {}) };
|
||||
const r = await api.fetchApi("/bfs/shotloop/mask", { method: "POST", body: JSON.stringify(body) });
|
||||
const j = await r.json(); if (j.error) throw new Error(j.error);
|
||||
if (intoModal) modal.prev = j; else maskPrev.value = { ...maskPrev.value, [segKey(s)]: j };
|
||||
if (intoModal) { modal.prev = j; modal.stale = false; modal.showMask = true; } else maskPrev.value = { ...maskPrev.value, [segKey(s)]: j };
|
||||
} catch (e) { error.value = String(e.message || e); }
|
||||
if (intoModal) modal.busy = ""; else busy.value = "";
|
||||
}
|
||||
@@ -333,33 +374,84 @@ const meta = i => { while (plan.segs.length <= i) plan.segs.push({}); return pla
|
||||
if (s && maskPrev.value[segKey(s)]) { const mp = { ...maskPrev.value }; delete mp[segKey(s)]; maskPrev.value = mp; }
|
||||
};
|
||||
const setMaskCfg = (k, v) => { plan.mask_cfg = { ...plan.mask_cfg, [k]: v }; save(); maskPrev.value = {}; };
|
||||
// The points window. A selection (region 1 = the shot's own mask, 2.. = its extra Colored Mark regions) has key
|
||||
// frames, each with its own points: SAM 3 starts from a key frame and tracks up to the next one, so points added on a
|
||||
// later frame correct the mask from there on. modal.key is the frame shown, modal.points its points (being edited).
|
||||
const keyFramesOf = m => (Array.isArray(m?.keys) && m.keys.length ? m.keys.map(k => ({ key: +k.key || 0, points: [...(k.points || [])] }))
|
||||
: (m?.points || []).length ? [{ key: +m.key || 0, points: [...m.points] }] : []);
|
||||
const pointGroup = () => modal.groups[modal.region];
|
||||
const keepFrame = () => { // the frame being edited back into its selection's key frames
|
||||
const g = pointGroup(); if (!g) return;
|
||||
g.keys = g.keys.filter(k => k.key !== modal.key);
|
||||
if (modal.points.length) g.keys = [...g.keys, { key: modal.key, points: [...modal.points] }].sort((a, b) => a.key - b.key);
|
||||
g.text = modal.text;
|
||||
};
|
||||
const showFrame = f => {
|
||||
keepFrame();
|
||||
modal.key = Math.max(0, Math.min((segs.value[modal.idx]?.len || 1) - 1, f));
|
||||
modal.points = [...(pointGroup()?.keys.find(k => k.key === modal.key)?.points || [])];
|
||||
};
|
||||
const dropKeyFrame = f => {
|
||||
const g = pointGroup(); g.keys = g.keys.filter(k => k.key !== f);
|
||||
if (f === modal.key) modal.points = [];
|
||||
modal.stale = !!modal.prev;
|
||||
};
|
||||
const openPoints = (i, region = 1) => {
|
||||
const s = segs.value[i]; if (!s) return;
|
||||
const g1 = { key: s.mask.key ?? Math.floor(s.len / 2), points: [...(s.mask.points || [])], text: s.mask.text || "" };
|
||||
const g2 = { key: g1.key, points: [...(s.color_guide_points2 || [])], text: s.color_guide_text2 || "" };
|
||||
Object.assign(modal, { open: true, idx: i, region: s.color_guide ? region : 1, groups: { 1: g1, 2: g2 },
|
||||
key: (region === 2 && s.color_guide ? g2 : g1).key, points: [...((region === 2 && s.color_guide ? g2 : g1).points)],
|
||||
text: (region === 2 && s.color_guide ? g2 : g1).text, prev: null, busy: "" });
|
||||
const groups = { 1: { keys: keyFramesOf(s.mask), text: s.mask.text || "" } };
|
||||
if (s.color_guide) s.regions.forEach((r, j) => { groups[j + 2] = { keys: keyFramesOf(r), text: r.text || "" }; });
|
||||
const r0 = groups[region] ? region : 1, key = groups[r0].keys[0]?.key ?? Math.floor(s.len / 2);
|
||||
Object.assign(modal, { open: true, idx: i, region: r0, groups, key, points: [...(groups[r0].keys[0]?.points || [])],
|
||||
text: groups[r0].text, prev: null, stale: false, showMask: true, busy: "" });
|
||||
};
|
||||
const switchPointRegion = region => {
|
||||
const old = modal.groups[modal.region] || {};
|
||||
modal.groups[modal.region] = { key: modal.key, points: [...modal.points], text: modal.text };
|
||||
const next = modal.groups[region] || { key: modal.key, points: [], text: "" };
|
||||
modal.region = region; modal.points = [...next.points]; modal.text = next.text; modal.prev = null;
|
||||
keepFrame();
|
||||
modal.region = region; modal.text = pointGroup()?.text || "";
|
||||
modal.points = [...(pointGroup()?.keys.find(k => k.key === modal.key)?.points || [])]; modal.prev = null;
|
||||
};
|
||||
// the selection being edited as a mask spec (what Segment sends)
|
||||
const pointSpec = () => { keepFrame(); const g = pointGroup(); return { keys: g.keys, points: g.keys[0]?.points || [], key: g.keys[0]?.key ?? modal.key, text: g.text }; };
|
||||
const commitPointGroups = i => {
|
||||
const m = meta(i), g1 = modal.groups[1] || {}, g2 = modal.groups[2] || {};
|
||||
m.mask = { ...(m.mask || {}), points: [...(modal.region === 1 ? modal.points : g1.points || [])],
|
||||
key: modal.region === 1 ? modal.key : g1.key, text: modal.region === 1 ? modal.text : g1.text };
|
||||
keepFrame();
|
||||
const m = meta(i), g1 = modal.groups[1];
|
||||
// keys is what 1.116 reads; points + key (the first key frame) for older versions
|
||||
m.mask = { ...(m.mask || {}), keys: g1.keys, points: g1.keys[0]?.points || [], key: g1.keys[0]?.key ?? 0, text: g1.text };
|
||||
if (segs.value[i]?.color_guide) {
|
||||
m.color_guide_points2 = [...(modal.region === 2 ? modal.points : g2.points || [])];
|
||||
m.color_guide_key2 = modal.region === 2 ? modal.key : (g1.key ?? modal.key);
|
||||
m.color_guide_text2 = modal.region === 2 ? modal.text : g2.text || "";
|
||||
m.color_guide_regions = markRegions(m).map((r, j) => {
|
||||
const g = modal.groups[j + 2];
|
||||
return g ? { ...r, keys: g.keys, points: g.keys[0]?.points || [], key: g.keys[0]?.key ?? 0, text: g.text } : r;
|
||||
});
|
||||
}
|
||||
};
|
||||
// the picture a region becomes (region 1 = the shot's own selection) and the extra regions
|
||||
const setMarkPicture = (i, region, k) => {
|
||||
const m = meta(i);
|
||||
if (region === 1) m.color_guide_picture = k;
|
||||
else m.color_guide_regions = markRegions(m).map((r, j) => (j === region - 2 ? { ...r, picture: k } : r));
|
||||
save();
|
||||
};
|
||||
const addMarkRegion = i => {
|
||||
const m = meta(i), used = new Set([+m.color_guide_picture || 1, ...markRegions(m).map(r => r.picture)]);
|
||||
const n = Math.max(2, segs.value[i]?.names.length || 0);
|
||||
let k = 1; while (used.has(k) && k < n) k++;
|
||||
m.color_guide_regions = [...markRegions(m), { picture: k, text: "", points: [], key: 0 }]; save();
|
||||
};
|
||||
const setMarkRegion = (i, j, field, v) => {
|
||||
const m = meta(i); m.color_guide_regions = markRegions(m).map((r, q) => (q === j ? { ...r, [field]: v } : r)); save();
|
||||
};
|
||||
const removeMarkRegion = i => j => { const m = meta(i); m.color_guide_regions = markRegions(m).filter((_, q) => q !== j); save(); };
|
||||
const setPictureColor = (k, color) => {
|
||||
const list = [...(plan.picture_colors || [])];
|
||||
while (list.length < k) list.push(pictureColor(plan, list.length + 1));
|
||||
list[k - 1] = color; plan.picture_colors = list; save();
|
||||
};
|
||||
const savePoints = () => {
|
||||
const i = modal.idx; commitPointGroups(i); save();
|
||||
if (modal.prev) maskPrev.value = { ...maskPrev.value, [segKey(segs.value[modal.idx])]: modal.prev };
|
||||
if (modal.prev && !modal.stale) { // the card's strip shows 6 of the window's frames
|
||||
const fr = modal.prev.frames, pick = [...new Set(Array.from({ length: Math.min(6, fr.length) }, (_, k) => Math.round(k * (fr.length - 1) / Math.max(1, Math.min(6, fr.length) - 1))))];
|
||||
maskPrev.value = { ...maskPrev.value, [segKey(segs.value[modal.idx])]: { ...modal.prev, frames: pick.map(k => fr[k]) } };
|
||||
}
|
||||
clearInterval(modal.playing); modal.playing = 0;
|
||||
modal.open = false;
|
||||
};
|
||||
const targets = (scope, from) => segs.value.map((_, i) => i).filter(i => i !== from && (scope === "all" || i > from));
|
||||
@@ -369,7 +461,8 @@ const meta = i => { while (plan.segs.length <= i) plan.segs.push({}); return pla
|
||||
const src = segs.value[fromIdx]; const rel = src ? key / Math.max(1, src.len - 1) : 0.5;
|
||||
[fromIdx, ...targets(scope, fromIdx)].forEach(i => {
|
||||
const x = segs.value[i], m = meta(i);
|
||||
m.mask = { ...(m.mask || {}), points: points.map(p => ({ ...p })), key: Math.round(rel * Math.max(0, x.len - 1)), text: text ?? (m.mask || {}).text };
|
||||
const key = Math.round(rel * Math.max(0, x.len - 1));
|
||||
m.mask = { ...(m.mask || {}), keys: [{ key, points: points.map(p => ({ ...p })) }], points: points.map(p => ({ ...p })), key, text: text ?? (m.mask || {}).text };
|
||||
});
|
||||
maskPrev.value = {}; save();
|
||||
};
|
||||
@@ -382,20 +475,20 @@ const meta = i => { while (plan.segs.length <= i) plan.segs.push({}); return pla
|
||||
// part of that track, so the subject is found wherever it moves (copied points can fall on the background)
|
||||
const saveTrack = () => {
|
||||
const s0 = segs.value[modal.idx]; if (!s0 || !modal.points.length) return;
|
||||
const own = segs.value.filter((x, i) => i !== modal.idx && (x.mask.text || (x.mask.points || []).length)).length;
|
||||
const own = segs.value.filter((x, i) => i !== modal.idx && (x.mask.text || (x.mask.points || []).length || (x.mask.keys || []).length)).length;
|
||||
if (own && !window.confirm(`${own} other shot(s) have their own mask text or points, which would win over the track. Clear them so every shot uses the track?`)) {
|
||||
// keep them: those shots keep their own selection
|
||||
} else {
|
||||
segs.value.forEach((_, i) => { const m = meta(i); if (m.mask) { m.mask = { ...m.mask, points: [], text: "" }; } });
|
||||
segs.value.forEach((_, i) => { const m = meta(i); if (m.mask) { m.mask = { ...m.mask, points: [], keys: [], text: "" }; } });
|
||||
}
|
||||
const m = meta(modal.idx); m.mask = { ...(m.mask || {}), points: [], text: "" };
|
||||
const m = meta(modal.idx); m.mask = { ...(m.mask || {}), points: [], keys: [], text: "" };
|
||||
plan.track = { points: modal.points.map(p => ({ ...p })), frame: s0.start + modal.key, text: modal.text || "",
|
||||
threshold: plan.mask_cfg?.threshold ?? 0.5 };
|
||||
maskPrev.value = {}; save(); modal.open = false;
|
||||
};
|
||||
const clearTrack = () => { plan.track = null; maskPrev.value = {}; save(); };
|
||||
const clearMask = i => {
|
||||
const m = meta(i); m.mask = {}; m.color_guide_points2 = []; m.color_guide_key2 = 0; m.color_guide_text2 = "";
|
||||
const m = meta(i); m.mask = {}; m.color_guide_regions = [];
|
||||
m.crop = false; m.inpaint = false; m.paste = false;
|
||||
const k = segs.value[i] ? segKey(segs.value[i]) : null;
|
||||
if (k) { const mp = { ...maskPrev.value }; delete mp[k]; maskPrev.value = mp; }
|
||||
@@ -409,17 +502,16 @@ const meta = i => { while (plan.segs.length <= i) plan.segs.push({}); return pla
|
||||
const to = targets(scope, from);
|
||||
to.forEach(i => {
|
||||
const m = meta(i), x = segs.value[i];
|
||||
if (fields.ref) { m.ref = src.ref; m.ref2 = src.ref2; }
|
||||
if (fields.ref) m.refs = [...src.refs];
|
||||
if (fields.prompt) m.prompt = src.prompt;
|
||||
if (fields.mask_text) m.mask = { ...(m.mask || {}), text: src.mask.text || "" };
|
||||
if (fields.mask_video) m.mask = { ...(m.mask || {}), video: src.mask.video || "" };
|
||||
if (fields.crop && (hasMask(x) || src.mask.text || fields.mask_text)) { m.crop = src.crop; m.inpaint = src.inpaint; m.paste = src.paste; m.paste_text = src.pasteText; m.strength = src.strength; }
|
||||
if (fields.target) m.target = src.target;
|
||||
if (fields.color_guide) {
|
||||
m.color_guide = src.color_guide; m.color_guide_color = src.color_guide_color;
|
||||
m.color_guide = src.color_guide; m.color_guide_picture = src.color_guide_picture;
|
||||
m.color_guide_opacity = src.color_guide_opacity;
|
||||
m.color_guide_text2 = src.color_guide_text2; m.color_guide_color2 = src.color_guide_color2;
|
||||
m.color_guide_points2 = [...(src.color_guide_points2 || [])]; m.color_guide_key2 = src.color_guide_key2;
|
||||
m.color_guide_regions = src.regions.map(r => ({ ...r, points: [...(r.points || [])] }));
|
||||
}
|
||||
if (fields.chain && i > 0) { m.chain = src.chain; m.chain_frame = src.chainFrame; m.chain_history = src.chainHistory; }
|
||||
});
|
||||
@@ -439,24 +531,54 @@ const meta = i => { while (plan.segs.length <= i) plan.segs.push({}); return pla
|
||||
busy.value = "";
|
||||
}
|
||||
// reference sets used by the plan (global and per shot): one description each
|
||||
const refSets = computed(() => {
|
||||
// every picture the plan uses, once, with where it is <Picture n>: one description each ({details@Picture n})
|
||||
const pictures = computed(() => {
|
||||
const out = new Map();
|
||||
const add = (a, b, where) => { if (!a && !b) return; const k = `${a || ""}|${b || ""}`; if (!out.has(k)) out.set(k, { a, b, where: [] }); out.get(k).where.push(where); };
|
||||
add(plan.global_ref, plan.global_ref2, "global");
|
||||
segs.value.forEach((x, i) => add(x.ref || plan.global_ref, x.ref2 || plan.global_ref2, `#${i + 1}`));
|
||||
return [...out.entries()].map(([key, v]) => ({ key, ...v }));
|
||||
const add = (name, where) => { if (!name) return; if (!out.has(name)) out.set(name, []); out.get(name).push(where); };
|
||||
plan.global_refs.forEach((x, k) => add(x, `global ${k + 1}`));
|
||||
segs.value.forEach((x, i) => x.names.forEach((name, k) => { if (x.refs[k] === name) add(name, `#${i + 1}: ${k + 1}`); }));
|
||||
return [...out.entries()].map(([name, where]) => ({ name, where }));
|
||||
});
|
||||
async function describeRefs() {
|
||||
busy.value = "Describing the references with the VLM…"; error.value = "";
|
||||
// names: the pictures to describe (all of them by default)
|
||||
async function describePictures(names = pictures.value.map(x => x.name)) {
|
||||
busy.value = "Describing the pictures with the VLM…"; error.value = "";
|
||||
try {
|
||||
const sets = refSets.value.map(x => [x.a || "", x.b || ""]);
|
||||
const r = await api.fetchApi("/bfs/shotloop/describe", { method: "POST", body: JSON.stringify({ plan: { ...plan }, sets }) });
|
||||
const r = await api.fetchApi("/bfs/shotloop/describe", { method: "POST",
|
||||
body: JSON.stringify({ plan: { ...plan }, pictures: names }) });
|
||||
const j = await r.json(); if (j.error) throw new Error(j.error);
|
||||
plan.ref_details = { ...plan.ref_details, ...j.texts }; save();
|
||||
plan.picture_details = { ...plan.picture_details, ...j.pictures }; save();
|
||||
} catch (e) { error.value = String(e.message || e); }
|
||||
busy.value = "";
|
||||
}
|
||||
const setDetail = (k, v) => { plan.ref_details = { ...plan.ref_details, [k]: v }; save(); };
|
||||
const setPictureDetail = (name, v) => { plan.picture_details = { ...plan.picture_details, [name]: v }; save(); };
|
||||
// a picture's own describe preset ("" = the card's) and custom instruction
|
||||
// 🤖 Auto prompt: what a shot's written prompt was made from (same string as bfs_shot_loop.auto_prompt_for), so the
|
||||
// panel shows when the request, pictures, marks or mode changed after it was written
|
||||
const autoFor = s => {
|
||||
const marks = s.color_guide ? [[s.color_guide_picture, pictureColor(plan, s.color_guide_picture)],
|
||||
...s.regions.filter(r => String(r.text || "").trim() || (r.points || []).length).map(r => [r.picture, pictureColor(plan, r.picture)])] : [];
|
||||
return JSON.stringify([s.prompt || plan.global_prompt || "", s.names, marks, modeOf(s), s.mask?.text || "", s.target || "",
|
||||
plan.auto_prompt?.template || "", plan.auto_prompt?.style || "h3"]);
|
||||
};
|
||||
const writtenOf = s => plan.auto_prompts?.[segKey(s)] || null;
|
||||
const setAuto = (k, v) => { plan.auto_prompt = { ...plan.auto_prompt, [k]: v }; save(); };
|
||||
const setWritten = (s, text) => {
|
||||
const k = segKey(s), cur = plan.auto_prompts[k] || {};
|
||||
plan.auto_prompts = { ...plan.auto_prompts, [k]: { ...cur, text, for: cur.for ?? autoFor(s) } }; save();
|
||||
};
|
||||
// the VLM writes the prompts of these shots (all by default)
|
||||
async function writePrompts(indices = segs.value.map((_, i) => i)) {
|
||||
busy.value = `Writing ${indices.length} prompt(s) with the VLM…`; error.value = "";
|
||||
try {
|
||||
const r = await api.fetchApi("/bfs/shotloop/autoprompt", { method: "POST", body: JSON.stringify({ plan: { ...plan }, indices }) });
|
||||
const j = await r.json(); if (j.error) throw new Error(j.error);
|
||||
plan.auto_prompts = { ...plan.auto_prompts, ...j.prompts }; save();
|
||||
} catch (e) { error.value = String(e.message || e); }
|
||||
busy.value = "";
|
||||
}
|
||||
const setPictureCfg = (name, k, v) => {
|
||||
plan.picture_cfg = { ...plan.picture_cfg, [name]: { ...(plan.picture_cfg[name] || {}), [k]: v } }; save();
|
||||
};
|
||||
const setVlmCfg = (k, v) => { plan.vlm_cfg = { ...plan.vlm_cfg, [k]: v }; save(); };
|
||||
const applySug = (i, what) => {
|
||||
const s = segs.value[i], g = s && vlmSug.value[segKey(s)]; if (!g) return;
|
||||
@@ -529,6 +651,16 @@ const meta = i => { while (plan.segs.length <= i) plan.segs.push({}); return pla
|
||||
const inp = document.createElement("input"); inp.type = "file"; inp.accept = accept;
|
||||
inp.onchange = () => inp.files[0] && upload(inp.files[0], cb); inp.click();
|
||||
};
|
||||
// several files: uploaded one after the other, cb gets the names in order
|
||||
const uploadAll = async (files, cb) => {
|
||||
const names = [];
|
||||
for (const f of files) await upload(f, name => names.push(name));
|
||||
if (names.length) cb(names);
|
||||
};
|
||||
const pickFiles = (accept, cb) => {
|
||||
const inp = document.createElement("input"); inp.type = "file"; inp.accept = accept; inp.multiple = true;
|
||||
inp.onchange = () => uploadAll([...inp.files], cb); inp.click();
|
||||
};
|
||||
async function progress(reset = false) {
|
||||
if (!plan.video || plan.run === "auto") { prog.done = 0; prog.count = 0; Object.assign(cache, { ok: [], stored: [], mb: 0 }); return; }
|
||||
try {
|
||||
@@ -665,16 +797,24 @@ const meta = i => { while (plan.segs.length <= i) plan.segs.push({}); return pla
|
||||
const onStatus = e => { if (busy.value || modal.busy) Object.assign(status, e.detail || {}); };
|
||||
let clock = 0;
|
||||
const onVlm = e => { if (e.detail?.video === plan.video) mergeSug(e.detail.segs); };
|
||||
// prompts the planner's VLM wrote while running (shots that had none): kept in the plan, so they stay the same
|
||||
const onAutoPrompt = e => {
|
||||
const d = e.detail; if (d?.video !== plan.video) return;
|
||||
const tpl = typeof d.template === "string" && d.template !== (plan.auto_prompt.template || "");
|
||||
if (tpl) plan.auto_prompt = { ...plan.auto_prompt, template: d.template }; // the prompt input of the last run
|
||||
if (tpl || Object.keys(d.prompts || {}).length) { plan.auto_prompts = { ...plan.auto_prompts, ...(d.prompts || {}) }; save(); }
|
||||
};
|
||||
const onNext = e => { onProg(e); if (plan.run === "queue" && plan.auto_continue) setTimeout(() => app.queuePrompt(0, 1), 300); };
|
||||
onMounted(() => {
|
||||
load(); refreshFiles(); analyze(); progress(); refreshVram(); raf = requestAnimationFrame(loop);
|
||||
if (!plan.video) tab.value = "video";
|
||||
api.addEventListener("bfs-shotloop-progress", onProg); api.addEventListener("bfs-shotloop-next", onNext);
|
||||
api.addEventListener("bfs-shotloop-vlm", onVlm);
|
||||
api.addEventListener("bfs-shotloop-autoprompt", onAutoPrompt);
|
||||
api.addEventListener("bfs-shotloop-status", onStatus);
|
||||
clock = setInterval(() => { if (busy.value || modal.busy) now.value = Date.now(); }, 500);
|
||||
});
|
||||
onBeforeUnmount(() => { cancelAnimationFrame(raf); api.removeEventListener("bfs-shotloop-progress", onProg); api.removeEventListener("bfs-shotloop-next", onNext); api.removeEventListener("bfs-shotloop-vlm", onVlm); api.removeEventListener("bfs-shotloop-status", onStatus); clearInterval(clock); });
|
||||
onBeforeUnmount(() => { cancelAnimationFrame(raf); api.removeEventListener("bfs-shotloop-progress", onProg); api.removeEventListener("bfs-shotloop-next", onNext); api.removeEventListener("bfs-shotloop-vlm", onVlm); api.removeEventListener("bfs-shotloop-autoprompt", onAutoPrompt); api.removeEventListener("bfs-shotloop-status", onStatus); clearInterval(clock); });
|
||||
io.expose({ reload: () => { load(); analyze(); progress(); } });
|
||||
|
||||
// 🧠 the memory status at the top: the plan's biggest shot against the biggest one that ran on this GPU
|
||||
@@ -712,24 +852,46 @@ const meta = i => { while (plan.segs.length <= i) plan.segs.push({}); return pla
|
||||
recent.value = [name, ...recent.value.filter(x => x !== name)].slice(0, 10);
|
||||
store.set(RECENT_KEY, recent.value);
|
||||
};
|
||||
// target: shot index or "global"; field: ref | ref2 (global_ref | global_ref2 for "global")
|
||||
const setRef = (target, field, name) => {
|
||||
// target: shot index or "global"; k: the picture's slot (0 = <Picture 1>). "" removes it: a global picture is taken
|
||||
// out of the list (the next ones move up), a shot's own picture falls back to the global one at that slot
|
||||
const refsOf = target => (target === "global" ? plan.global_refs : (meta(target).refs ||= []));
|
||||
const setRef = (target, k, name) => {
|
||||
if (isColorValue(name)) return; // colors belong to Colored Mark settings, never to image references
|
||||
if (target === "global") plan[field] = name; else meta(target)[field] = name;
|
||||
const list = refsOf(target);
|
||||
if (name) { while (list.length < k) list.push(""); list[k] = name; }
|
||||
else if (target === "global" || k >= plan.global_refs.length) list.splice(k, 1);
|
||||
else list[k] = "";
|
||||
remember(name); save();
|
||||
};
|
||||
const uploadRef = (target, field) => pickFile("image/*", name => setRef(target, field, name));
|
||||
// pictures put in from slot k on (k = the list's end: added), never past 12
|
||||
const putRefs = (target, k, names) => {
|
||||
const list = refsOf(target);
|
||||
names.slice(0, Math.max(0, MAX_REFS - k)).forEach((name, j) => { while (list.length < k + j) list.push(""); list[k + j] = name; remember(name); });
|
||||
save();
|
||||
};
|
||||
const uploadRef = (target, k, many = false) => many
|
||||
? pickFiles("image/*", names => putRefs(target, k, names))
|
||||
: pickFile("image/*", name => setRef(target, k, name));
|
||||
// a drop on slot k replaces it (more files go on the next slots); k = null (the box): added at the end. A recent
|
||||
// thumbnail dragged in carries its name, image files are uploaded
|
||||
const dropRefs = (target, k, e) => {
|
||||
const at = k ?? refsOf(target).length;
|
||||
const name = e.dataTransfer?.getData("text/bfs-ref");
|
||||
if (name) { putRefs(target, at, [name]); return; }
|
||||
const files = [...(e.dataTransfer?.files || [])].filter(f => f.type.startsWith("image/"));
|
||||
if (files.length) uploadAll(files, names => putRefs(target, at, names));
|
||||
};
|
||||
const clearRecent = () => { recent.value = []; store.set(RECENT_KEY, []); };
|
||||
// everything: the recent list and every reference this plan uses (global and per shot)
|
||||
const clearAllRefs = () => {
|
||||
if (!confirm("Remove every reference picture from this plan (global, every shot) and forget the recent list?")) return;
|
||||
clearRecent();
|
||||
plan.global_ref = ""; plan.global_ref2 = "";
|
||||
plan.segs.forEach(m => { if (m) { m.ref = ""; m.ref2 = ""; } });
|
||||
plan.global_refs = [];
|
||||
plan.segs.forEach(m => { if (m) m.refs = []; });
|
||||
save();
|
||||
};
|
||||
const usedRefs = computed(() => {
|
||||
const set = new Set([...recent.value, plan.global_ref, plan.global_ref2, ...plan.segs.flatMap(m => [m?.ref, m?.ref2])].filter(Boolean));
|
||||
const set = new Set([...recent.value, ...plan.global_refs, ...plan.segs.flatMap(m => m?.refs || [])].filter(Boolean));
|
||||
return [...set].slice(0, 10);
|
||||
});
|
||||
|
||||
@@ -737,12 +899,14 @@ const meta = i => { while (plan.segs.length <= i) plan.segs.push({}); return pla
|
||||
const c = {
|
||||
api, plan, files, an, cuts, detectorUsed, size, busy, error, status, busySince, now, upPct, sel, zoom, hover, prog, tlEl,
|
||||
stats, people, maskPrev, showMasks, vlmSug, modal, segPeople, targetCrop, tab, copyOpts, COPY_FIELDS, vid, play, recent, clearRecent, clearAllRefs,
|
||||
n, range, setRange, maxLen, segs, allSegCount, active, pxPerFrame, tlWidth, filtersOn, refSets, usedRefs, checks, tour, memory, vram, refreshVram, cache, cacheOf, redo, combine,
|
||||
n, range, setRange, maxLen, segs, allSegCount, active, pxPerFrame, tlWidth, filtersOn, usedRefs, checks, tour, memory, vram, refreshVram, cache, cacheOf, redo, combine,
|
||||
MODES, modeOf, modeName, viewUrl, save, meta, setMeta, setMode, setPlan, whoIn, castOf, included, personOf, statFor, skipWhy, refOf, promptOf, checksFor,
|
||||
refreshFiles, analyze, autoSplit, analyzeContent, findPeople, setCast, describeTarget, previewMask, setMask, setMaskCfg,
|
||||
openPoints, switchPointRegion, savePoints, savePointsAll, saveTrack, clearTrack, pointsTo, clearMask, copyFrom, analyseVLM, describeRefs, setDetail, setVlmCfg,
|
||||
openPoints, switchPointRegion, savePoints, savePointsAll, saveTrack, clearTrack, pointsTo, clearMask, copyFrom, analyseVLM, setVlmCfg,
|
||||
applySug, applyAllSug, setCastOpt, setGlobalMaskVideo, setFilter, pickFile, progress, setVideo, splitAt, mergeNext, removeCut, select, drag,
|
||||
frameAt, thumbFor, playShot, playAll, stop, seek, setRef, uploadRef,
|
||||
frameAt, thumbFor, playShot, playAll, stop, seek, setRef, putRefs, uploadRef, dropRefs, showFrame, dropKeyFrame, pointSpec, keyFramesOf, pictures, describePictures, setPictureDetail, setPictureCfg,
|
||||
setMarkPicture, addMarkRegion, setMarkRegion, removeMarkRegion, setPictureColor,
|
||||
autoFor, writtenOf, setAuto, setWritten, writePrompts,
|
||||
};
|
||||
|
||||
return () => {
|
||||
@@ -786,8 +950,81 @@ const meta = i => { while (plan.segs.length <= i) plan.segs.push({}); return pla
|
||||
};
|
||||
}
|
||||
|
||||
// A workflow saved before 1.113 keeps the old slots of these nodes (ComfyUI restores a node's saved slots): the
|
||||
// planner's ref_image / ref_image_2 inputs are gone, the planner's and Unpack's ref_image / ref_image_2 outputs are now
|
||||
// refs / ref_count, and what fed Repack's ref_image now feeds its refs input.
|
||||
// Saved widget orders of the conditioning nodes (ComfyUI restores values by position): use_ref_2 (always second) left
|
||||
// in 1.114 and the Advanced node's duet widgets in 1.113, so older values are put back by name
|
||||
const OLD_ADVANCED_WIDGETS = ["guide_mode", "use_ref_2", "first_frame", "ref_image_size", "with_audio", "duet",
|
||||
"panel_position", "panel_size", "panel_noise", "rope_gap", "task", "instruction", "setting_ref", "inpaint", "mask_guide",
|
||||
"source_blur", "source_blur_strength", "source_blur_segmenter", "source_blur_effect", "mask_guide_look", "mask_ref_size"];
|
||||
const ADVANCED_1113 = ["guide_mode", "use_ref_2", "first_frame", "ref_image_size", "with_audio", "setting_ref", "inpaint",
|
||||
"mask_guide", "source_blur", "source_blur_strength", "source_blur_segmenter", "source_blur_effect", "mask_guide_look", "mask_ref_size"];
|
||||
const OLD_SIMPLE_WIDGETS = ["guide_mode", "use_ref_2", "ref_image_size", "with_audio", "inpaint", "source_blur",
|
||||
"source_blur_strength", "source_blur_segmenter", "source_blur_effect"];
|
||||
const NO_BLUR_PICKERS = n => n !== "source_blur_segmenter" && n !== "source_blur_effect"; // added later
|
||||
function oldWidgetNames(cls, v) {
|
||||
if (typeof v[1] !== "boolean") return null; // already without use_ref_2
|
||||
if (cls === "BFSShotH3ConditioningSimple") return v.length >= 9 ? OLD_SIMPLE_WIDGETS : OLD_SIMPLE_WIDGETS.filter(NO_BLUR_PICKERS);
|
||||
if (["off", "canvas", "shifted RoPE"].includes(v[5]) && ["left", "right", "top", "bottom"].includes(v[6]))
|
||||
return v.length >= 21 ? OLD_ADVANCED_WIDGETS : OLD_ADVANCED_WIDGETS.filter(NO_BLUR_PICKERS);
|
||||
return v.length >= 14 ? ADVANCED_1113 : ADVANCED_1113.filter(NO_BLUR_PICKERS);
|
||||
}
|
||||
function upgradeConditioningWidgets(node, info) {
|
||||
const v = info?.widgets_values;
|
||||
const names = Array.isArray(v) && oldWidgetNames(node.comfyClass, v);
|
||||
if (!names) return;
|
||||
names.forEach((name, k) => { const w = node.widgets?.find(x => x.name === name); if (w && k < v.length) w.value = v[k]; });
|
||||
// widgets the saved workflow did not have got a shifted value: back to their default (a list's first choice)
|
||||
node.widgets?.forEach(w => {
|
||||
const choices = w.options?.values;
|
||||
if (!names.includes(w.name) && Array.isArray(choices) && !choices.includes(w.value)) w.value = w.options.default ?? choices[0];
|
||||
});
|
||||
}
|
||||
|
||||
function upgradeLegacySlots(node, info) {
|
||||
const cls = node.comfyClass;
|
||||
if (cls === "BFSShotH3Conditioning" || cls === "BFSShotH3ConditioningSimple") { upgradeConditioningWidgets(node, info); return; }
|
||||
// ref_image / ref_image_2 outputs at slots `at`, at+1 -> refs (list) / ref_count
|
||||
const listOutputs = at => {
|
||||
const a = node.outputs?.[at], b = node.outputs?.[at + 1];
|
||||
if (a?.name === "ref_image") Object.assign(a, { name: "refs", label: undefined, localized_name: "refs", shape: 6 });
|
||||
if (b?.name === "ref_image_2") { node.disconnectOutput(at + 1); Object.assign(b, { name: "ref_count", label: undefined, localized_name: "ref_count", type: "INT", shape: undefined }); }
|
||||
};
|
||||
if (cls === "BFSShotPlanner") {
|
||||
for (const name of ["ref_image", "ref_image_2"]) {
|
||||
const k = node.inputs?.findIndex(x => x.name === name) ?? -1;
|
||||
if (k >= 0) node.removeInput(k);
|
||||
}
|
||||
listOutputs(8);
|
||||
} else if (cls === "BFSShotUnpack") {
|
||||
listOutputs(1);
|
||||
} else if (cls === "BFSShotRepack") {
|
||||
// what fed ref_image now feeds refs; ref_image / ref_image_2 go away
|
||||
const at = name => node.inputs?.findIndex(x => x.name === name) ?? -1;
|
||||
for (const name of ["ref_image", "ref_image_2"]) {
|
||||
const k = at(name); if (k < 0) continue;
|
||||
const lid = node.inputs[k].link, l = lid == null ? null : (node.graph?.links?.get?.(lid) ?? node.graph?.links?.[lid]);
|
||||
const origin = l ? node.graph.getNodeById(l.origin_id) : null;
|
||||
node.removeInput(k);
|
||||
const to = at("refs");
|
||||
if (name === "ref_image" && origin && to >= 0 && node.inputs[to].link == null) origin.connect(l.origin_slot, node, to);
|
||||
}
|
||||
} else return;
|
||||
node.setDirtyCanvas?.(true, true);
|
||||
}
|
||||
|
||||
app.registerExtension({
|
||||
name: "BFSNodes.ShotPlanner",
|
||||
async beforeRegisterNodeDef(nodeType, nodeData) {
|
||||
if (!["BFSShotPlanner", "BFSShotUnpack", "BFSShotRepack", "BFSShotH3Conditioning", "BFSShotH3ConditioningSimple"].includes(nodeData.name)) return;
|
||||
const prev = nodeType.prototype.onConfigure;
|
||||
nodeType.prototype.onConfigure = function (info) {
|
||||
const r = prev?.apply(this, arguments);
|
||||
upgradeLegacySlots(this, info);
|
||||
return r;
|
||||
};
|
||||
},
|
||||
async nodeCreated(node) {
|
||||
if (node.comfyClass !== "BFSShotPlanner") return;
|
||||
styles();
|
||||
@@ -801,6 +1038,7 @@ app.registerExtension({
|
||||
// node (Vue nodes mode sizes widgets by their content); min-width keeps it from collapsing to its content width
|
||||
host.style.cssText = "position:relative;width:100%;height:100%;min-width:900px;min-height:640px";
|
||||
node.addDOMWidget("bfs_shot_planner", "div", host, { serialize: false, hideOnZoom: false });
|
||||
wheelToCanvas(host); // the wheel zooms the canvas over the panel too, except on something that scrolls
|
||||
const handle = {};
|
||||
createApp({
|
||||
setup: () => Panel({
|
||||
|
||||
+261
-7
@@ -4,12 +4,21 @@
|
||||
* Wipe (drag the divider), overlay (drag to fade) or side by side, with any of guide / target / difference on top
|
||||
* or below. The strip under the scrubber is the difference per frame, measured by the node on the real frames:
|
||||
* click it to jump to a frame. The node sends three temporary MP4s and that list when it runs.
|
||||
*
|
||||
* ⬇ save plays the video once from the start and downloads it as shown (same layout, divider, labels); ● rec records
|
||||
* whatever you do in the player (dragging the divider, pausing, scrubbing) until ■ stop. Both draw the two videos into
|
||||
* a canvas and record it in the browser, in real time. The browser's file (WebM, or a fragmented MP4 many apps won't
|
||||
* play) is then converted by ComfyUI to the chosen format: MP4 (H.264), a small MP4 (under 10 MB) or a GIF.
|
||||
*/
|
||||
import { app } from "../../scripts/app.js";
|
||||
import { api } from "../../scripts/api.js";
|
||||
import { wheelToCanvas } from "./bfs_wheel.js";
|
||||
|
||||
const SOURCES = ["guide", "target", "difference"];
|
||||
const MODES = [["wipe", "wipe ↔"], ["wipe-v", "wipe ↕"], ["overlay", "overlay"], ["side", "side by side"]];
|
||||
const FORMATS = [["mp4", "MP4"], ["small", "MP4 small (<10 MB)"], ["gif", "GIF"], ["webm", "browser file (WebM)"]];
|
||||
const FORMAT_KEY = "bfs.videoCompare.format";
|
||||
const PART = 8 * 1024 * 1024; // upload pieces, under ComfyUI's request limit
|
||||
|
||||
function styles() {
|
||||
if (document.getElementById("bfs-vcmp-css")) return;
|
||||
@@ -19,12 +28,16 @@ function styles() {
|
||||
.bfsvc{display:flex;flex-direction:column;gap:6px;height:100%;box-sizing:border-box;padding:6px;
|
||||
font:12px/1.4 var(--font-family,system-ui,sans-serif);color:var(--descrip-text,#c9c9cf);
|
||||
background:var(--comfy-input-bg,#1b1b1f);border-radius:8px}
|
||||
.bfsvc .stage{position:relative;flex:1;min-height:160px;background:#0e0e11;border-radius:6px;overflow:hidden;
|
||||
.bfsvc .stage{position:relative;flex:1 1 0;min-height:160px;background:#0e0e11;border-radius:6px;overflow:hidden;
|
||||
cursor:ew-resize;touch-action:none;user-select:none}
|
||||
.bfsvc .stage.v{cursor:ns-resize}.bfsvc .stage.o{cursor:col-resize}.bfsvc .stage.s{cursor:default;display:grid;
|
||||
grid-template-columns:1fr 1fr;gap:4px}
|
||||
.bfsvc .stage.v{cursor:ns-resize}.bfsvc .stage.o{cursor:col-resize}.bfsvc .stage.s{cursor:default}
|
||||
.bfsvc video{position:absolute;inset:0;width:100%;height:100%;object-fit:contain;pointer-events:none}
|
||||
.bfsvc .stage.s video{position:relative}
|
||||
.bfsvc .stage.s video{width:calc(50% - 2px)}.bfsvc .stage.s video.r{left:auto;right:0}
|
||||
.bfsvc .refs{flex:0 0 auto;height:84px;display:flex;gap:6px;justify-content:center;overflow-x:auto;padding:4px;
|
||||
background:#0e0e11;border-radius:6px}
|
||||
.bfsvc .refs figure{margin:0;height:100%;position:relative;flex:none}
|
||||
.bfsvc .refs img{height:100%;border-radius:4px;box-shadow:0 0 0 1px #000a;background:#000;display:block}
|
||||
.bfsvc .refs figcaption{position:absolute;left:3px;top:2px;font-size:10px;padding:0 4px;border-radius:3px;background:#000a;color:#fff}
|
||||
.bfsvc .line{position:absolute;background:#fff;box-shadow:0 0 0 1px #0008;pointer-events:none}
|
||||
.bfsvc .tag{position:absolute;top:6px;padding:1px 7px;border-radius:4px;background:#000a;color:#fff;font-size:11px;
|
||||
pointer-events:none}
|
||||
@@ -32,6 +45,9 @@ function styles() {
|
||||
.bfsvc select,.bfsvc button{background:#131317;border:1px solid #34343d;color:#e6e6ea;border-radius:4px;
|
||||
padding:2px 6px;font:inherit;cursor:pointer}
|
||||
.bfsvc button:hover,.bfsvc select:hover{border-color:#3a86ff}
|
||||
.bfsvc button.rec{border-color:#e0573a;color:#ff7a5c}
|
||||
.bfsvc button:disabled{opacity:.45;cursor:default}
|
||||
.bfsvc label{display:flex;gap:3px;align-items:center;cursor:pointer}
|
||||
.bfsvc input[type=range]{flex:1;accent-color:#3a86ff;min-width:80px}
|
||||
.bfsvc canvas{width:100%;height:26px;cursor:pointer;border-radius:3px;background:#131317;display:block}
|
||||
.bfsvc .info{font-family:ui-monospace,monospace;font-size:11px;white-space:nowrap}
|
||||
@@ -55,13 +71,15 @@ function el(tag, attrs = {}, kids = []) {
|
||||
}
|
||||
|
||||
function buildPlayer(host) {
|
||||
const st = { data: null, top: "target", bottom: "guide", mode: "wipe", pos: 0.5, playing: false, muted: true, frame: 0 };
|
||||
const st = { data: null, top: "target", bottom: "guide", mode: "wipe", pos: 0.5, playing: false, muted: true, frame: 0,
|
||||
labels: true, refs: true };
|
||||
const vTop = el("video", { playsinline: "", preload: "auto", loop: "" });
|
||||
const vBot = el("video", { playsinline: "", preload: "auto", loop: "" });
|
||||
vTop.muted = vBot.muted = true;
|
||||
const line = el("div", { class: "line" });
|
||||
const tagTop = el("div", { class: "tag" }), tagBot = el("div", { class: "tag" });
|
||||
const empty = el("div", { class: "empty" }, "Run the node to load the guide and the target.");
|
||||
const refBox = el("div", { class: "refs", style: "display:none" });
|
||||
const stage = el("div", { class: "stage", tabindex: "0" }, [vBot, vTop, line, tagTop, tagBot, empty]);
|
||||
const sel = (opts, val, on) => {
|
||||
const s = el("select", {}, opts.map(([v, l]) => { const o = el("option", { value: v }, l); if (v === val) o.selected = true; return o; }));
|
||||
@@ -83,6 +101,22 @@ function buildPlayer(host) {
|
||||
const btnNext = el("button", { title: "Next frame (→)", onclick: () => step(1) }, "|▶");
|
||||
const btnMute = el("button", { title: "Sound (from the target)", onclick: () => { st.muted = !st.muted; sound(); } }, "🔇");
|
||||
const btnFull = el("button", { title: "Full screen", onclick: () => (document.fullscreenElement ? document.exitFullscreen() : host.requestFullscreen?.())?.catch?.(() => {}) }, "⛶");
|
||||
const btnSave = el("button", { title: "Play the video once from the start and download it as shown", onclick: () => saveFull() }, "⬇ save");
|
||||
let format = "mp4";
|
||||
try { format = localStorage.getItem(FORMAT_KEY) || format; } catch { }
|
||||
if (!FORMATS.some(([v]) => v === format)) format = "mp4";
|
||||
const selFormat = sel(FORMATS, format, v => { format = v; try { localStorage.setItem(FORMAT_KEY, v); } catch { } });
|
||||
selFormat.title = "Format of what save / rec download. MP4: H.264 + AAC, plays and previews anywhere. MP4 small: "
|
||||
+ "the same, sized to stay under 10 MB. GIF: no sound, under 10 MB. browser file: the recording as it is, no conversion.";
|
||||
const btnRec = el("button", { title: "Record what you do in the player until stop, then download it", onclick: () => (rec.mr ? stopRec() : startRec()) }, "● rec");
|
||||
const chkLabels = el("input", { type: "checkbox" });
|
||||
chkLabels.checked = true;
|
||||
chkLabels.addEventListener("change", () => { st.labels = chkLabels.checked; });
|
||||
const chkRefs = el("input", { type: "checkbox" });
|
||||
chkRefs.checked = true;
|
||||
const showRefs = () => { refBox.style.display = st.refs && refBox.childElementCount ? "flex" : "none"; };
|
||||
chkRefs.addEventListener("change", () => { st.refs = chkRefs.checked; showRefs(); });
|
||||
const lblRefs = el("label", { title: "Show the reference pictures in a strip below the videos (also in what save / rec download)", style: "display:none" }, [chkRefs, "refs"]);
|
||||
const scrub = el("input", { type: "range", min: "0", max: "0", value: "0", step: "1" });
|
||||
scrub.addEventListener("input", () => seek(+scrub.value));
|
||||
const info = el("span", { class: "info" }, "");
|
||||
@@ -94,7 +128,9 @@ function buildPlayer(host) {
|
||||
host.append(el("div", { class: "bfsvc" }, [
|
||||
el("div", { class: "row" }, [el("span", {}, "top"), selTop, btnSwap, el("span", {}, "bottom"), selBot, selMode, btnFull]),
|
||||
stage,
|
||||
refBox,
|
||||
el("div", { class: "row" }, [btnPrev, btnPlay, btnNext, scrub, btnMute, info]),
|
||||
el("div", { class: "row" }, [btnSave, btnRec, selFormat, el("label", { title: "Draw the names on the saved / recorded video" }, [chkLabels, "labels"]), lblRefs]),
|
||||
strip,
|
||||
]));
|
||||
|
||||
@@ -118,6 +154,7 @@ function buildPlayer(host) {
|
||||
if (m === "wipe") line.style.cssText = `display:block;left:calc(${p * 100}% - 1px);top:0;bottom:0;width:2px`;
|
||||
if (m === "wipe-v") line.style.cssText = `display:block;top:calc(${p * 100}% - 1px);left:0;right:0;height:2px`;
|
||||
stage.prepend(vTop); stage.prepend(vBot); // under the divider and tags; side by side: bottom left, top right
|
||||
vTop.classList.toggle("r", m === "side");
|
||||
const name = s => s === "guide" ? (st.data?.labels?.[0] || "guide") : s === "target" ? (st.data?.labels?.[1] || "target") : "difference";
|
||||
tagTop.textContent = name(st.top) + (m === "overlay" ? ` ${Math.round(p * 100)}%` : "");
|
||||
tagBot.textContent = name(st.bottom);
|
||||
@@ -175,12 +212,17 @@ function buildPlayer(host) {
|
||||
st.playing = !st.playing;
|
||||
btnPlay.textContent = st.playing ? "⏸" : "▶";
|
||||
if (st.playing) { vTop.currentTime = vBot.currentTime; for (const v of videos()) v.play().catch(() => {}); tick(); }
|
||||
else for (const v of videos()) v.pause();
|
||||
else for (const v of videos()) { v.pause(); v.playbackRate = 1; }
|
||||
}
|
||||
|
||||
function tick() { // the bottom video is the clock; the top one follows it within half a frame
|
||||
if (!st.playing) return;
|
||||
if (Math.abs(vTop.currentTime - vBot.currentTime) > 0.5 / fps()) vTop.currentTime = vBot.currentTime;
|
||||
// small drift: nudge the top one's speed (seeking it every frame kept it seeking, so it stuttered and drew blank)
|
||||
const drift = vTop.currentTime - vBot.currentTime;
|
||||
if (!vTop.seeking) {
|
||||
if (Math.abs(drift) > 0.25) { vTop.currentTime = vBot.currentTime; vTop.playbackRate = 1; }
|
||||
else vTop.playbackRate = Math.abs(drift) > 0.5 / fps() ? Math.min(1.05, Math.max(0.95, 1 - 2 * drift)) : 1;
|
||||
}
|
||||
show(frameOf(vBot.currentTime));
|
||||
requestAnimationFrame(tick);
|
||||
}
|
||||
@@ -200,8 +242,219 @@ function buildPlayer(host) {
|
||||
layout();
|
||||
}
|
||||
|
||||
// --- save / record: the two videos drawn into a canvas the way the stage shows them, encoded by MediaRecorder
|
||||
const rec = { mr: null, chunks: [], raf: 0, t0: 0, full: false, onEnd: null, type: "", busy: "" };
|
||||
|
||||
function pickType(audio) { // name an audio codec only with a sound track: Firefox stalls on one it never gets
|
||||
const list = audio ? ["video/mp4;codecs=avc1.640028,mp4a.40.2", "video/mp4", "video/webm;codecs=vp9,opus", "video/webm;codecs=vp8,opus", "video/webm"]
|
||||
: ["video/mp4;codecs=avc1.640028", "video/mp4;codecs=avc1", "video/mp4", "video/webm;codecs=vp9", "video/webm;codecs=vp8", "video/webm"];
|
||||
return list.find(t => window.MediaRecorder?.isTypeSupported?.(t)) || "";
|
||||
}
|
||||
|
||||
function fit(aw, ah, x, y, w, h) { // the largest aw:ah box centred in (x, y, w, h)
|
||||
const k = Math.min(w / aw, h / ah), bw = aw * k, bh = ah * k;
|
||||
return [x + (w - bw) / 2, y + (h - bh) / 2, bw, bh];
|
||||
}
|
||||
|
||||
function label(ctx, text, x, y, size, right = false, bottom = false) {
|
||||
ctx.font = `${size}px system-ui,sans-serif`;
|
||||
const pad = size * 0.45, w = ctx.measureText(text).width + 2 * pad, h = size * 1.5;
|
||||
const lx = right ? x - w : x, ly = bottom ? y - h : y;
|
||||
ctx.fillStyle = "rgba(0,0,0,.65)"; ctx.fillRect(lx, ly, w, h);
|
||||
ctx.fillStyle = "#fff"; ctx.textBaseline = "middle"; ctx.fillText(text, lx + pad, ly + h / 2);
|
||||
}
|
||||
|
||||
const last = new Map(); // video -> its last whole frame, drawn while it seeks (a seeking video draws blank)
|
||||
function pic(v) {
|
||||
if (v.readyState < 2 || v.seeking) return last.get(v) || v;
|
||||
let c = last.get(v);
|
||||
if (!c) last.set(v, c = el("canvas"));
|
||||
if (c.width !== v.videoWidth || c.height !== v.videoHeight) { c.width = v.videoWidth; c.height = v.videoHeight; }
|
||||
c.getContext("2d").drawImage(v, 0, 0);
|
||||
return c;
|
||||
}
|
||||
|
||||
const refImages = () => (st.refs ? [...refBox.querySelectorAll("img")].filter(i => i.complete && i.naturalWidth) : []);
|
||||
// save / rec: the pictures in a strip below the videos, as in the player (22% of the video's height)
|
||||
const stripHeight = h => (refImages().length ? Math.round(h * 0.22) & ~1 : 0);
|
||||
function drawRefs(ctx, top) {
|
||||
const ims = refImages();
|
||||
if (!ims.length) return;
|
||||
const cw = ctx.canvas.width, sh = ctx.canvas.height - top, gap = Math.max(2, Math.round(sh * 0.05));
|
||||
let h = sh - 2 * gap;
|
||||
const widths = () => ims.map(im => Math.round(im.naturalWidth * h / im.naturalHeight));
|
||||
let total = widths().reduce((a, b) => a + b, 0) + gap * (ims.length - 1);
|
||||
if (total > cw - 2 * gap) { h = Math.floor(h * (cw - 2 * gap) / total); total = widths().reduce((a, b) => a + b, 0) + gap * (ims.length - 1); }
|
||||
ctx.fillStyle = "#0e0e11"; ctx.fillRect(0, top, cw, sh);
|
||||
let x = Math.round((cw - total) / 2);
|
||||
const y = top + Math.round((sh - h) / 2), fs = Math.max(10, Math.round(h * 0.12));
|
||||
ims.forEach((im, k) => {
|
||||
const w = Math.round(im.naturalWidth * h / im.naturalHeight);
|
||||
ctx.drawImage(im, x, y, w, h);
|
||||
if (st.labels) label(ctx, `Picture ${k + 1}`, x + 3, y + 3, fs);
|
||||
x += w + gap;
|
||||
});
|
||||
}
|
||||
|
||||
function compose(ctx) {
|
||||
const sh = ctx.canvas.height - (ctx.canvas.videoHeight || ctx.canvas.height);
|
||||
drawVideos(ctx, ctx.canvas.height - sh);
|
||||
drawRefs(ctx, ctx.canvas.height - sh);
|
||||
}
|
||||
|
||||
function drawVideos(ctx, area) {
|
||||
const cw = ctx.canvas.width, ch = area ?? ctx.canvas.height, m = st.mode, p = st.pos;
|
||||
const B = pic(vBot), T = pic(vTop);
|
||||
const aw = vBot.videoWidth || st.data.width, ah = vBot.videoHeight || st.data.height;
|
||||
const name = s => s === "guide" ? (st.data?.labels?.[0] || "guide") : s === "target" ? (st.data?.labels?.[1] || "target") : "difference";
|
||||
ctx.fillStyle = "#000"; ctx.fillRect(0, 0, cw, ch);
|
||||
if (m === "side") { // bottom left, top right, as on the stage
|
||||
const gap = Math.round(cw * 0.004), half = (cw - gap) / 2;
|
||||
const a = fit(aw, ah, 0, 0, half, ch), b = fit(aw, ah, half + gap, 0, half, ch);
|
||||
ctx.drawImage(B, ...a); ctx.drawImage(T, ...b);
|
||||
if (st.labels) {
|
||||
const fs = Math.max(12, Math.round(a[3] * 0.032)), o = fs * 0.5;
|
||||
label(ctx, name(st.bottom), a[0] + o, a[1] + o, fs); label(ctx, name(st.top), b[0] + o, b[1] + o, fs);
|
||||
}
|
||||
return;
|
||||
}
|
||||
const [x, y, w, h] = fit(aw, ah, 0, 0, cw, ch);
|
||||
ctx.drawImage(B, x, y, w, h);
|
||||
ctx.save();
|
||||
if (m === "wipe" || m === "wipe-v") { ctx.beginPath(); m === "wipe" ? ctx.rect(x, y, w * p, h) : ctx.rect(x, y, w, h * p); ctx.clip(); }
|
||||
if (m === "overlay") ctx.globalAlpha = p;
|
||||
ctx.drawImage(T, x, y, w, h);
|
||||
ctx.restore();
|
||||
const lw = Math.max(2, Math.round(h / 360));
|
||||
ctx.fillStyle = "#fff";
|
||||
if (m === "wipe") ctx.fillRect(x + w * p - lw / 2, y, lw, h);
|
||||
if (m === "wipe-v") ctx.fillRect(x, y + h * p - lw / 2, w, lw);
|
||||
if (st.labels) {
|
||||
const fs = Math.max(12, Math.round(h * 0.032)), o = fs * 0.5;
|
||||
label(ctx, name(st.top) + (m === "overlay" ? ` ${Math.round(p * 100)}%` : ""), x + o, y + o, fs);
|
||||
if (m === "wipe-v") label(ctx, name(st.bottom), x + o, y + h - o, fs, false, true);
|
||||
else label(ctx, name(st.bottom), x + w - o, y + o, fs, true);
|
||||
}
|
||||
}
|
||||
|
||||
function stem() {
|
||||
const d = st.data, clean = s => String(s || "").replace(/[^\w.-]+/g, "_").slice(0, 40);
|
||||
const stamp = new Date().toISOString().replace(/[:T]/g, "-").slice(0, 19);
|
||||
return `bfs_compare_${clean(d?.labels?.[0])}_vs_${clean(d?.labels?.[1])}_${st.mode}_${stamp}`;
|
||||
}
|
||||
|
||||
function download(href, name) {
|
||||
const a = el("a", { href, download: name });
|
||||
document.body.append(a); a.click(); a.remove();
|
||||
}
|
||||
|
||||
async function exportBlob(blob, ext, seconds) { // the browser's file -> ComfyUI converts it -> download
|
||||
const name = stem();
|
||||
if (format === "webm") {
|
||||
const href = URL.createObjectURL(blob);
|
||||
download(href, `${name}.${ext}`);
|
||||
setTimeout(() => URL.revokeObjectURL(href), 60000);
|
||||
return;
|
||||
}
|
||||
const id = [...crypto.getRandomValues(new Uint8Array(8))].map(b => b.toString(16).padStart(2, "0")).join("");
|
||||
rec.busy = "uploading"; buttons();
|
||||
try {
|
||||
for (let off = 0; off < blob.size; off += PART) {
|
||||
const r = await api.fetchApi(`/bfs/compare/part?id=${id}&offset=${off}`, { method: "POST", body: blob.slice(off, off + PART) });
|
||||
if (!r.ok) throw new Error((await r.json().catch(() => ({}))).error || `upload failed (${r.status})`);
|
||||
}
|
||||
rec.busy = "converting"; buttons();
|
||||
const r = await api.fetchApi("/bfs/compare/export", { method: "POST", headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ id, format, seconds, name }) });
|
||||
const f = await r.json();
|
||||
if (!r.ok || f.error) throw new Error(f.error || `conversion failed (${r.status})`);
|
||||
download(url(f), f.filename);
|
||||
} catch (e) {
|
||||
alert(`BFS Video Compare: could not convert the recording (${e.message}). Choose "browser file" to download it as recorded.`);
|
||||
} finally {
|
||||
rec.busy = ""; buttons();
|
||||
}
|
||||
}
|
||||
|
||||
function buttons() {
|
||||
if (rec.busy) { // uploading / converting the finished recording
|
||||
btnRec.classList.remove("rec");
|
||||
btnRec.disabled = btnSave.disabled = selFormat.disabled = true;
|
||||
btnSave.textContent = rec.busy === "uploading" ? "⬆ uploading…" : "⚙ converting…";
|
||||
btnRec.textContent = "● rec";
|
||||
return;
|
||||
}
|
||||
btnRec.disabled = selFormat.disabled = false;
|
||||
const busy = !!rec.mr;
|
||||
btnRec.classList.toggle("rec", busy);
|
||||
btnRec.textContent = busy ? (rec.full ? "■ stop" : `■ stop ${((performance.now() - rec.t0) / 1000).toFixed(0)}s`) : "● rec";
|
||||
btnSave.disabled = busy;
|
||||
btnSave.textContent = busy && rec.full ? `⬇ ${Math.round(100 * (vBot.currentTime / (vBot.duration || 1)))}%` : "⬇ save";
|
||||
}
|
||||
|
||||
function startRec(full = false) {
|
||||
if (!st.data || rec.mr || rec.busy) return false;
|
||||
if (!window.MediaRecorder || !HTMLCanvasElement.prototype.captureStream) { alert("This browser can't record a canvas (MediaRecorder)."); return false; }
|
||||
const aw = vBot.videoWidth || st.data.width, ah = vBot.videoHeight || st.data.height;
|
||||
const side = st.mode === "side", canvas = el("canvas");
|
||||
// the size of the layout it starts in (side by side is twice as wide); a later mode change is fitted into it
|
||||
canvas.width = (side ? 2 * aw + Math.round(2 * aw * 0.004) : aw) & ~1;
|
||||
canvas.videoHeight = ah & ~1; // the videos; the reference strip goes below them
|
||||
canvas.height = canvas.videoHeight + stripHeight(canvas.videoHeight);
|
||||
const ctx = canvas.getContext("2d");
|
||||
compose(ctx);
|
||||
const stream = canvas.captureStream(Math.max(30, Math.round(fps())));
|
||||
// the soundtrack is on the target. Not Firefox's mozCaptureStream: it takes the element's picture away from the canvas
|
||||
const vt = st.top === "target" ? vTop : st.bottom === "target" ? vBot : null;
|
||||
try { for (const t of vt?.captureStream?.()?.getAudioTracks?.() || []) stream.addTrack(t); } catch { }
|
||||
const type = pickType(stream.getAudioTracks().length > 0);
|
||||
if (!type) { for (const t of stream.getTracks()) t.stop(); alert("This browser can't record MP4 or WebM."); return false; }
|
||||
rec.type = type; rec.chunks = []; rec.full = full; rec.t0 = performance.now();
|
||||
rec.mr = new MediaRecorder(stream, { mimeType: type, videoBitsPerSecond: 16e6 });
|
||||
rec.mr.ondataavailable = e => { if (e.data?.size) rec.chunks.push(e.data); };
|
||||
rec.mr.onstop = () => {
|
||||
for (const t of stream.getTracks()) t.stop();
|
||||
const chunks = rec.chunks, seconds = (performance.now() - rec.t0) / 1000;
|
||||
rec.mr = null; rec.chunks = []; buttons();
|
||||
if (chunks.length) exportBlob(new Blob(chunks, { type: rec.type.split(";")[0] }), rec.type.includes("mp4") ? "mp4" : "webm", seconds);
|
||||
};
|
||||
rec.mr.start(1000);
|
||||
const frame = () => { if (!rec.mr) return; compose(ctx); buttons(); rec.raf = requestAnimationFrame(frame); };
|
||||
rec.raf = requestAnimationFrame(frame);
|
||||
buttons();
|
||||
return true;
|
||||
}
|
||||
|
||||
function stopRec() {
|
||||
if (!rec.mr) return;
|
||||
cancelAnimationFrame(rec.raf);
|
||||
if (rec.onEnd) { vBot.removeEventListener("ended", rec.onEnd); rec.onEnd = null; }
|
||||
if (rec.full) { for (const v of videos()) v.loop = true; if (st.playing) toggle(); }
|
||||
if (rec.mr.state !== "inactive") rec.mr.stop();
|
||||
}
|
||||
|
||||
async function saveFull() { // once from the first frame to the last, then stop and download
|
||||
if (!st.data || rec.mr || rec.busy) return;
|
||||
if (st.playing) toggle();
|
||||
for (const v of videos()) v.loop = false;
|
||||
const seeked = v => new Promise(res => { const t = setTimeout(res, 2000); v.addEventListener("seeked", () => { clearTimeout(t); res(); }, { once: true }); });
|
||||
const wait = Promise.all(videos().map(seeked));
|
||||
seek(0);
|
||||
await wait;
|
||||
if (!startRec(true)) { for (const v of videos()) v.loop = true; return; }
|
||||
rec.onEnd = () => { rec.onEnd = null; stopRec(); };
|
||||
vBot.addEventListener("ended", rec.onEnd, { once: true });
|
||||
toggle();
|
||||
}
|
||||
|
||||
return {
|
||||
setData(d) {
|
||||
stopRec();
|
||||
last.clear();
|
||||
refBox.replaceChildren(...(d.refs || []).map((f, k) => el("figure", { title: `Picture ${k + 1}` },
|
||||
[el("img", { src: url(f), crossorigin: "anonymous" }), el("figcaption", {}, `Picture ${k + 1}`)])));
|
||||
lblRefs.style.display = d.refs?.length ? "flex" : "none";
|
||||
showRefs();
|
||||
st.data = d;
|
||||
empty.style.display = "none";
|
||||
scrub.max = String(Math.max(0, d.frames - 1));
|
||||
@@ -223,6 +476,7 @@ app.registerExtension({
|
||||
styles();
|
||||
const host = el("div", { style: "width:100%;height:100%;min-height:320px" });
|
||||
this._bfsCompare = buildPlayer(host);
|
||||
wheelToCanvas(host); // the wheel zooms the canvas over the player too
|
||||
this.addDOMWidget("bfs_video_compare", "div", host, { serialize: false, hideOnZoom: false, getMinHeight: () => 320 });
|
||||
this.setSize([Math.max(this.size[0], 560), Math.max(this.size[1], 620)]);
|
||||
new ResizeObserver(() => this._bfsCompare?.drawStrip()).observe(host);
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
/**
|
||||
* The mouse wheel over a node's HTML panel zooms / pans the canvas like anywhere else on the graph, unless the pointer
|
||||
* is on something that scrolls that way and still can (a list, a text box, a tall panel): that one scrolls, and once
|
||||
* it reaches its end the wheel goes back to the canvas. Ctrl / Cmd + wheel always goes to the canvas.
|
||||
*/
|
||||
import { app } from "../../scripts/app.js";
|
||||
|
||||
function scrolls(start, stop, dx, dy) {
|
||||
for (let n = start; n && n.nodeType === 1; n = n.parentElement) {
|
||||
if (n.tagName === "SELECT" && (n.size > 1 || n.multiple)) return true;
|
||||
const cs = getComputedStyle(n);
|
||||
if (dy && /(auto|scroll)/.test(cs.overflowY) && n.scrollHeight > n.clientHeight + 1
|
||||
&& (dy < 0 ? n.scrollTop > 0 : n.scrollTop + n.clientHeight < n.scrollHeight - 1)) return true;
|
||||
if (dx && /(auto|scroll)/.test(cs.overflowX) && n.scrollWidth > n.clientWidth + 1
|
||||
&& (dx < 0 ? n.scrollLeft > 0 : n.scrollLeft + n.clientWidth < n.scrollWidth - 1)) return true;
|
||||
if (n === stop) break;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export function wheelToCanvas(el) {
|
||||
el.addEventListener("wheel", e => {
|
||||
const canvas = app.canvas?.canvas;
|
||||
if (!canvas || e.defaultPrevented || (!(e.ctrlKey || e.metaKey) && scrolls(e.target, el, e.deltaX, e.deltaY))) return;
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
canvas.dispatchEvent(new WheelEvent("wheel", {
|
||||
deltaX: e.deltaX, deltaY: e.deltaY, deltaZ: e.deltaZ, deltaMode: e.deltaMode, clientX: e.clientX, clientY: e.clientY,
|
||||
screenX: e.screenX, screenY: e.screenY, ctrlKey: e.ctrlKey, shiftKey: e.shiftKey, altKey: e.altKey, metaKey: e.metaKey,
|
||||
bubbles: true, cancelable: true,
|
||||
}));
|
||||
}, { passive: false });
|
||||
}
|
||||
+123
-35
@@ -17,10 +17,10 @@ every shot inside that range.
|
||||
| node | what it does |
|
||||
|---|---|
|
||||
| **BFS Shot Planner** | Pick a video, split it on a timeline, set a reference and prompt per shot. Outputs the shots as a ComfyUI **list**. |
|
||||
| **BFS Shot Unpack** | Opens one shot into plain values: guide frames, reference, second reference, prompt, length, first frame, audio, size, index. Use it with any model. |
|
||||
| **BFS Shot Unpack** | Opens one shot into plain values: guide frames, reference pictures (a list), prompt, length, first frame, audio, size, index. Use it with any model. |
|
||||
| **BFS Shot Repack** | Puts edited pieces back into a shot: Unpack's `shot` output goes to Repack's `shot`, the edited piece (e.g. the reference with its background removed) to its input. Unconnected inputs keep their values; timing, cuts and audio are unchanged. |
|
||||
| **BFS Shot H3 Conditioning** | The common MiniMax H3 path with the important controls only: aligned Add Guide and/or native video reference, references, optional audio conditioning, the Planner's inpaint mode and source-subject blur. |
|
||||
| **BFS Shot H3 Conditioning (Advanced)** | The full conditioning node, preserving the previous node and old workflows. Adds duet canvas, shifted RoPE, prompt writer, setting reference and experimental mask-guide controls. |
|
||||
| **BFS Shot H3 Conditioning (Advanced)** | The full conditioning node. Adds the setting reference, the first-frame anchor and the experimental mask-guide controls; its `model` input passes through to `model` (older workflows routed the model through it). The duet lives in **BFS Shot H3 Duet** (1.113 took it out of this node). |
|
||||
| **BFS Shot Join** | Concatenates the decoded shots in order, trims each to its true length, cross-fades soft joins and returns the soundtrack. With *comparison* on it also returns a side-by-side video (original shot \| references \| result) with the shot's info and your *label* on top and its prompt below, ready for Create Video. *save to = mp4 file* writes the video (and the comparison) straight to MP4 files instead, one shot in memory at a time (long videos); `video_path` gives the file. |
|
||||
| **BFS Shot H3 Duet** | Renders one shot with H3: the shot's clip pinned beside the video (duet, no LoRA), the shot as an aligned guide (for body-swap LoRAs), or both. |
|
||||
|
||||
@@ -133,8 +133,10 @@ card manages them.
|
||||
to the planner). The next Queue **runs only the shots that are not stored or whose settings changed**, plus the
|
||||
shots that **continue from a shot that was made again** (head, history, previous shot as reference / first
|
||||
frame), so every seam matches. Shot cards show 💾 (stored, up to date) or ⏳ (runs next time).
|
||||
- **Model, LoRA, seed, sampler:** the planner does not see them. After changing them, redo the shots (↻) or *Redo all*,
|
||||
or *Reset loop* (deletes the stored shots of this video).
|
||||
- **Model, LoRA, seed, sampler (1.112.0):** the planner reads them from the queued workflow (every node that feeds
|
||||
BFS Shot Join's `images`: loaders, LoRAs, sampler, seed, conditioning settings), so changing any of them runs every
|
||||
shot again. The join's own settings (crossfade, comparison) don't. A seed set to *randomize* changes on every Queue,
|
||||
so every Queue then runs every shot; fix the seed to keep the stored shots.
|
||||
- **↻ Redo** a shot (Stored shots card): it runs again on the next Queue, and the shots that continue from it run
|
||||
after it. Tick *↻ redoes only that shot* to keep the next shots as they are (the seam after it may then not match).
|
||||
- **🎬 Combine stored shots:** puts the stored shots together into an MP4 in the output folder **now, without
|
||||
@@ -236,16 +238,26 @@ Face identities are used for filtering and optional shot splitting. References a
|
||||
|
||||
## References and prompts
|
||||
|
||||
Every shot can have its own reference, second reference and prompt; shots without them use the
|
||||
global ones. Click an empty reference slot to upload an image straight from your computer; a filled
|
||||
slot has **✕** to remove it; **Copy to other shots** gives a shot's references to every shot. The last ten
|
||||
references you picked are shown under the shot editor for one-click reuse (click = reference,
|
||||
shift+click = second reference). Connected `ref_image`, `ref_image_2` and `prompt` inputs override
|
||||
the panel's global values, so an edited first frame or a prompt from another node can drive the
|
||||
defaults.
|
||||
Up to **12 reference pictures** (1.113.0): the global list starts with one slot and **+** adds more; they are
|
||||
`<Picture 1>` to `<Picture 12>`, in order. Click a slot to upload a picture (the **+** takes several files at once),
|
||||
or drop image files on a slot (replaces it) or anywhere on the box (adds them). A filled slot has **✕**: a global
|
||||
picture is taken out (the next ones move up), a shot's own picture falls back to the global one.
|
||||
|
||||
The planner also outputs `ref_image` and `ref_image_2`: the references the shots actually use,
|
||||
without repeats, so a plan where every shot shares one reference returns a single image.
|
||||
Every shot can have its own pictures and prompt. A shot's empty slot uses the global picture at the same position
|
||||
(shown faded), so a shot can change only `<Picture 2>` and keep the rest. **Copy to other shots** gives a shot's
|
||||
pictures to every shot. The last ten pictures you picked are shown under the shot editor: click = picture 1,
|
||||
shift+click = add it as the next picture, or drag it onto any slot.
|
||||
|
||||
Under every prompt (global and per shot) there is a chip per picture: **PictureN** inserts `<Picture N>` at the
|
||||
cursor, **📝** inserts `{details@Picture N}` (that picture's description, below).
|
||||
|
||||
The node no longer has `ref_image` / `ref_image_2` inputs (the pictures live in the panel); a workflow saved with
|
||||
them connected still runs, and the log says the images are ignored. A connected `prompt` input still overrides the
|
||||
global prompt. The planner outputs `refs` (every picture the shots use, without repeats, as a list) and `ref_count`
|
||||
where `ref_image` / `ref_image_2` were. **BFS Shot Unpack** outputs `refs`, every picture of the shot as a list (the next node runs once per picture,
|
||||
e.g. a background remover), and `ref_count`; **BFS Shot Repack**'s `refs` input takes that list back and gives each
|
||||
shot its own pictures (a workflow from before 1.113 that wired `ref_image` still works). **BFS Shot H3 Conditioning** passes every picture of the shot (the `use_ref_2` switch left in 1.114: the pictures are chosen in the planner); H3's own
|
||||
node stops at 9 and LoRAs trained like OpenJutsu saw up to 9, so the log notes a shot with more.
|
||||
|
||||
## Mask & crop (SAM 3)
|
||||
|
||||
@@ -274,11 +286,18 @@ box (yellow) over the original column (`comparison_mask`, on by default).
|
||||
1 regenerates the masked area completely. A grey mask (e.g. 0.85) keeps part of the original there: H3 puts those
|
||||
rows at *opacity* × the noise level, so they start from the original partly visible. 0.8-0.9 keeps pose, outline and lighting while still swapping; too low copies the original
|
||||
person. With `guide_mode = aligned guide` the model also sees the whole original shot as its guide, so it follows
|
||||
the motion either way. (Not applied with the duet panel.)
|
||||
the motion either way.
|
||||
|
||||
**🎨 Colored Mark** is an optional translucent overlay on the source video. Enable it before entering a SAM 3 description or opening **🎯 Points**; the selected color applies to that shot's marked region. This is only a visual marker: it does not create an inpainting mask or change **Full frame**, **Mask only** or crop behavior. The default colors are set under each global reference in **Prompts & refs**; choose a per-shot color when needed. The overlay keeps movement and facial expression visible.
|
||||
|
||||
To mark two regions in one shot, use the main description or points for **Ref 1**, then open **Mark a second region** and describe the second region for **Ref 2**. Give the two references different marker colors and enable *use_ref_2* on BFS Shot H3 Conditioning. The second region currently uses a text description.
|
||||
**🎨 Colored Mark** is an optional translucent overlay on the source video: each marked person is tinted with the
|
||||
color of the picture they become, and the prompt gets the mapping ("the person marked in red becomes the character
|
||||
from `<Picture 3>`"), for any model or LoRA. Every picture has its **Mark color** (under it in **Prompts & refs**:
|
||||
green, purple, red, blue, yellow, magenta, cyan, orange, white, brown, pink, olive by default); change it when it is
|
||||
too close to a color in the video. In a shot, turn on Colored Mark and pick which picture the shot's selection (its
|
||||
SAM 3 text or points) becomes; **+ Mark another person** adds a person with their own picture and text or 🎯 points
|
||||
(the points window has one point set per person). This is only a visual marker: it does not create an inpainting mask
|
||||
or change **Full frame**, **Mask only** or crop behavior. More than one marked person needs Crop off, and the
|
||||
shot must have every picture the marks use. The overlay keeps movement and facial
|
||||
expression visible. Plans before 1.113 (a second region for ref 2) open with that region as `<Picture 2>`.
|
||||
|
||||
```text
|
||||
body_swap: subject_definitions: <Subject 1> is the character shown in <Picture 1>, whose identity, face, head, hair or fur, skin, body shape, proportions and clothing define the green-marked replacement. <Subject 2> is the character shown in <Picture 2>, whose identity, face, head, hair or fur, skin, body shape, proportions and clothing define the purple-marked replacement. In the guide video, the translucent green and purple tints are temporary editing markers, not character appearance. Replace the green-tinted person only with <Subject 1> and the purple-tinted person only with <Subject 2>. The guide video provides the scene and the complete performance of both people, including their body poses, gestures, head motion, gaze, speech and facial expressions.
|
||||
@@ -290,10 +309,8 @@ detailed_description: [Shot 1] Replace the person marked by the translucent gree
|
||||
|
||||
The VLM's shot analysis suggests a SAM 3 text selection from the original frames; it does not assign guide colors or
|
||||
decide that two detections are the same identity. **Describe target** only describes the first shot selection. For
|
||||
two-color swaps, choose both selections and identities yourself, and use the explicit mapping above. The VLM prompt
|
||||
writer for *duet* is a separate split-screen task; its generic character-swap draft assumes the supplied pictures
|
||||
show one character, so use *planner prompt* for this two-character template. `Describe refs` also treats ref and ref 2
|
||||
as views of one identity; write the two subject definitions explicitly when ref 2 is a different person.
|
||||
two-color swaps, choose both selections and identities yourself, and use the explicit mapping above (or let
|
||||
🤖 Auto prompt write it: it is told which tinted person becomes which picture).
|
||||
|
||||
**Mask guide** (experimental, *Mask only* shots, BFS Shot H3 Conditioning → `mask_guide`): the shot with only the
|
||||
masked region visible, the rest grey, so the model looks at the subject's pose and outline on its own while the mask
|
||||
@@ -357,22 +374,71 @@ editor shows its suggestion with buttons to apply it. The summary output lists t
|
||||
|
||||
The VLM can describe the references, for any task (not tied to swaps), and any subject: a person, a creature, a robot,
|
||||
an animal or a cartoon character (it says what the subject is first). In the **Prompts & refs** tab pick an instruction
|
||||
preset: *short* (the default: one noun phrase of at most ~20 words that fits inside a sentence, e.g. `green goblin with
|
||||
preset: *object* (a noun phrase for a prop, product, vehicle...: what it is, shape, materials, colors, text), *short* (the default: one noun phrase of at most ~20 words that fits inside a sentence, e.g. `green goblin with
|
||||
long pointed ears in a torn brown leather vest`, for `<Subject 1> is the {details} shown in <Picture 1>`), *full body* (face, hair, skin, age, build, clothing piece by piece), *head / face*, *face attributes* (a
|
||||
short comma-separated list), *outfit*, or *custom* (your own instruction). **📝 Describe refs** writes one
|
||||
description per reference set: the global references and every shot's. They are saved in
|
||||
the plan and editable. Write **`{details}`** anywhere in a prompt and every shot gets the description of its own
|
||||
references; a set without a description is described by the VLM at run time (no click needed).
|
||||
description per picture (see below), saved in the plan and editable. Write **`{details}`** anywhere in a prompt and
|
||||
every shot gets the description of its own first picture; a picture without a description is described by the VLM
|
||||
at run time (no click needed).
|
||||
|
||||
**Image max side (VLM)** controls the longest side of reference and shot images prepared for VLM analysis (1024 px by
|
||||
default). Aspect ratio is preserved; the model's own image processor may resize them further. Increase it when fine
|
||||
reference details matter, or lower it to reduce analysis cost. This does not change the generated video size.
|
||||
|
||||
When a shot has two references, the planner expands a one-reference template at run time to include `<Subject 2>` and
|
||||
`{details2}` for `<Picture 2>`. The Describe refs panel provides a separate description field for each reference.
|
||||
`{details2}` for `<Picture 2>` (not when the prompt already names `<Picture 2>` or uses `{details@Picture n}`).
|
||||
Existing one-reference templates remain unchanged in shots with only one reference. You can also use `{details2}`
|
||||
explicitly in any template.
|
||||
|
||||
### 🤖 Auto prompt (1.113.0)
|
||||
|
||||
In **Prompts & refs → 🤖 Auto prompt**, *Write the prompts with the VLM* turns the prompt boxes (global and per shot)
|
||||
into **requests** ("replace the bottle with the red can from Picture 2", "Picture 1 dances in the rain"), and the VLM
|
||||
writes each shot's MiniMax H3 prompt in the official full-reference format (MiniMaxAI/MiniMax-H3
|
||||
`docs/VIDEO_PROMPT_WRITING_GUIDE_ref_en.md`): `subject_definitions`, `summary` with its task type, `retention_analysis`,
|
||||
`detailed_description` (`[Shot 1]`, the official camera vocabulary, sounds in time), `overall_soundscape` and
|
||||
`non_diegetic_music`. It looks at:
|
||||
|
||||
- **key frames of the shot** with their times (*Key frames*: auto = about 1.5 per second, 3 to 8);
|
||||
- **the shot's pictures** and their descriptions (Describe references);
|
||||
- **the request**, the shot's **Colored Marks** (which tinted person becomes which picture; the mapping is added if the
|
||||
writer leaves a color out), its selection / `{target}` and its mode (*Mask only*: only the subject is regenerated;
|
||||
*Crop*: the frames are a crop);
|
||||
- **how the source is given**: *guide video* (aligned guide, no tag) or `<Video 1>` (native reference video) — match the
|
||||
conditioning node's `guide_mode`.
|
||||
|
||||
**Template:** with Auto prompt on, the planner's `prompt` input (e.g. the template a LoRA was trained with,
|
||||
`body_swap: subject_definitions: ...`) is the **template** and the panel's text is the **request** applied to it
|
||||
("Troque a mulher no vídeo pela mulher da <Picture 1>"). The template is kept as it is: same sentences, same order, its
|
||||
trigger word first; the writer applies the request, adds the pictures' visible features and, after `[Shot 1]`, this
|
||||
shot's camera, place, light and actions. The panel's 🤖 Write uses the template of the last run (shown in the card).
|
||||
A request that is itself a whole prompt (H3 sections) is used as the template the same way. **Style** picks how a prompt without template is written: *H3
|
||||
official* (the six sections) or *OpenJutsu captions*, the short style of the OpenJutsu training captions ("Replace the
|
||||
dog in the video with the gaunt goblin … from <Picture 1>. Replace the cat … from <Picture 2>. Keep the same background
|
||||
and location as in the video, and maintain the motion, camera angle, and timing."). Without a template, a
|
||||
short request gets the full six-section prompt. Without a VLM, the template (else the request) is the prompt.
|
||||
|
||||
**Cuts:** the writer gets the shot's real camera cuts (the planner's cut detector) and writes `[Shot N] At MM:SS.mmm`
|
||||
exactly there, with at least one key frame in each piece between cuts; a shot with no cut gets no timestamp at all.
|
||||
Labels always keep their `< >`.
|
||||
|
||||
**🤖 Write all prompts** (or *only the missing*) writes them now; each shot shows its written prompt in the Shots tab,
|
||||
editable, with **🤖 Write / Write again**. A shot without one is written when the workflow runs (with the VLM on the
|
||||
planner, before any SAM 3 work) and the panel keeps it, so the next run uses the same text. When the request,
|
||||
pictures, marks, selection or mode change after a prompt was written, the shot says it is *older*; it is still used
|
||||
until you write it again. A written prompt is part of the stored shot's signature, so editing it re-renders the shot.
|
||||
Without a VLM and without a written prompt, the request is used as the prompt.
|
||||
|
||||
### One description per picture (1.113.0)
|
||||
|
||||
Descriptions are per picture: **📝 Describe refs** describes every picture the plan uses by itself, once per image.
|
||||
`{details}` is picture 1's description, `{details2}` picture 2's and **`{details@Picture n}`** (or
|
||||
`{details@Picture3}`) picture n's, so each shot gets the descriptions of *its* pictures. The dropdown next to the button
|
||||
sets what is described for every picture; a picture can have its own (e.g. *full body* for a character sheet, *head /
|
||||
face* for a close-up, *outfit*, or *custom* with its own instruction), and **🔄** describes only that picture again. A
|
||||
picture without a description is described at run time by the planner's VLM, else by the conditioning node's VLM, else
|
||||
it becomes "character". Descriptions saved by older plans (per reference set) are read as those pictures'.
|
||||
|
||||
**Fit the prompt** (on by default, next to 📝 Describe refs): the VLM gets the sentence of the prompt that holds the
|
||||
marker, with a blank in its place (e.g. `<Subject 1> is the ___ shown in <Picture 1>`), and writes the description for
|
||||
it, so the description adapts to the prompt and not the other way round (no "the A goblin", no paragraph inside a
|
||||
@@ -433,15 +499,37 @@ references before any SAM 3 work; with no VLM on the planner, the slot is left f
|
||||
**BFS Shot H3 Duet**, whose `vlm` input (the same Qwen3-VL CLIPLoader) describes them (the *short* preset, fitted to
|
||||
the sentence). Edited descriptions always win. With no VLM anywhere it becomes "character" and the log says so.
|
||||
|
||||
## Source blur (keep the mouth) (1.91.0)
|
||||
## Source blur (1.91.0, finer in 1.110.0)
|
||||
|
||||
`source_blur` on **BFS Shot H3 Conditioning**, **BFS Shot H3 Duet**, **BFS H3 Duet** and **BFS H3 Duet Conditioning**
|
||||
blurs the source person in what the model *sees* of the source (aligned guide, native reference video, duet panel),
|
||||
keeping the **mouth** sharp: the model still reads the pose, the motion and the lips but has no face to copy, so the
|
||||
identity comes from the reference. Masks from SAM 3.1 (the part, minus a slightly grown "mouth" mask), cached per shot.
|
||||
`face (keep mouth)`, `head + hands (keep mouth)` (also hides hair and hands), `person (keep mouth)` (the whole body:
|
||||
only for big changes, the pose then comes from the blurred shape). The kept area of *Mask only* and the output are
|
||||
never blurred. Trick shared by JalenBrunson (Sapiens2 there) for reliable ref2v swaps.
|
||||
`source_blur` on **BFS Shot H3 Conditioning** (and its Simple version), **BFS Shot H3 Duet**, **BFS H3 Duet** and
|
||||
**BFS H3 Duet Conditioning** blurs the source person in what the model *sees* of the source (aligned guide, native
|
||||
reference video, duet panel): the model still reads the pose and the motion but has no face to copy, so the identity
|
||||
comes from the reference. The kept area of *Mask only* and the output are never blurred. Trick shared by JalenBrunson
|
||||
(Sapiens2 there) for reliable ref2v swaps.
|
||||
|
||||
Modes read `<region> (<what stays sharp>)`:
|
||||
|
||||
| region | what is blurred |
|
||||
|---|---|
|
||||
| `face` | the face (and neck) |
|
||||
| `head + hands` | also the hair and the hands |
|
||||
| `person` | the whole body: only for big changes, the pose then comes from the blurred shape |
|
||||
| `masked area` | the shot's own mask (the person being replaced, with its expand); no extra segmentation. Without a mask on the shot (or in the plain Duet nodes) it blurs the whole person |
|
||||
|
||||
| sharp | |
|
||||
|---|---|
|
||||
| `keep mouth` | lips, teeth and tongue stay sharp (the lips carry a lot of the likeness and the lip sync) |
|
||||
| `keep lips + tongue` | the teeth are blurred too (always uses Sapiens2 for the lips) |
|
||||
| `mouth too` | nothing stays sharp |
|
||||
|
||||
`source_blur_segmenter`: **SAM 3.1** (text prompts, already used by the planner; the 1.91 behaviour) or **Sapiens2**
|
||||
(Meta's body-part segmentation, 28 classes with lips, teeth and tongue apart from the face; one pass for every part,
|
||||
finer around the mouth). Sapiens2 marks every person in the frame, like the SAM prompts here. Its weights
|
||||
(`facebook/sapiens2-seg-0.4b`, 1.6 GB) download to `models/sapiens2/` the first time; it runs on the GPU in half
|
||||
precision and goes back to RAM after each shot. Sapiens2 License.
|
||||
|
||||
`source_blur_effect`: **blur** (no detail left) or **invert colours** (keeps the shapes and the expression but not the
|
||||
skin tone or the colours; some users get better likeness with it).
|
||||
|
||||
## Big videos (upload)
|
||||
|
||||
@@ -529,7 +617,7 @@ BFS Shot Join ◄── images ────────────────
|
||||
└─► Create Video (fps, audio) ─► Save Video
|
||||
```
|
||||
|
||||
For other models, use **BFS Shot Unpack** and wire `guide_frames`, `ref_image`, `prompt` and
|
||||
For other models, use **BFS Shot Unpack** and wire `guide_frames`, `refs`, `prompt` and
|
||||
`length` into that model's own guide and reference nodes; the list mapping works the same way.
|
||||
|
||||
## Choosing who is replaced (`{target}`), step by step
|
||||
|
||||
@@ -371,11 +371,11 @@ export const rawH = (...args) => vueH(...args);
|
||||
export const GRIDS = { "H3 (17n+5)": [17, 5], "LTX / Wan (8n+1)": [8, 1], "Wan (4n+1)": [4, 1], "any": [1, 0] };
|
||||
export const DEFAULTS = {
|
||||
video: "", fps: 24, grid: "H3 (17n+5)", mode: "shots", max_s: 4.5, min_s: 1.0, sensitivity: 0.5,
|
||||
max_parts: 0, max_total_s: 0, start_s: 0, end_s: 0, start_f: null, end_f: null, track: null, bounds: [], segs: [], global_ref: "", global_ref2: "", global_prompt: "", global_color_guide_ref1: "#00ff00", global_color_guide_ref2: "#9b30ff", mask_video: "",
|
||||
max_parts: 0, max_total_s: 0, start_s: 0, end_s: 0, start_f: null, end_f: null, track: null, bounds: [], segs: [], global_refs: [], global_ref: "", global_ref2: "", global_prompt: "", global_color_guide_ref1: "#00ff00", global_color_guide_ref2: "#9b30ff", mask_video: "",
|
||||
megapixels: 0.5, multiple: 32, generation_size_mode: "megapixels", custom_width: 1024, custom_height: 1024,
|
||||
second_pass: { enabled: false, mode: "scale", scale: 2, width: 1280, height: 704, megapixels: 1, align: 32 },
|
||||
detector: "adaptive", run: "recursive", between_shots: "keep", shot_cache: "output", shot_storage: "lossless", auto_continue: true,
|
||||
skip_fill: "original", audio_mode: "auto", ref_details: {}, mask_cfg: {}, vlm_cfg: {}, cast: {}, cast_split: false, cast_only: false,
|
||||
detector: "adaptive", run: "recursive", between_shots: "keep", shot_cache: "output", shot_storage: "lossless", unload_before_join: true, auto_continue: true,
|
||||
skip_fill: "original", audio_mode: "auto", ref_details: {}, picture_details: {}, auto_prompt: {}, auto_prompts: {}, mask_cfg: {}, vlm_cfg: {}, cast: {}, cast_split: false, cast_only: false,
|
||||
filters: { person: false, min_person_area: 0, max_persons: 0, face: false, skip_dark: false, dark_level: 0.06,
|
||||
skip_static: false, static_level: 0.004, min_frames: 0, samples: 6 },
|
||||
};
|
||||
@@ -470,3 +470,84 @@ export const secondPassSize = (w, h, sp) => {
|
||||
const al = Math.max(1, Math.round(+sp.align || 1)), r = x => Math.round(x / al) * al, c16 = x => Math.max(16, Math.round(x / 16) * 16);
|
||||
return { w: c16(r(tw)), h: c16(r(th)) };
|
||||
};
|
||||
|
||||
// ---- reference pictures (<Picture 1..12>)
|
||||
export const MAX_REFS = 12;
|
||||
// a reference list of the plan: `refs` (plans from 1.113 on), else the old ref / ref2 pair; empty slots at the end dropped
|
||||
export const refList = (refs, a = "", b = "") => {
|
||||
const out = (Array.isArray(refs) ? refs : [a, b]).map(x => (typeof x === "string" ? x : "")).slice(0, MAX_REFS);
|
||||
while (out.length && !out[out.length - 1]) out.pop();
|
||||
return out;
|
||||
};
|
||||
// the pictures a shot uses, in <Picture n> order: its own slot, else the global one at the same position; a slot with
|
||||
// neither is left out (same rule as bfs_shot_loop.shot_ref_names)
|
||||
export const shotRefNames = (own, glob) => {
|
||||
const out = [];
|
||||
for (let k = 0; k < Math.max(own.length, glob.length); k++) { const x = own[k] || glob[k]; if (x) out.push(x); }
|
||||
return out.slice(0, MAX_REFS);
|
||||
};
|
||||
// one slot per picture, then "+" (up to 12). Click: upload (the "+" takes several files at once); drop image files or a
|
||||
// recent thumbnail on a slot to replace it, anywhere else in the box to add them. `fallback`: the global pictures a
|
||||
// shot's empty slots use (shown faded). `extra(k, name)`: more controls under slot k.
|
||||
export function refBox(c, target, own, fallback = [], extra = () => null) {
|
||||
const n = Math.max(own.length, fallback.length);
|
||||
const drop = k => ({ onDragover: e => e.preventDefault(),
|
||||
onDrop: e => { e.preventDefault(); e.stopPropagation(); c.dropRefs(target, k, e); } });
|
||||
const slots = [];
|
||||
for (let k = 0; k < n; k++) {
|
||||
const name = own[k] || "", via = name ? "" : fallback[k] || "";
|
||||
slots.push(h("div", { class: "rslot" }, [
|
||||
h("div", { class: "refslot", ...drop(k),
|
||||
title: name ? `<Picture ${k + 1}>: ${name} (click to replace)` : via ? `<Picture ${k + 1}>: ${via} (global; click to use another picture in this shot)` : "click to upload",
|
||||
onClick: () => c.uploadRef(target, k) },
|
||||
name || via ? [h("img", { src: c.viewUrl(name || via), style: via ? "opacity:.45" : "" }), h("span", { class: "rnum" }, String(k + 1))]
|
||||
: [`⬆ ${k + 1}`]),
|
||||
h("div", { class: "rbtns" }, [name ? h("button", { title: target === "global" || k >= fallback.length ? "remove this picture"
|
||||
: "remove (use the global picture)", onClick: () => c.setRef(target, k, "") }, "✕") : null]),
|
||||
extra(k, name || via),
|
||||
]));
|
||||
}
|
||||
if (n < MAX_REFS) slots.push(h("div", { class: "rslot" }, [
|
||||
h("div", { class: "refslot add", ...drop(n), title: `add pictures (up to ${MAX_REFS}): click to upload one or more, or drop image files here`,
|
||||
onClick: () => c.uploadRef(target, n, true) }, "+"),
|
||||
h("div", { class: "rbtns" }, []),
|
||||
]));
|
||||
return h("div", { class: "refbox", ...drop(null) }, slots);
|
||||
}
|
||||
// chips under a prompt, one per picture: PictureN inserts <Picture N> at the cursor, 📝 inserts {details@Picture N}
|
||||
// (the description of that picture alone). The prompt box and its chips sit in one .pbox.
|
||||
export function promptChips(c, names, onSet) {
|
||||
if (!names.length) return null;
|
||||
const insert = (e, tag) => {
|
||||
e.preventDefault(); // mousedown: the prompt keeps its focus and cursor
|
||||
const ta = e.currentTarget.closest(".pbox")?.querySelector("textarea"); if (!ta) return;
|
||||
const v = ta.value, a = ta.selectionStart ?? v.length, b = ta.selectionEnd ?? v.length;
|
||||
const before = v.slice(0, a) + (a && !/\s$/.test(v.slice(0, a)) ? " " : "") + tag;
|
||||
onSet(before + v.slice(b));
|
||||
requestAnimationFrame(() => { ta.focus(); ta.setSelectionRange(before.length, before.length); });
|
||||
};
|
||||
return h("div", { class: "pchips" }, names.map((name, k) => h("span", { class: "pchip" }, [
|
||||
h("button", { class: "sm", title: `insert <Picture ${k + 1}> (${name})`, onMousedown: e => insert(e, `<Picture ${k + 1}>`) },
|
||||
[h("img", { src: c.viewUrl(name) }), `Picture${k + 1}`]),
|
||||
h("button", { class: "sm", title: `insert {details@Picture ${k + 1}}: the description of this picture (📝 Describe refs, or the VLM at run time)`,
|
||||
onMousedown: e => insert(e, `{details@Picture ${k + 1}}`) }, "📝"),
|
||||
])));
|
||||
}
|
||||
|
||||
// ---- Colored Mark: one color per picture (<Picture 1> first; the first seven are the colors of the first mark sets)
|
||||
export const MARK_PALETTE = [["#00ff00", "green"], ["#9b30ff", "purple"], ["#ff2020", "red"], ["#168cff", "blue"],
|
||||
["#ffe600", "yellow"], ["#ff00d4", "magenta"], ["#00e5c0", "cyan"], ["#ff8000", "orange"], ["#ffffff", "white"],
|
||||
["#8b4513", "brown"], ["#ff9ec4", "pink"], ["#808000", "olive"]];
|
||||
export const markName = color => (MARK_PALETTE.find(([v]) => v === String(color).toLowerCase()) || [0, color])[1];
|
||||
// the mark color of <Picture k> (same rule as bfs_shot_loop.picture_color)
|
||||
export const pictureColor = (plan, k) => {
|
||||
const c = Array.isArray(plan.picture_colors) ? plan.picture_colors[k - 1] : "";
|
||||
return /^#[0-9a-f]{6}$/i.test(c || "") ? c.toLowerCase() : MARK_PALETTE[(k - 1) % MARK_PALETTE.length][0];
|
||||
};
|
||||
// a shot's extra Colored Mark regions (the second person on): plans before 1.113 had one (text2 / points2 = picture 2)
|
||||
export const markRegions = m => Array.isArray(m.color_guide_regions) ? m.color_guide_regions
|
||||
: (m.color_guide_text2 || (m.color_guide_points2 || []).length)
|
||||
? [{ picture: 2, text: m.color_guide_text2 || "", points: m.color_guide_points2 || [], key: m.color_guide_key2 || 0 }] : [];
|
||||
|
||||
// ---- 🤖 Auto prompt: the VLM writes each shot's H3 prompt from the request in the prompt box
|
||||
export const AUTO_DEFAULTS = { enabled: false, frames: 0, source: "guide", style: "h3" };
|
||||
|
||||
@@ -140,6 +140,14 @@ const CSS = `
|
||||
.bsl .refslot:hover{border-color:var(--acc)}
|
||||
.bsl .refslot.sm{width:56px;height:56px;font-size:10px}
|
||||
.bsl .refslot.via{border-style:solid;border-color:#3b5b3b}
|
||||
.bsl .refslot{position:relative}
|
||||
.bsl .refslot .rnum{position:absolute;left:3px;top:2px;font-size:10px;padding:0 4px;border-radius:4px;background:#000a;color:#fff}
|
||||
.bsl .refslot.add{font-size:22px;color:#8a8a99}
|
||||
.bsl .refbox{flex-wrap:wrap;max-width:100%;align-items:flex-start}
|
||||
.bsl .pchips{display:flex;gap:4px;flex-wrap:wrap;margin-top:4px}
|
||||
.bsl .pchip{display:inline-flex;gap:1px}
|
||||
.bsl .pchip button{display:inline-flex;align-items:center;gap:3px;padding:1px 5px;font-size:10px}
|
||||
.bsl .pchip img{width:16px;height:16px;border-radius:3px;object-fit:cover}
|
||||
.bsl .rslot{display:flex;flex-direction:column;gap:3px;align-items:center}
|
||||
.bsl .guidecolor{display:flex;flex-direction:column;align-items:center;gap:2px;max-width:112px}
|
||||
.bsl .guidecolor select{width:100%;font-size:10px;padding:2px 4px}
|
||||
@@ -191,13 +199,22 @@ const CSS = `
|
||||
|
||||
/* points window (teleported to <body>) */
|
||||
.bsl .mmodal{position:fixed;inset:0;background:rgba(0,0,0,.72);z-index:10000;display:flex;align-items:center;justify-content:center}
|
||||
.bsl .mbox{background:#1d1d23;border:1px solid #3a3a45;border-radius:10px;padding:12px;width:min(960px,94vw);max-height:94vh;overflow:auto}
|
||||
.bsl .mimg{position:relative;cursor:crosshair;user-select:none;line-height:0}
|
||||
.bsl .mimg img{width:100%;border-radius:6px}
|
||||
.bsl .mbox{background:#1d1d23;border:1px solid #3a3a45;border-radius:10px;padding:12px;width:min(960px,94vw);height:min(900px,94vh);overflow:auto;box-sizing:border-box}
|
||||
/* the frame fits a fixed stage whatever the video's size or orientation; points are stored relative to the frame */
|
||||
.bsl .mstage{display:flex;justify-content:center;align-items:center;height:min(540px,56vh);background:#0d0d10;border-radius:6px}
|
||||
.bsl .mimg{position:relative;cursor:crosshair;user-select:none;line-height:0;display:inline-block}
|
||||
.bsl .mimg img{display:block;max-width:100%;max-height:min(540px,56vh);border-radius:6px}
|
||||
.bsl .mthumb{width:22px;height:22px;border-radius:4px;object-fit:cover;vertical-align:middle;border:2px solid var(--mark,#555)}
|
||||
.bsl .pt{position:absolute;width:14px;height:14px;margin:-7px 0 0 -7px;border-radius:50%;border:2px solid #fff;cursor:pointer}
|
||||
.bsl .pt.pos{background:#3ccf6b}.bsl .pt.neg{background:#e8455a}
|
||||
.bsl .pt.cg.pos{background:var(--point-color,#00ff00)}
|
||||
.bsl .pt.r2{box-shadow:0 0 0 2px #9b30ff}
|
||||
.bsl .pt.other{opacity:.5}
|
||||
.bsl .kslider{position:relative;flex:1;min-width:160px;padding-bottom:8px}
|
||||
.bsl .kmarks{position:absolute;left:8px;right:8px;bottom:0;height:8px}
|
||||
.bsl .kmark{position:absolute;width:8px;height:8px;margin-left:-4px;border-radius:50%;background:var(--mark,#3ccf6b);opacity:.55;cursor:pointer;border:1px solid #000}
|
||||
.bsl .kmark.on{opacity:1}
|
||||
.bsl .kchip{display:inline-flex;gap:1px}.bsl .kchip.on button:first-child{border-color:var(--acc)}
|
||||
.bsl .mreg{display:flex;gap:6px;align-items:center;flex-wrap:wrap;margin-top:6px}
|
||||
`;
|
||||
|
||||
export function styles() {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// The busy bar and the points window (🎯 Points…: click on a frame to pick the subject for SAM 3).
|
||||
import { Teleport } from "../vendor/vue.esm-browser.prod.mjs";
|
||||
import { h, hint, t } from "./common.js";
|
||||
import { h, hint, t, pictureColor, markName } from "./common.js";
|
||||
|
||||
export function busyBar(c, msg) {
|
||||
const { busySince, now, upPct, status } = c;
|
||||
@@ -15,65 +15,106 @@ export function busyBar(c, msg) {
|
||||
}
|
||||
|
||||
export function pointsModal(c) {
|
||||
const { modal, segs, plan, api, setMeta } = c;
|
||||
const { modal, segs, plan, api } = c;
|
||||
const ms = segs.value[modal.idx];
|
||||
if (!modal.open || !ms) return null;
|
||||
const pointOverlays = ms.color_guide
|
||||
? [1, 2].flatMap(region => (region === modal.region ? modal.points : modal.groups[region]?.points || []).map((p, k) => h("div", {
|
||||
class: ["pt", "cg", region === 2 && "r2", p.label ? "pos" : "neg"],
|
||||
style: `left:${p.x * 100}%;top:${p.y * 100}%;--point-color:${region === 2 ? (ms.color_guide_color2 || plan.global_color_guide_ref2 || "#9b30ff") : (ms.color_guide_color || plan.global_color_guide_ref1 || "#00ff00")}`,
|
||||
title: `Region ${region}${region === modal.region ? " · click to remove" : ""}`,
|
||||
onPointerdown: e => { if (region !== modal.region) return; e.stopPropagation(); modal.points = modal.points.filter((_, j) => j !== k); modal.prev = null; },
|
||||
})))
|
||||
: modal.points.map((p, k) => h("div", { class: ["pt", p.label ? "pos" : "neg"], style: `left:${p.x * 100}%;top:${p.y * 100}%`,
|
||||
title: "click to remove", onPointerdown: e => { e.stopPropagation(); modal.points = modal.points.filter((_, j) => j !== k); modal.prev = null; } }));
|
||||
// Colored Mark: region 1 = the shot's own selection, 2.. = its extra regions; each one is a picture, in its color
|
||||
const regionPicture = r => (r === 1 ? ms.color_guide_picture : ms.regions[r - 2]?.picture) || 1;
|
||||
const regionColor = r => (ms.color_guide ? pictureColor(plan, regionPicture(r)) : "#3ccf6b");
|
||||
const regionsN = ms.color_guide ? 1 + ms.regions.length : 1;
|
||||
const regions = Array.from({ length: regionsN }, (_, q) => q + 1);
|
||||
// the points of a region on the frame shown (the one being edited are in modal.points)
|
||||
const pointsAt = r => (r === modal.region ? modal.points : modal.groups[r]?.keys.find(k => k.key === modal.key)?.points || []);
|
||||
const keysOf = r => {
|
||||
const ks = new Set((modal.groups[r]?.keys || []).map(k => k.key));
|
||||
if (r === modal.region) { if (modal.points.length) ks.add(modal.key); else ks.delete(modal.key); }
|
||||
return [...ks].sort((a, b) => a - b);
|
||||
};
|
||||
const edited = () => { modal.stale = !!modal.prev; };
|
||||
const pointOverlays = regions.flatMap(r => pointsAt(r).map((p, k) => h("div", {
|
||||
class: ["pt", ms.color_guide && "cg", r !== modal.region && "other", p.label ? "pos" : "neg"],
|
||||
style: `left:${p.x * 100}%;top:${p.y * 100}%;--point-color:${regionColor(r)}`,
|
||||
title: r === modal.region ? "click to remove" : `Picture ${regionPicture(r)}`,
|
||||
onPointerdown: e => { if (r !== modal.region) return; e.stopPropagation(); modal.points = modal.points.filter((_, j) => j !== k); edited(); },
|
||||
})));
|
||||
const regionButton = r => {
|
||||
const k = regionPicture(r), name = ms.names[k - 1];
|
||||
return h("button", { class: modal.region === r ? "pri" : "", onClick: () => c.switchPointRegion(r),
|
||||
title: `Region ${r}: becomes <Picture ${k}> (${markName(regionColor(r))})` }, [
|
||||
name ? h("img", { class: "mthumb", src: c.viewUrl(name), style: `--mark:${regionColor(r)}` }) : null,
|
||||
` Picture ${k} · ${markName(regionColor(r))} (${keysOf(r).length} key frame${keysOf(r).length === 1 ? "" : "s"})`]);
|
||||
};
|
||||
// after Segment: the mask over every frame of the shot (modal.prev.frames, one per frame)
|
||||
const masked = modal.prev && modal.showMask ? modal.prev.frames.find(f => f.f === ms.start + modal.key) : null;
|
||||
const play = () => {
|
||||
if (modal.playing) { clearInterval(modal.playing); modal.playing = 0; return; }
|
||||
modal.playing = setInterval(() => {
|
||||
if (modal.key >= ms.len - 1 || !modal.open) { clearInterval(modal.playing); modal.playing = 0; return; }
|
||||
c.showFrame(modal.key + 1);
|
||||
}, 1000 / Math.min(24, plan.fps || 24));
|
||||
};
|
||||
const marks = h("div", { class: "kmarks" }, regions.flatMap(r => keysOf(r).map(f => h("span", {
|
||||
class: ["kmark", r === modal.region && "on"], style: `left:${(f / Math.max(1, ms.len - 1)) * 100}%;--mark:${regionColor(r)}`,
|
||||
title: `key frame ${ms.start + f}`, onClick: () => c.showFrame(f) }))));
|
||||
return h(Teleport, { to: "body" }, h("div", { class: "bsl", style: "background:none;padding:0" }, h("div", { class: "mmodal",
|
||||
onKeydown: e => e.stopPropagation(), onPointerdown: e => { if (e.target === e.currentTarget) modal.open = false; } }, [
|
||||
h("div", { class: "mbox" }, [
|
||||
onKeydown: e => {
|
||||
e.stopPropagation();
|
||||
if (e.target.tagName === "INPUT" && e.target.type === "text") return;
|
||||
if (e.key === "ArrowLeft") c.showFrame(modal.key - 1);
|
||||
if (e.key === "ArrowRight") c.showFrame(modal.key + 1);
|
||||
}, onPointerdown: e => { if (e.target === e.currentTarget) modal.open = false; } }, [
|
||||
h("div", { class: "mbox", tabindex: 0 }, [
|
||||
h("div", { class: "row", style: "margin-bottom:6px" }, [h("span", { class: "sect" }, `Shot #${modal.idx + 1} · points`),
|
||||
ms.color_guide ? h("label", { class: "row", style: "gap:6px;align-items:center" }, [h("span", { class: "hint" }, `Region ${modal.region} · color`),
|
||||
h("select", { value: modal.region === 2 ? (ms.color_guide_color2 || plan.global_color_guide_ref2 || "#9b30ff") : (ms.color_guide_color || plan.global_color_guide_ref1 || "#00ff00"), style: "width:auto", onChange: e => {
|
||||
setMeta(modal.idx, modal.region === 2 ? "color_guide_color2" : "color_guide_color", e.target.value);
|
||||
if (modal.prev) c.previewMask(modal.idx, { points: modal.points, key: modal.key, text: modal.text,
|
||||
preview_color: e.target.value }, true);
|
||||
} },
|
||||
[["#00ff00", "Green"], ["#9b30ff", "Purple"], ["#ff2020", "Red"], ["#168cff", "Blue"], ["#ffe600", "Yellow"], ["#ff00d4", "Magenta"], ["#00e5c0", "Cyan"]].map(([v, n]) => h("option", { value: v, disabled: ms.color_guide && v === (modal.region === 2 ? (ms.color_guide_color || "#00ff00") : (ms.color_guide_color2 || "#9b30ff")) }, n)))]) : null,
|
||||
ms.color_guide ? h("div", { class: "row" }, [hint("Colored Mark keeps two independent point sets; switch regions to edit each person. Both use the frame selected below."),
|
||||
h("button", { class: modal.region === 1 ? "pri" : "", onClick: () => c.switchPointRegion(1) }, `Region 1 (${modal.groups[1]?.points?.length || 0})`),
|
||||
h("button", { class: modal.region === 2 ? "pri" : "", onClick: () => c.switchPointRegion(2) }, `Region 2 (${modal.groups[2]?.points?.length || 0})`)]) : null,
|
||||
hint("click = keep (green) · right-click or shift+click = exclude (red) · click a point to remove it")]),
|
||||
ms.color_guide ? h("div", { class: "row" }, regions.map(regionButton)) : null,
|
||||
hint("click = keep · right-click or shift+click = exclude · click a point to remove it · points on another frame = a new key frame: the mask is tracked from each key frame up to the next one")]),
|
||||
h("div", { class: "mstage" }, [
|
||||
h("div", { class: "mimg", onContextmenu: e => e.preventDefault(), onPointerdown: e => {
|
||||
const box = e.currentTarget.getBoundingClientRect();
|
||||
const x = (e.clientX - box.left) / box.width, y = (e.clientY - box.top) / box.height;
|
||||
modal.points = [...modal.points, { x: +x.toFixed(4), y: +y.toFixed(4), label: (e.button === 2 || e.shiftKey) ? 0 : 1 }];
|
||||
modal.prev = null;
|
||||
edited();
|
||||
} }, [
|
||||
h("img", { draggable: false, src: api.apiURL(`/bfs/shotloop/frame?video=${encodeURIComponent(plan.video)}&fps=${plan.fps}&f=${ms.start + modal.key}&w=900`) }),
|
||||
h("img", { draggable: false, src: masked ? masked.src : api.apiURL(`/bfs/shotloop/frame?video=${encodeURIComponent(plan.video)}&fps=${plan.fps}&f=${ms.start + modal.key}&w=900`) }),
|
||||
...pointOverlays,
|
||||
]),
|
||||
h("div", { class: "row", style: "margin-top:6px" }, [
|
||||
])]),
|
||||
h("div", { class: "row", style: "margin-top:6px;align-items:center" }, [
|
||||
h("button", { title: "play the shot (with the mask after Segment)", onClick: play }, modal.playing ? "⏸" : "▶"),
|
||||
h("button", { title: "previous frame (←)", onClick: () => c.showFrame(modal.key - 1) }, "◀"),
|
||||
h("button", { title: "next frame (→)", onClick: () => c.showFrame(modal.key + 1) }, "▶|"),
|
||||
hint(`frame ${ms.start + modal.key} (${modal.key + 1}/${ms.len})`),
|
||||
h("input", { type: "range", min: 0, max: ms.len - 1, step: 1, value: modal.key, style: "flex:1",
|
||||
onInput: e => { modal.key = parseInt(e.target.value); modal.prev = null; } }),
|
||||
h("div", { class: "kslider" }, [
|
||||
h("input", { type: "range", min: 0, max: ms.len - 1, step: 1, value: modal.key, style: "width:100%",
|
||||
onInput: e => c.showFrame(parseInt(e.target.value)) }),
|
||||
marks,
|
||||
]),
|
||||
modal.prev ? h("label", { class: "row", style: "gap:4px" }, [h("input", { type: "checkbox", checked: modal.showMask,
|
||||
onChange: e => { modal.showMask = e.target.checked; } }), "mask"]) : null,
|
||||
]),
|
||||
h("div", { class: "row", style: "margin-top:4px;gap:4px;flex-wrap:wrap" }, [
|
||||
hint("key frames:"),
|
||||
...(keysOf(modal.region).length ? keysOf(modal.region).map(f => h("span", { class: ["kchip", f === modal.key && "on"] }, [
|
||||
h("button", { class: "sm", onClick: () => c.showFrame(f) }, `frame ${ms.start + f}`),
|
||||
h("button", { class: "sm dng", title: "remove this key frame's points", onClick: () => { c.dropKeyFrame(f); edited(); } }, "✕")]))
|
||||
: [hint("none yet: click on the frame to add points")]),
|
||||
]),
|
||||
h("div", { class: "row", style: "margin-top:6px" }, [
|
||||
h("input", { type: "text", value: modal.text, style: "flex:1", placeholder: "text prompt (used when there are no points)",
|
||||
onChange: e => { modal.text = e.target.value; } }),
|
||||
h("button", { onClick: () => { modal.points = []; modal.prev = null; } }, "Clear points"),
|
||||
h("button", { class: "pri", disabled: !!modal.busy || !(modal.points.length || modal.text),
|
||||
onClick: () => c.previewMask(modal.idx, { points: modal.points, key: modal.key, text: modal.text,
|
||||
preview_color: modal.region === 2 ? (ms.color_guide_color2 || plan.global_color_guide_ref2 || "#9b30ff")
|
||||
: (ms.color_guide_color || plan.global_color_guide_ref1 || "#00ff00") }, true) }, "👁 Segment"),
|
||||
onChange: e => { modal.text = e.target.value; edited(); } }),
|
||||
h("button", { title: "remove the points of this frame", onClick: () => { modal.points = []; edited(); } }, "Clear frame"),
|
||||
h("button", { class: "pri", disabled: !!modal.busy || !(keysOf(modal.region).length || modal.text),
|
||||
onClick: () => c.previewMask(modal.idx, { ...c.pointSpec(), ...(modal.region > 1 ? { max_objects: 1 } : {}),
|
||||
preview_color: regionColor(modal.region) }, true) }, modal.prev && modal.stale ? "👁 Segment again" : "👁 Segment"),
|
||||
]),
|
||||
modal.busy ? busyBar(c, modal.busy) : null,
|
||||
modal.prev ? h("div", { class: "mstrip" }, [...modal.prev.frames.map(f => h("img", { src: f.src, title: `frame ${f.f}` })),
|
||||
hint(modal.prev.empty ? "nothing found" : `covers ${(modal.prev.coverage * 100).toFixed(1)}%`)]) : null,
|
||||
modal.prev ? hint(modal.prev.empty ? "nothing found" : `covers ${(modal.prev.coverage * 100).toFixed(1)}% of the frames`
|
||||
+ (modal.stale ? " · the points changed: Segment again to see the new mask" : " · play or scrub to check every frame; where it slips, add points on that frame"),
|
||||
"display:block;margin-top:4px") : null,
|
||||
h("div", { class: "row", style: "margin-top:8px;justify-content:flex-end" }, [
|
||||
h("button", { onClick: () => { modal.open = false; } }, "Cancel"),
|
||||
!ms.color_guide ? h("button", { title: "Apply these points to EVERY shot, each on its frame at the same relative position (a subject that stays in place)",
|
||||
!ms.color_guide ? h("button", { title: "Apply this frame's points to EVERY shot, each on its frame at the same relative position (a subject that stays in place)",
|
||||
onClick: c.savePointsAll }, "Save → all shots") : null,
|
||||
h("button", { disabled: !modal.points.length || ms.color_guide,
|
||||
title: "Track this selection with SAM 3 through the whole video (or the range), forward and backward from this frame; every shot without its own selection takes its part of the track. Also type a short text (e.g. 'woman in pink top') so the subject is found again after camera cuts.",
|
||||
title: "Track this frame's selection with SAM 3 through the whole video (or the range), forward and backward from this frame; every shot without its own selection takes its part of the track. Also type a short text (e.g. 'woman in pink top') so the subject is found again after camera cuts.",
|
||||
onClick: c.saveTrack }, "🎯 Track → whole video"),
|
||||
h("button", { class: "pri", onClick: c.savePoints }, "Save (this shot)"),
|
||||
]),
|
||||
|
||||
@@ -1,42 +1,37 @@
|
||||
// 📝 Prompts & refs tab: the global reference / prompt, the VLM (shot suggestions) and the reference descriptions ({details}).
|
||||
import { h, fld, check, section, pill, hint, select } from "./common.js";
|
||||
import { h, fld, check, section, pill, hint, select, refBox, promptChips, MAX_REFS, MARK_PALETTE, pictureColor } from "./common.js";
|
||||
|
||||
const GUIDE_COLORS = [["#00ff00", "Green"], ["#9b30ff", "Purple"], ["#ff2020", "Red"], ["#168cff", "Blue"],
|
||||
["#ffe600", "Yellow"], ["#ff00d4", "Magenta"], ["#00e5c0", "Cyan"]];
|
||||
const PRESETS = ["short", "object", "full body", "head / face", "face attributes", "outfit", "custom"];
|
||||
|
||||
function globalSection(c) {
|
||||
const { plan } = c;
|
||||
const slot = (name, label, field) => h("div", { class: "rslot" }, [
|
||||
h("div", { class: "refslot", title: name ? name + " (click to replace)" : "click to upload", onClick: () => c.uploadRef("global", field) },
|
||||
name ? [h("img", { src: c.viewUrl(name) })] : [label]),
|
||||
h("div", { class: "rbtns" }, [name ? h("button", { title: "remove", onClick: () => c.setRef("global", field, "") }, "✕") : null]),
|
||||
h("label", { class: "guidecolor" }, [
|
||||
h("span", { class: "hint" }, field === "global_ref" ? "Marker color · Ref 1" : "Marker color · Ref 2"),
|
||||
h("select", { value: field === "global_ref" ? (plan.global_color_guide_ref1 || "#00ff00") : (plan.global_color_guide_ref2 || "#9b30ff"),
|
||||
disabled: !name,
|
||||
title: "Default Colored Mark for this global reference",
|
||||
onChange: e => c.setPlan(field === "global_ref" ? "global_color_guide_ref1" : "global_color_guide_ref2", e.target.value) },
|
||||
GUIDE_COLORS.map(([v, text]) => h("option", { value: v }, text))),
|
||||
]),
|
||||
// Colored Mark: every picture has its mark color (a person marked in it becomes that picture)
|
||||
const marker = k => h("label", { class: "guidecolor" }, [
|
||||
h("span", { class: "hint" }, "Mark color"),
|
||||
h("select", { value: pictureColor(plan, k + 1), title: `Colored Mark color of <Picture ${k + 1}>`,
|
||||
onChange: e => c.setPictureColor(k + 1, e.target.value) },
|
||||
MARK_PALETTE.map(([v, name]) => h("option", { value: v }, name))),
|
||||
]);
|
||||
return section("Global reference & prompt", [
|
||||
h("div", { class: "row", style: "align-items:flex-start;gap:10px" }, [
|
||||
h("div", { class: "refbox" }, [slot(plan.global_ref, "⬆ global\nreference", "global_ref"), slot(plan.global_ref2, "⬆ global\nref 2", "global_ref2")]),
|
||||
h("div", { style: "flex:1;min-width:200px" }, [
|
||||
h("textarea", { style: "min-height:78px", placeholder: "Global prompt (a connected `prompt` input overrides it)",
|
||||
value: plan.global_prompt, onChange: e => c.setPlan("global_prompt", e.target.value) }),
|
||||
]),
|
||||
refBox(c, "global", plan.global_refs, [], marker),
|
||||
h("div", { class: "pbox", style: "margin-top:6px" }, [
|
||||
h("textarea", { style: "min-height:78px", placeholder: plan.auto_prompt?.enabled
|
||||
? "Request for every shot (🤖 Auto prompt writes the H3 prompt from it), e.g. 'replace the bottle with the red can from Picture 2'"
|
||||
: "Global prompt (a connected `prompt` input overrides it)",
|
||||
value: plan.global_prompt, onChange: e => c.setPlan("global_prompt", e.target.value) }),
|
||||
promptChips(c, plan.global_refs.filter(Boolean), v => c.setPlan("global_prompt", v)),
|
||||
]),
|
||||
h("div", { class: "hint", style: "margin-top:4px" }, [
|
||||
"Used by shots without their own. Connected ref_image / ref_image_2 / prompt inputs on the node override these. Placeholders: ",
|
||||
`Used by shots without their own (up to ${MAX_REFS} pictures, <Picture 1> first; click + or drop image files on the box). A connected prompt input on the node overrides the global prompt. Placeholders: `,
|
||||
h("b", "{target}"), " who is replaced (per shot) · ", h("b", "{details}"), " description of the shot's references · ",
|
||||
h("b", "{details@Picture n}"), " description of picture n alone · ",
|
||||
h("b", "{shot}"), " the VLM's description of the shot · ", h("b", "{setting}"), " the setting picture (H3 Conditioning)."]),
|
||||
h("div", { class: "hint", style: "margin-top:4px" }, "These are the default marker colors for the global references. Turn on Colored Mark on a shot to apply them; shot-level colors can override them."),
|
||||
h("div", { class: "hint", style: "margin-top:4px" }, "Mark color: the Colored Mark color of each picture. A person marked for <Picture n> in a shot is drawn in that color, and the prompt says which picture the color becomes. Change it when it is too close to a color in the video."),
|
||||
], { sub: "defaults for every shot" });
|
||||
}
|
||||
|
||||
function vlmSection(c) {
|
||||
const { plan, vlmSug, busy, refSets } = c;
|
||||
const { plan, vlmSug, busy, pictures } = c;
|
||||
const VC = plan.vlm_cfg || {};
|
||||
const opt = (k, label, title) => check(VC[k], v => c.setVlmCfg(k, v), label, title);
|
||||
const sugN = Object.keys(vlmSug.value).length;
|
||||
@@ -65,42 +60,79 @@ function vlmSection(c) {
|
||||
right: h("span", { class: "row" }, [VC.enabled ? pill("on", "ok") : pill("off"), sugN ? pill(`${sugN} suggestions`) : null]) }),
|
||||
section("Describe references ({details})", [
|
||||
h("div", { class: "row", style: "gap:8px" }, [
|
||||
select(VC.describe_preset, ["short", "full body", "head / face", "face attributes", "outfit", "custom"], v => c.setVlmCfg("describe_preset", v), { style: "width:auto" }),
|
||||
h("button", { class: "pri", disabled: !!busy.value || !refSets.value.length, onClick: c.describeRefs }, "📝 Describe refs"),
|
||||
h("button", { disabled: !!busy.value || !refSets.value.length,
|
||||
title: "Try another description: a new seed (and some temperature if it is 0), then describe again",
|
||||
onClick: () => { if (!(+VC.temperature > 0)) c.setVlmCfg("temperature", 0.7); c.setVlmCfg("seed", (+VC.seed || 0) + 1); c.describeRefs(); } }, "🎲"),
|
||||
fld("Describe", select(VC.describe_preset, PRESETS, v => c.setVlmCfg("describe_preset", v), { style: "width:auto" }),
|
||||
"what every picture gets, unless it has its own below"),
|
||||
h("button", { class: "pri", disabled: !!busy.value || !pictures.value.length, onClick: () => c.describePictures() }, "📝 Describe refs"),
|
||||
h("button", { disabled: !!busy.value || !pictures.value.length,
|
||||
title: "Try other descriptions: a new seed (and some temperature if it is 0), then describe again",
|
||||
onClick: () => { if (!(+VC.temperature > 0)) c.setVlmCfg("temperature", 0.7); c.setVlmCfg("seed", (+VC.seed || 0) + 1); c.describePictures(); } }, "🎲"),
|
||||
fld("Temperature", h("input", { type: "number", min: 0, max: 1.5, step: 0.1, value: VC.temperature ?? 0, style: "width:70px",
|
||||
title: "0 = always the same answer (greedy); 0.5-1 = varied answers, picked by the seed",
|
||||
onChange: e => c.setVlmCfg("temperature", Math.max(0, parseFloat(e.target.value) || 0)) })),
|
||||
fld("Seed", h("input", { type: "number", min: 0, step: 1, value: VC.seed ?? 0, style: "width:80px",
|
||||
title: "Used when the temperature is above 0: another seed, another description",
|
||||
onChange: e => c.setVlmCfg("seed", Math.max(0, parseInt(e.target.value) || 0)) })),
|
||||
hint(`${refSets.value.length} reference set(s) · editable below`),
|
||||
hint(`${pictures.value.length} picture(s) · editable below`),
|
||||
check(VC.fit_prompt !== false, v => c.setVlmCfg("fit_prompt", v), "fit the prompt",
|
||||
"The VLM gets the global prompt's sentence that holds {details} (and the shot prompt's {target} sentence for 🧑 Describe target) with a blank in its place, and writes the description to fit it: the description adapts to the prompt, not the other way round."),
|
||||
"The VLM gets the global prompt's sentence about that picture ({details}, {details2} or {details@Picture n}; and the shot prompt's {target} sentence for 🧑 Describe target) with a blank in its place, and writes the description to fit it: the description adapts to the prompt, not the other way round."),
|
||||
]),
|
||||
VC.describe_preset === "custom" ? h("textarea", { style: "margin-top:6px;min-height:52px", value: VC.describe_custom || "",
|
||||
placeholder: "Your instruction for the VLM, e.g. 'Describe the character's costume and props piece by piece…'",
|
||||
onChange: e => c.setVlmCfg("describe_custom", e.target.value) }) : null,
|
||||
refSets.value.length ? h("div", { class: "dlist" }, refSets.value.map(x => {
|
||||
const saved = plan.ref_details?.[x.key] || "";
|
||||
const input = (field, placeholder) => h("textarea", { style: "min-height:44px", value:
|
||||
typeof saved === "object" ? (saved[field] || "") : field === "details" ? saved : "",
|
||||
placeholder, onChange: e => c.setDetail(x.key, { ...(typeof saved === "object" ? saved : { details: saved }), [field]: e.target.value }) });
|
||||
pictures.value.length ? h("div", { class: "dlist" }, pictures.value.map(x => {
|
||||
const own = plan.picture_cfg?.[x.name] || {};
|
||||
return h("div", { class: "drow" }, [
|
||||
h("div", { class: "dthumbs" }, [x.a ? h("img", { src: c.viewUrl(x.a) }) : null, x.b ? h("img", { src: c.viewUrl(x.b) }) : null]),
|
||||
h("div", { class: "dthumbs" }, [h("img", { src: c.viewUrl(x.name) })]),
|
||||
h("div", { style: "flex:1" }, [
|
||||
hint(x.where.length > 4 ? `${x.where.slice(0, 4).join(", ")} +${x.where.length - 4}` : x.where.join(", ")),
|
||||
x.b ? h("div", { class: "grid" }, [input("details", "Ref 1 description ({details})"), input("details2", "Ref 2 description ({details2})")])
|
||||
: input("details", "no description yet: Describe refs, write your own, or leave it to the VLM at run time"),
|
||||
h("div", { class: "row", style: "gap:6px;margin-bottom:3px" }, [
|
||||
hint(`Picture n in ${x.where.length > 4 ? `${x.where.slice(0, 4).join(", ")} +${x.where.length - 4}` : x.where.join(", ")}`),
|
||||
h("span", { class: "grow" }),
|
||||
select(own.preset || "", [["", `default (${VC.describe_preset || "short"})`], ...PRESETS],
|
||||
v => c.setPictureCfg(x.name, "preset", v), { style: "width:auto", title: "What to describe in this picture" }),
|
||||
h("button", { class: "sm", disabled: !!busy.value, title: "Describe only this picture again",
|
||||
onClick: () => c.describePictures([x.name]) }, "🔄"),
|
||||
]),
|
||||
own.preset === "custom" ? h("textarea", { style: "min-height:36px;margin-bottom:3px", value: own.custom || "",
|
||||
placeholder: "Instruction for this picture, e.g. 'Describe only the outfit, piece by piece'",
|
||||
onChange: e => c.setPictureCfg(x.name, "custom", e.target.value) }) : null,
|
||||
h("textarea", { style: "min-height:44px", value: plan.picture_details?.[x.name] || "",
|
||||
placeholder: "no description yet: 📝 Describe refs, 🔄, write your own, or leave it to the VLM at run time",
|
||||
onChange: e => c.setPictureDetail(x.name, e.target.value) }),
|
||||
]),
|
||||
]);
|
||||
})) : hint("Add a reference (global, per shot or per shot) first."),
|
||||
], { sub: "write {details} in any prompt: each shot gets the description of its own references" }),
|
||||
})) : hint("Add a reference (global or per shot) first."),
|
||||
], { sub: "{details} = picture 1, {details2} = picture 2, {details@Picture n} = picture n: each shot gets the descriptions of its own pictures" }),
|
||||
];
|
||||
}
|
||||
|
||||
// 🤖 Auto prompt: the VLM writes each shot's prompt in H3's full-reference format from key frames of the shot, its
|
||||
// pictures and their descriptions, the request in the prompt box and the shot's marks and mode
|
||||
function autoSection(c) {
|
||||
const { plan, busy, segs } = c;
|
||||
const A = plan.auto_prompt || {};
|
||||
const S = segs.value, missing = S.filter(s => !c.writtenOf(s)?.text).length;
|
||||
const stale = S.filter(s => c.writtenOf(s)?.text && c.writtenOf(s).for !== c.autoFor(s)).length;
|
||||
return section("🤖 Auto prompt (H3)", [
|
||||
h("div", { class: "row", style: "gap:12px;align-items:center" }, [
|
||||
check(A.enabled, v => c.setAuto("enabled", v), "Write the prompts with the VLM",
|
||||
"On: the prompt boxes (global and per shot) hold your request ('change the bottle into a red can', 'Picture 1 dances in the rain'); the VLM writes each shot's MiniMax H3 prompt from key frames of the shot, the pictures and their descriptions, the request and the shot's Colored Marks, selection and mode, in H3's official full-reference format."),
|
||||
fld("Style", select(A.style || "h3", [["h3", "H3 official (six sections)"], ["captions", "OpenJutsu captions (short)"]],
|
||||
v => c.setAuto("style", v), { style: "width:auto" }), "captions: 'Replace the ... in the video with ... from <Picture 1>. Keep ...', the training captions' style (OpenJutsu); a template on the prompt input wins over both"),
|
||||
fld("Key frames", select(String(A.frames || 0), [["0", "auto (by length)"], ...[3, 4, 5, 6, 8, 10, 12].map(n => [String(n), String(n)])],
|
||||
v => c.setAuto("frames", parseInt(v) || 0), { style: "width:auto" }), "auto: about 1.5 per second, 3 to 8"),
|
||||
fld("Source video in the prompt", select(A.source || "guide", [["guide", "guide video (aligned guide)"], ["video", "<Video 1> (native reference video)"]],
|
||||
v => c.setAuto("source", v), { style: "width:auto" }), "match the conditioning node's guide_mode"),
|
||||
h("button", { class: "pri", disabled: !!busy.value || !S.length, onClick: () => c.writePrompts() }, "🤖 Write all prompts"),
|
||||
h("button", { disabled: !!busy.value || !missing, onClick: () => c.writePrompts(S.map((s, i) => (c.writtenOf(s)?.text ? -1 : i)).filter(i => i >= 0)) },
|
||||
`only the missing (${missing})`),
|
||||
]),
|
||||
A.enabled && A.template ? h("div", { class: "hint", style: "display:block;margin-top:4px", title: A.template },
|
||||
`Template (the node's prompt input, last run): ${A.template.slice(0, 110)}${A.template.length > 110 ? "…" : ""} · the prompt boxes are the request applied to it`) : null,
|
||||
hint(A.enabled ? `${S.length - missing}/${S.length} shots have a written prompt${stale ? ` · ${stale} older than their request, pictures or marks` : ""} · shots without one are written when the workflow runs (with the VLM on the planner) · edit them in each shot (Shots tab)`
|
||||
: "Off: the prompt boxes are the prompts.", "display:block;margin-top:4px"),
|
||||
], { sub: "request + video + pictures -> MiniMax H3 prompt", right: A.enabled ? pill("on", "ok") : pill("off") });
|
||||
}
|
||||
|
||||
export function promptsTab(c) {
|
||||
return h("div", { class: "card" }, [globalSection(c), ...vlmSection(c)]);
|
||||
return h("div", { class: "card" }, [globalSection(c), autoSection(c), ...vlmSection(c)]);
|
||||
}
|
||||
|
||||
@@ -30,6 +30,10 @@ function loopSection(c) {
|
||||
["free", "Free memory: unload models (low VRAM)"]], v => c.setPlan("between_shots", v))) : null,
|
||||
fld("Skipped shots in the output", select(plan.skip_fill, [["original", "Keep original video"], ["drop", "Remove them"]], v => c.setPlan("skip_fill", v))),
|
||||
]),
|
||||
h("div", { class: "row", style: "margin-top:6px" }, [
|
||||
check(plan.unload_before_join !== false, v => c.setPlan("unload_before_join", v), "Unload models before the join",
|
||||
"Once every shot is rendered, unload the models so the join (and the nodes after it) get the VRAM and RAM. Turn it off if a model runs again after BFS Shot Join (e.g. an upscale pass on the joined video): it would load again."),
|
||||
]),
|
||||
plan.run !== "auto" ? h("div", { style: "margin-top:8px" }, [
|
||||
h("div", { class: "row", style: "margin-bottom:4px" }, [h("b", plan.run === "queue" ? "Queue loop" : "Shot by shot"), h("span", { class: "grow" }),
|
||||
hint(`${prog.done}/${prog.count || active.value.length} shots done`)]),
|
||||
@@ -104,7 +108,7 @@ function cacheSection(c) {
|
||||
hint(`✓ ${cache.combined.shots}/${cache.combined.of} shots → `),
|
||||
h("a", { href: cache.combined.url, target: "_blank", style: "margin-left:4px" }, cache.combined.filename)]) : null,
|
||||
hint("Each shot remembers the settings it was made with: the next Queue runs only the shots whose settings changed "
|
||||
+ "(and the ones that continue from them). Model, LoRA, seed or sampler changes are not seen by the planner: use ↻ or Redo all. "
|
||||
+ "(and the ones that continue from them). Changing what renders them (model, LoRA, seed, sampler, conditioning) runs every shot again; the join's own settings do not. "
|
||||
+ (cache.folder ? `Folder: ${cache.folder}` : ""), "display:block;margin-top:6px"),
|
||||
], { sub: ok.length ? `${done}/${ok.length} stored` : "nothing stored yet" });
|
||||
}
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
// 🎬 Shots tab: one card per shot, then the selected shot's editor (preview, reference & prompt, who is replaced,
|
||||
// continuity, copy to other shots).
|
||||
import { h, hue, fmtT, segKey, hasMask, maskSource, pill, hint, check, section } from "./common.js";
|
||||
import { h, hue, fmtT, segKey, hasMask, maskSource, pill, hint, check, section, refBox, promptChips, pictureColor, markName,
|
||||
MAX_REFS } from "./common.js";
|
||||
import { h as rawH } from "../vendor/vue.esm-browser.prod.mjs";
|
||||
|
||||
const GUIDE_COLORS = [["#00ff00", "Green"], ["#9b30ff", "Purple"], ["#ff2020", "Red"], ["#168cff", "Blue"],
|
||||
["#ffe600", "Yellow"], ["#ff00d4", "Magenta"], ["#00e5c0", "Cyan"]];
|
||||
|
||||
const bdg = (text, kind = "", title = "") => h("span", { class: ["bdg", kind], title }, text);
|
||||
|
||||
function shotCard(c, s, i) {
|
||||
@@ -23,7 +21,8 @@ function shotCard(c, s, i) {
|
||||
track: bdg("🎯 track", "", "its part of the whole-video SAM 3 track"),
|
||||
sam: bdg("🎯 " + (s.mask.text ? s.mask.text.slice(0, 14) : `${(s.mask.points || []).length} pts`), "", s.mask.text || `${(s.mask.points || []).length} points`) }[maskSource(s)] || null,
|
||||
s.crop || s.inpaint || s.paste ? bdg({ paste: "🧩 frame+paste", mask: "🎭 mask only", crop: "✂ crop", cropmask: "✂🎭 crop+mask" }[c.modeOf(s)] + (s.inpaint && s.strength < 1 ? ` ${Number(s.strength).toFixed(2)}` : ""), "on", c.modeName(s)) : null,
|
||||
s.color_guide ? bdg("🎨 Colored Mark", "on", `${s.color_guide_color || "green"} · opacity ${s.color_guide_opacity ?? 0.35}`) : null,
|
||||
s.color_guide ? bdg(`🎨 ${1 + s.regions.length}`, "on", [s.color_guide_picture, ...s.regions.map(r => r.picture)]
|
||||
.map(k => `Picture ${k} · ${markName(pictureColor(c.plan, k))}`).join("\n") + ` · opacity ${s.color_guide_opacity ?? 0.35}`) : null,
|
||||
s.target ? bdg("🧑 target", "on", s.target) : null,
|
||||
i > 0 && s.chain !== "off" ? bdg("⛓", "on", s.chain === "history" ? `continues the previous shot's last ${s.chainHistory} frames (history)` : s.chain === "head" ? `starts with the previous shot's last ${Math.min(s.chainHistory, 39)} frames, kept as they are (head)` : `continues from the previous shot's ${s.chainFrame} frame as ${s.chain}`) : null,
|
||||
{ ok: bdg("💾", "on", "stored and up to date: the next Queue does not generate it again (Run tab → Stored shots to redo it)"),
|
||||
@@ -38,11 +37,10 @@ function shotCard(c, s, i) {
|
||||
why ? h("div", { class: "t", style: "color:#ffc46b" }, why) : null,
|
||||
h("div", { class: "t" }, `${fmtT(s.start, fps)} → ${fmtT(s.end, fps)} · ${s.len}f → ${s.gen}f`),
|
||||
h("div", { class: "thumbs" }, [
|
||||
...["ref", "ref2"].map(k => {
|
||||
const own = s[k], name = own || plan["global_" + k];
|
||||
return name ? h("img", { class: "rt", src: c.viewUrl(name), title: own ? name : `${name} (global)`,
|
||||
style: own ? "" : "opacity:.45" }) : h("div", { class: "rt ph2" }, k);
|
||||
}),
|
||||
...(s.names.length ? s.names.slice(0, 3).map((name, k) => h("img", { class: "rt", src: c.viewUrl(name),
|
||||
title: s.refs.includes(name) ? name : `${name} (global)`, style: s.refs.includes(name) ? "" : "opacity:.45" }))
|
||||
: [h("div", { class: "rt ph2" }, "ref")]),
|
||||
s.names.length > 3 ? h("div", { class: "rt ph2", title: s.names.slice(3).join("\n") }, `+${s.names.length - 3}`) : null,
|
||||
h("img", { class: "rt", style: "width:56px", src: showMasks.value && mp?.frames?.length
|
||||
? mp.frames[Math.floor(mp.frames.length / 2)].src : c.thumbFor(s.start + Math.floor(s.len / 2)) }),
|
||||
]),
|
||||
@@ -76,19 +74,9 @@ function player(c, cur) {
|
||||
]);
|
||||
}
|
||||
|
||||
function refSlot(c, name, label, target, field) {
|
||||
return h("div", { class: "rslot" }, [
|
||||
h("div", { class: "refslot", title: name ? name + " (click to replace)" : "click to upload",
|
||||
onClick: () => c.uploadRef(target, field) },
|
||||
name ? [h("img", { src: c.viewUrl(name) })] : [label]),
|
||||
h("div", { class: "rbtns" }, [name ? h("button", { title: "remove (use the global reference)", onClick: () => c.setRef(target, field, "") }, "✕") : null]),
|
||||
]);
|
||||
}
|
||||
|
||||
function editor(c, cur) {
|
||||
const { plan, sel, segs, maskPrev, targetCrop, vlmSug, busy, usedRefs, copyOpts, COPY_FIELDS } = c;
|
||||
const i = sel.value, S = segs.value, k = segKey(cur), mp = maskPrev.value[k], g = vlmSug.value[k];
|
||||
const hasRef2 = !!(cur.ref2 || plan.global_ref2);
|
||||
const issues = c.checksFor(i);
|
||||
|
||||
const head = h("div", { class: "edhd" }, [
|
||||
@@ -110,53 +98,77 @@ function editor(c, cur) {
|
||||
h("div", { class: "ck" }, [h("span", { class: "lvl" }, x.lvl === "warn" ? "⚠" : "ℹ"), h("span", x.text)]))) : null;
|
||||
|
||||
const refPrompt = section("Reference & prompt", [
|
||||
h("div", { class: "row", style: "align-items:flex-start;gap:10px" }, [
|
||||
h("div", { class: "refbox" }, [
|
||||
refSlot(c, cur.ref, "⬆ reference", i, "ref"),
|
||||
refSlot(c, cur.ref2, "⬆ ref 2", i, "ref2"),
|
||||
]),
|
||||
h("div", { style: "flex:1;min-width:200px" }, [
|
||||
h("textarea", { placeholder: "Prompt for this shot (empty = global prompt). Placeholders: {target} {details} {shot} {setting}",
|
||||
value: cur.prompt, onChange: e => c.setMeta(i, "prompt", e.target.value) }),
|
||||
!cur.prompt && plan.global_prompt ? rawH("span", { class: "hint" }, `↳ uses the global prompt: ${plan.global_prompt.slice(0, 120)}${plan.global_prompt.length > 120 ? "…" : ""}`) : null,
|
||||
]),
|
||||
refBox(c, i, cur.refs, plan.global_refs),
|
||||
h("div", { class: "pbox", style: "margin-top:6px" }, [
|
||||
h("textarea", { placeholder: plan.auto_prompt?.enabled
|
||||
? "Request for this shot (empty = the global request); 🤖 Auto prompt writes the H3 prompt below from it"
|
||||
: "Prompt for this shot (empty = global prompt). Placeholders: {target} {details} {details@Picture n} {shot} {setting}",
|
||||
value: cur.prompt, onChange: e => c.setMeta(i, "prompt", e.target.value) }),
|
||||
promptChips(c, cur.names, v => c.setMeta(i, "prompt", v)),
|
||||
!cur.prompt && plan.global_prompt ? rawH("span", { class: "hint" }, `↳ uses the global prompt: ${plan.global_prompt.slice(0, 120)}${plan.global_prompt.length > 120 ? "…" : ""}`) : null,
|
||||
]),
|
||||
usedRefs.value.length ? h("div", { class: "recent" }, [hint("recent (click = ref, shift+click = ref 2):"),
|
||||
...usedRefs.value.map(n => h("img", { src: c.viewUrl(n), title: n, onClick: e => c.setRef(i, e.shiftKey ? "ref2" : "ref", n) })),
|
||||
plan.auto_prompt?.enabled ? (() => {
|
||||
const w = c.writtenOf(cur), old = w?.text && w.for !== c.autoFor(cur);
|
||||
return h("div", { class: "note", style: "margin-top:6px;padding:6px 8px" }, [
|
||||
h("div", { class: "row", style: "gap:6px;align-items:center;margin-bottom:4px" }, [
|
||||
h("b", "🤖 Written prompt"), old ? pill("older than the request / pictures / marks", "warn") : null, h("span", { class: "grow" }),
|
||||
h("button", { class: w?.text ? "" : "pri", disabled: !!busy.value, onClick: () => c.writePrompts([i]) }, w?.text ? "🤖 Write again" : "🤖 Write"),
|
||||
w?.text ? h("button", { class: "dng", title: "forget it: the next run writes a new one", onClick: () => c.setWritten(cur, "") }, "✕") : null,
|
||||
]),
|
||||
h("textarea", { style: "min-height:140px;font-size:11px", value: w?.text || "",
|
||||
placeholder: "not written yet: 🤖 Write, or the planner's VLM writes it when the workflow runs",
|
||||
onChange: e => c.setWritten(cur, e.target.value) }),
|
||||
]);
|
||||
})() : null,
|
||||
usedRefs.value.length ? h("div", { class: "recent" }, [hint("recent (click = picture 1, shift+click = add, or drag onto a slot):"),
|
||||
...usedRefs.value.map(n => h("img", { src: c.viewUrl(n), title: n, draggable: true,
|
||||
onDragstart: e => e.dataTransfer.setData("text/bfs-ref", n),
|
||||
onClick: e => (e.shiftKey ? c.putRefs(i, cur.refs.length, [n]) : c.setRef(i, 0, n)) })),
|
||||
h("button", { class: "sm", title: "Forget the recent pictures (the ones this plan still uses stay)", onClick: () => c.clearRecent() }, "✕ clear"),
|
||||
h("button", { class: "sm dng", title: "Remove EVERY reference from this plan (global and all shots) and the recent list", onClick: () => c.clearAllRefs() }, "✕ clear all")]) : null,
|
||||
h("div", { class: "row", style: "margin-top:6px" }, [
|
||||
h("button", { class: "dng", title: "clear this shot's references and prompt: it uses the global ones",
|
||||
onClick: () => { ["ref", "ref2", "prompt"].forEach(f => { c.meta(i)[f] = ""; }); c.save(); } }, "Use global"),
|
||||
onClick: () => { Object.assign(c.meta(i), { refs: [], prompt: "" }); c.save(); } }, "Use global"),
|
||||
hint("empty slots use the global references (Prompts & refs tab)"),
|
||||
]),
|
||||
], { sub: "who the shot becomes" });
|
||||
|
||||
const mode = (on, title, text, onClick) => h("div", { class: ["mode", on && "on"], onClick }, [h("b", title), h("span", text)]);
|
||||
// which picture a marked person becomes: "Picture 3 · red" with its thumbnail (the color is the picture's)
|
||||
const picturePick = (k, onPick) => {
|
||||
const n = Math.max(cur.names.length, k), color = pictureColor(plan, k);
|
||||
return h("span", { class: "row", style: "gap:4px;align-items:center" }, [
|
||||
cur.names[k - 1] ? h("img", { class: "mthumb", src: c.viewUrl(cur.names[k - 1]), style: `--mark:${color}`, title: cur.names[k - 1] })
|
||||
: h("span", { class: "mthumb", style: `--mark:${color};display:inline-block;background:${color}` }),
|
||||
h("select", { value: String(k), style: "width:auto", title: "The picture this person becomes; the mark is drawn in its color",
|
||||
onChange: e => onPick(+e.target.value) },
|
||||
Array.from({ length: Math.min(MAX_REFS, n) }, (_, q) => h("option", { value: String(q + 1) },
|
||||
`Picture ${q + 1} · ${markName(pictureColor(plan, q + 1))}${cur.names[q] ? "" : " (no picture)"}`))),
|
||||
]);
|
||||
};
|
||||
const who = section("Select region", [
|
||||
h("div", { class: "row", style: "margin-top:4px;align-items:center;gap:8px" }, [
|
||||
check(!!cur.color_guide, v => c.setMeta(i, "color_guide", v), "🎨 Colored Mark",
|
||||
"Color the selected region in the source video as a translucent visual marker. It does not create or enable an inpainting mask."),
|
||||
h("label", { class: "row", style: "gap:6px;align-items:center" }, [h("span", { class: "hint" }, "Region color · Ref 1"),
|
||||
h("select", { value: cur.color_guide_color || plan.global_color_guide_ref1 || "#00ff00", style: "width:auto", onChange: e => {
|
||||
c.setMeta(i, "color_guide_color", e.target.value);
|
||||
if (maskPrev.value[segKey(cur)]) c.previewMask(i);
|
||||
} },
|
||||
GUIDE_COLORS.map(([v, label]) => h("option", { value: v, disabled: v === (cur.color_guide_color2 || plan.global_color_guide_ref2 || "#9b30ff") }, label)))]),
|
||||
"Color each selected person in the source video with the color of the picture they become, as a translucent marker; the prompt gets 'the person marked in <color> becomes the character from <Picture n>'. It does not create or enable an inpainting mask."),
|
||||
cur.color_guide ? h("span", { class: "hint" }, "this selection becomes") : null,
|
||||
cur.color_guide ? picturePick(cur.color_guide_picture, k => { c.setMarkPicture(i, 1, k); if (maskPrev.value[segKey(cur)]) c.previewMask(i); }) : null,
|
||||
]),
|
||||
cur.color_guide ? h("details", { class: "note", style: "margin:4px 0;padding:6px 8px", open: !!cur.color_guide_text2 || hasRef2 }, [
|
||||
h("summary", { style: "cursor:pointer" }, cur.color_guide_text2 || (cur.color_guide_points2 || []).length ? "Mark a second region · Ref 2" : "+ Mark a second region (optional)"),
|
||||
h("div", { class: "row", style: "margin-top:8px;align-items:center;gap:8px" }, [
|
||||
h("span", { class: "hint", style: "white-space:nowrap" }, "Describe second region"),
|
||||
h("input", { type: "text", value: cur.color_guide_text2 || "", style: "flex:1;min-width:160px", placeholder: "e.g. the person on the left", onChange: e => c.setMeta(i, "color_guide_text2", e.target.value) }),
|
||||
h("button", { disabled: !!busy.value || !cur.color_guide_text2, onClick: () => c.previewMask(i, { text: cur.color_guide_text2, max_objects: 1 }) }, "👁 Preview"),
|
||||
h("button", { onClick: () => c.openPoints(i, 2) }, (cur.color_guide_points2 || []).length ? `🎯 Points 2 (${cur.color_guide_points2.length})` : "🎯 Mark with points"),
|
||||
h("select", { value: cur.color_guide_color2 || plan.global_color_guide_ref2 || "#9b30ff", onChange: e => {
|
||||
c.setMeta(i, "color_guide_color2", e.target.value);
|
||||
if ((cur.color_guide_points2 || []).length) c.previewMask(i, { points: cur.color_guide_points2, key: cur.color_guide_key2 || 0,
|
||||
text: cur.color_guide_text2 || "", preview_color: e.target.value });
|
||||
} },
|
||||
GUIDE_COLORS.map(([v, label]) => h("option", { value: v, disabled: v === (cur.color_guide_color || plan.global_color_guide_ref1 || "#00ff00") }, label))),
|
||||
cur.color_guide ? h("div", { class: "note", style: "margin:4px 0;padding:6px 8px" }, [
|
||||
...cur.regions.map((r, j) => h("div", { class: "mreg" }, [
|
||||
h("span", { class: "hint", style: "white-space:nowrap" }, `Person ${j + 2}`),
|
||||
picturePick(r.picture, k => c.setMarkRegion(i, j, "picture", k)),
|
||||
h("input", { type: "text", value: r.text || "", style: "flex:1;min-width:160px", placeholder: "who, in English: e.g. the man on the left",
|
||||
onChange: e => c.setMarkRegion(i, j, "text", e.target.value) }),
|
||||
h("button", { disabled: !!busy.value || !(r.text || (r.points || []).length),
|
||||
onClick: () => c.previewMask(i, { text: r.text || "", points: r.points || [], key: r.key || 0, max_objects: 1,
|
||||
preview_color: pictureColor(plan, r.picture) }) }, "👁 Preview"),
|
||||
h("button", { onClick: () => c.openPoints(i, j + 2) }, c.keyFramesOf(r).length ? `🎯 Points (${c.keyFramesOf(r).length} key frame${c.keyFramesOf(r).length > 1 ? "s" : ""})` : "🎯 Points…"),
|
||||
h("button", { class: "dng", title: "remove this person's mark", onClick: () => c.removeMarkRegion(i)(j) }, "✕"),
|
||||
])),
|
||||
h("div", { class: "row", style: "margin-top:6px;align-items:center;gap:8px" }, [
|
||||
h("button", { disabled: 1 + cur.regions.length >= MAX_REFS, onClick: () => c.addMarkRegion(i) }, "+ Mark another person"),
|
||||
hint(cur.names.length > 1 + cur.regions.length ? "each person: a picture (its color) and a text or points"
|
||||
: "add more pictures (Reference & prompt) to give each person their own"),
|
||||
]),
|
||||
]) : null,
|
||||
cur.color_guide ? h("details", { style: "margin:4px 0" }, [h("summary", { class: "hint", style: "cursor:pointer" }, `Opacity · ${Number(cur.color_guide_opacity ?? 0.35).toFixed(2)}`),
|
||||
@@ -168,7 +180,7 @@ function editor(c, cur) {
|
||||
onInput: e => c.setMask(i, "text", e.target.value),
|
||||
onKeydown: e => { if (e.key === "Enter") { e.preventDefault(); c.setMask(i, "text", e.target.value); if (e.target.value || (cur.mask.points || []).length) c.previewMask(i); } } }),
|
||||
h("button", { title: "Pick positive / negative points on a frame of this shot (more precise than text when several people look alike)",
|
||||
onClick: () => c.openPoints(i) }, (cur.mask.points || []).length ? `🎯 Points (${cur.mask.points.length})` : "🎯 Points…"),
|
||||
onClick: () => c.openPoints(i) }, c.keyFramesOf(cur.mask).length ? `🎯 Points (${c.keyFramesOf(cur.mask).length} key frame${c.keyFramesOf(cur.mask).length > 1 ? "s" : ""})` : "🎯 Points…"),
|
||||
h("button", { class: hasMask(cur) && !mp ? "pri" : "", disabled: !!busy.value || !hasMask(cur),
|
||||
title: (cur.mask.points || []).length ? "Segment this shot (the points win over the text)" : "Segment this shot with the text and show a few frames",
|
||||
onClick: () => c.previewMask(i) }, "👁 Preview"),
|
||||
@@ -265,7 +277,7 @@ function editor(c, cur) {
|
||||
const copy = section("Copy to other shots", [
|
||||
h("div", { class: "copy" }, Object.entries(COPY_FIELDS).map(([f, label]) =>
|
||||
check(copyOpts[f], v => { copyOpts[f] = v; }, label, {
|
||||
ref: "reference + ref 2", prompt: "this shot's prompt", mask_text: "the mask text (shots keep their own points)",
|
||||
ref: "every reference picture", prompt: "this shot's prompt", mask_text: "the mask text (shots keep their own points)",
|
||||
points: "the points, on each shot's frame at the same relative position (a subject that stays in place)",
|
||||
mask_video: "the shot's mask video (it covers only its own shot: usually you want a different one per shot)",
|
||||
crop: "the generation mode (Full frame / Frame + paste / Mask only / Crop / Crop + mask) and its mask strength (shots with a mask)", target: "the {target} description",
|
||||
|
||||
Reference in New Issue
Block a user