Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9dcb9e4d08 |
@@ -2,6 +2,20 @@
|
||||
|
||||
All notable changes to **ComfyUI-BFSNodes** are documented in this file.
|
||||
|
||||
## [1.120.0] - 2026-11-10
|
||||
|
||||
### Added
|
||||
- **BFS H3 Image Character Swap Conditioning**: native H3 Image conditioning and automatic still-image prompts. The guide is Picture 1; red, blue and green references are Pictures 2, 3 and 4. Supports normal maps, DWPose, overlays and outlines. Requires ComfyUI 0.45.0 or later.
|
||||
- **Add Guide for MiniMax H3 Image (BFS)**: experimental frozen guide reference aligned to the target image positions through a cloned ModelPatcher. Uses native H3 Image latents without an audio stream.
|
||||
- **Video Segment Picker**: optional `point_model` input and MolmoPoint-8B tracking method. Connect the loader from ComfyUI-Molmo2-H3.
|
||||
- Protected-object selections, a normal-map-inside-mask mode with feathered edges, translucent overlays and colored outlines.
|
||||
|
||||
### Fixed
|
||||
- Bound picker mask caches by memory size, keep cached masks as boolean tensors and preserve existing masks when filling empty stretches.
|
||||
- Track empty stretches and nearest-anchor windows instead of repeatedly recomputing the full clip.
|
||||
- Correct SAM inference tensor handling for point edits, rendering invalidation, normal-map lazy input requests and Apply fix messages.
|
||||
- Stabilize compare preview sizing; expose memory cleanup after comparison.
|
||||
|
||||
## [1.119.5] - 2026-11-09
|
||||
|
||||
### Changed
|
||||
|
||||
@@ -319,3 +319,13 @@ them with their spec), **BFS Keyframe Marks** / **BFS Track From Frame (SAM 3)**
|
||||
tracked person's mask at each keyframe), **BFS Workflow Loop** (run a part of the
|
||||
graph once per item, injecting the item and changing widgets per iteration),
|
||||
**BFS XY Sweep** (a grid of any two widgets' values) and **BFS Frame Loop Start/End**.
|
||||
|
||||
### MiniMax H3 Image character swap
|
||||
|
||||
Requires **ComfyUI 0.45.0+** and the H3 Image model, image VAE and matching text encoder.
|
||||
|
||||
Use **BFS H3 Image Character Swap Conditioning** with the Picker's single-image `hybrid` as `guide_image`. Connect character references in red, blue and green order. The generated prompt treats the guide as **Picture 1** and the characters as **Pictures 2–4**. Connect `positive` and `latent` to the sampler; the node also outputs the prompt for inspection. Normal map, DWPose, translucent overlay and outline guides are supported.
|
||||
|
||||
**Add Guide for MiniMax H3 Image (BFS)** is an optional experimental alignment step. Connect the model, positive conditioning, native image latent, image VAE and marked guide; use its returned MODEL and positive with the sampler. It aligns the guide's frozen reference positions to the target canvas. It does not use the video/audio Add Guide latent format. Structural checks passed; visual equivalence to a video-trained guide LoRA is not guaranteed.
|
||||
|
||||
For **MolmoPoint-8B** in Video Segment Picker, connect `point_model` from the loader in [ComfyUI-Molmo2-H3](https://github.com/alisson-anjos/ComfyUI-Molmo2-H3), select `MolmoPoint-8B` in `propagation_method`, and use **Apply fix** with the desired repair scope. SAM 3 still produces the masks.
|
||||
|
||||
@@ -242,3 +242,10 @@ for _mod in ("bfs_keyframe_marks", "bfs_video_segment_picker", "bfs_h3_swap_cond
|
||||
WEB_DIRECTORY = "./web/js"
|
||||
|
||||
__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"]
|
||||
|
||||
try:
|
||||
from .bfs_h3_image_swap_conditioning import BFSH3ImageCharacterSwapConditioning
|
||||
NODE_CLASS_MAPPINGS["BFSH3ImageCharacterSwapConditioning"] = BFSH3ImageCharacterSwapConditioning
|
||||
NODE_DISPLAY_NAME_MAPPINGS["BFSH3ImageCharacterSwapConditioning"] = "BFS H3 Image Character Swap Conditioning"
|
||||
except ImportError as _e:
|
||||
print(f"[BFSNodes] H3 Image conditioning requires updated ComfyUI: {_e}")
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Aligned image guide for the native MiniMax H3 Image reference stream."""
|
||||
import copy
|
||||
|
||||
import comfy.utils
|
||||
from comfy_api.latest import io
|
||||
|
||||
MARKER = 'bfs_h3_image_guide'
|
||||
|
||||
|
||||
def align_guide_layout(layout, refs):
|
||||
result = copy.copy(layout)
|
||||
result.position_ids = layout.position_ids.clone()
|
||||
target = next((a, b) for a, b, kind in layout.segments if kind == 'video')
|
||||
image_refs = iter(ref for ref in refs if ref['kind'] == 'image')
|
||||
for a, b, kind in layout.segments:
|
||||
if kind != 'ref_img':
|
||||
continue
|
||||
ref = next(image_refs)
|
||||
if ref.get(MARKER):
|
||||
ta, tb = target
|
||||
if b - a != tb - ta:
|
||||
raise ValueError('H3 Image guide must match the target canvas')
|
||||
result.position_ids[a:b] = layout.position_ids[ta:tb]
|
||||
return result
|
||||
|
||||
|
||||
def patch_image_guide(model):
|
||||
patched = model.clone()
|
||||
if patched.model_options.get(MARKER):
|
||||
return patched
|
||||
previous = patched.model_options.get('model_function_wrapper')
|
||||
|
||||
def wrapper(model_function, args):
|
||||
conds = args['c']
|
||||
payload = conds.get('minimax_payload', {})
|
||||
refs = payload.get('refs', [])
|
||||
if any(ref.get(MARKER) for ref in refs):
|
||||
layout = align_guide_layout(payload['layout'], refs)
|
||||
args = dict(args, c=dict(conds, minimax_payload=dict(payload, layout=layout)))
|
||||
if previous is not None:
|
||||
return previous(model_function, args)
|
||||
return model_function(args['input'], args['timestep'], **args['c'])
|
||||
|
||||
patched.set_model_unet_function_wrapper(wrapper)
|
||||
patched.model_options[MARKER] = True
|
||||
return patched
|
||||
|
||||
|
||||
class BFSH3ImageAddGuide(io.ComfyNode):
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
return io.Schema(node_id='BFSH3ImageAddGuide', display_name='Add Guide for MiniMax H3 Image (BFS)',
|
||||
category='BFS/MiniMax H3',
|
||||
description='Experimental aligned guide for H3 Image. Uses a frozen image reference with target-aligned positions. Connect the returned MODEL and positive to the sampler; use the native H3 Image latent.',
|
||||
inputs=[io.Model.Input('model'), io.Conditioning.Input('positive'), io.Vae.Input('vae'),
|
||||
io.Image.Input('guide'), io.Latent.Input('latent')],
|
||||
outputs=[io.Model.Output(), io.Conditioning.Output(display_name='positive')])
|
||||
|
||||
@classmethod
|
||||
def execute(cls, model, positive, vae, guide, latent):
|
||||
samples = latent['samples']
|
||||
if samples.is_nested or samples.ndim != 5 or samples.shape[1:3] != (24, 1):
|
||||
raise ValueError('Use the LATENT from MiniMax H3 Image Conditioning or Empty MiniMax H3 Image Latent')
|
||||
if model.model.diffusion_model.__class__.__name__ != 'MiniMaxH3ImageModel':
|
||||
raise ValueError('This guide requires the native MiniMax H3 Image model')
|
||||
h, w = samples.shape[-2:]
|
||||
pixels = comfy.utils.common_upscale(guide[:1, ..., :3].movedim(-1, 1), w * 16, h * 16, 'bicubic', 'center').movedim(1, -1)
|
||||
z = vae.encode(pixels)
|
||||
if z.ndim != 5 or tuple(z.shape[1:]) != (24, 1, h, w):
|
||||
raise ValueError('Guide VAE must produce native H3 Image latents matching the target')
|
||||
guide_ref = {'kind': 'image', 'latent_h': h, 'latent_w': w, 'latent': z, MARKER: True}
|
||||
out = []
|
||||
for cond, options in positive:
|
||||
refs = [r for r in options.get('minimax_refs', []) if not r.get(MARKER)]
|
||||
updated = dict(options, minimax_refs=refs + [guide_ref])
|
||||
out.append([cond, updated])
|
||||
return io.NodeOutput(patch_image_guide(model), out)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {'BFSH3ImageAddGuide': BFSH3ImageAddGuide}
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {'BFSH3ImageAddGuide': 'Add Guide for MiniMax H3 Image (BFS)'}
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Native MiniMax H3 Image conditioning with color-marked character references."""
|
||||
from comfy_api.latest import io
|
||||
from comfy_extras.nodes_minimax_h3 import MiniMaxH3ImageConditioning
|
||||
|
||||
COLORS = ('red', 'blue', 'green')
|
||||
MODES = ('normal map guide', 'DWPose guide', 'translucent overlay guide', 'outline guide')
|
||||
|
||||
|
||||
def image_swap_prompt(count, mode):
|
||||
if mode not in MODES:
|
||||
raise ValueError(f'Unknown guide mode: {mode}')
|
||||
if not 1 <= count <= len(COLORS):
|
||||
raise ValueError('Connect one to three character reference images')
|
||||
marks = {
|
||||
'normal map guide': ('normal map', 'each subject to replace is marked with its surface-normal map colorized in one solid color (its 3D surface, without its real colors)'),
|
||||
'DWPose guide': ('pose skeleton', 'each subject to replace is shown as its pose skeleton (body, hands and face) drawn in one color'),
|
||||
'translucent overlay guide': ('translucent overlay', 'each subject to replace is identified by a translucent color overlay; the appearance beneath the tint is the source appearance to replace'),
|
||||
'outline guide': ('outline', 'each subject to replace is identified by a thin colored outline around that subject'),
|
||||
}
|
||||
mark, explanation = marks[mode]
|
||||
definitions = '\n'.join(
|
||||
f'<Subject {j}> is the subject marked by the {COLORS[j-1]} {mark} in <Picture 1>; its appearance comes from <Picture {j+1}>, its position, pose, head orientation, gaze and facial expression from the {COLORS[j-1]} mark.'
|
||||
for j in range(1, count+1))
|
||||
replacements = ', '.join(f'the {COLORS[j-1]} subject replaced by <Picture {j+1}>' for j in range(1, count+1))
|
||||
retention = '\n'.join(
|
||||
f'<Subject {j}>: attribute_transfer - keeps the position, pose, head orientation, gaze and facial expression of the {COLORS[j-1]} mark and takes the face, hair, body and whole outfit of <Picture {j+1}>.'
|
||||
for j in range(1, count+1))
|
||||
return (
|
||||
'character_swap:\nsubject_definitions:\n'
|
||||
f'<Picture 1> is the source of the edit. In it, {explanation}; the mark says who changes, not where the new look must end, and everything else in the guide is the original image.\n'
|
||||
f'{definitions}\n'
|
||||
f'summary: [image editing + reference generation] The target image is <Picture 1> with {replacements}, following each mark\'s pose and expression. '
|
||||
'Every other person, the layout, the background, lighting, camera perspective and framing stay exactly as in <Picture 1>. '
|
||||
'Keep held objects and foreground occluders in their original positions, shapes and colors; they are separate from the subjects being replaced. '
|
||||
'Remove the normal-map colors, skeletons, overlays and outlines from the final image.\n'
|
||||
f'retention_analysis:\n{retention}\n'
|
||||
'Everyone else, held objects, foreground occluders, the layout and the background: fully_preserved.')
|
||||
|
||||
|
||||
class BFSH3ImageCharacterSwapConditioning(io.ComfyNode):
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
return io.Schema(
|
||||
node_id='BFSH3ImageCharacterSwapConditioning',
|
||||
display_name='BFS H3 Image Character Swap Conditioning',
|
||||
category='BFS/MiniMax H3',
|
||||
description='Native H3 Image character swap: the marked guide is Picture 1; red, blue and green subjects take their appearance from Pictures 2, 3 and 4. Produces native image reference latents and a still-image prompt.',
|
||||
inputs=[
|
||||
io.Clip.Input('clip'), io.Vae.Input('vae'),
|
||||
io.Image.Input('guide_image', tooltip='Marked hybrid image from BFS Video Segment Picker; becomes Picture 1.'),
|
||||
io.Autogrow.Input('references', template=io.Autogrow.TemplatePrefix(
|
||||
input=io.Image.Input('reference', tooltip='Character appearance in color order: red, blue, green; becomes Picture 2, 3, 4.'),
|
||||
prefix='reference_', min=1, max=3)),
|
||||
io.Combo.Input('mode', options=MODES, default=MODES[0]),
|
||||
io.Int.Input('width', default=0, min=0, max=16384, step=32, tooltip='0 uses the native H3 Image sizing based on the guide.'),
|
||||
io.Int.Input('height', default=0, min=0, max=16384, step=32),
|
||||
io.String.Input('prompt_override', default='', multiline=True, optional=True,
|
||||
tooltip='Optional complete replacement prompt. Guide remains Picture 1; character references start at Picture 2.'),
|
||||
],
|
||||
outputs=[io.Conditioning.Output(display_name='positive'), io.Latent.Output(display_name='latent'),
|
||||
io.String.Output(display_name='prompt')])
|
||||
|
||||
@classmethod
|
||||
def execute(cls, clip, vae, guide_image, references, mode=MODES[0], width=0, height=0, prompt_override=''):
|
||||
names = sorted(references, key=lambda name: int(name.rsplit('_', 1)[-1]))
|
||||
pictures = [references[name] for name in names if references[name] is not None]
|
||||
prompt = prompt_override.strip() or image_swap_prompt(len(pictures), mode)
|
||||
images = {'image_1': guide_image}
|
||||
images.update({f'image_{j}': picture for j, picture in enumerate(pictures, 2)})
|
||||
positive, latent = MiniMaxH3ImageConditioning.execute(
|
||||
clip=clip, vae=vae, prompt=prompt, width=width, height=height, images=images).args[:2]
|
||||
return io.NodeOutput(positive, latent, prompt)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {'BFSH3ImageCharacterSwapConditioning': BFSH3ImageCharacterSwapConditioning}
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {'BFSH3ImageCharacterSwapConditioning': 'BFS H3 Image Character Swap Conditioning'}
|
||||
+59
-12
@@ -1,12 +1,16 @@
|
||||
"""BFS H3 Character Swap Conditioning: MiniMax H3 Character Swap where the guide says who is replaced.
|
||||
|
||||
Three ways to mark the subjects (`mode`):
|
||||
Five ways to mark the subjects (`mode`):
|
||||
- "keyframe pins": the source video as an aligned guide (Add Guide at frame 0) plus one Add Guide per edited keyframe
|
||||
at its own frame index (frames of the result with the new people in place, e.g. from BFS Keyframe Planner).
|
||||
- "normal map guide": the guide video with each subject to replace drawn as its normal map in gray detail, tinted with
|
||||
the subject's color (1 red, 2 blue, 3 green = Picture 1..3).
|
||||
- "DWPose guide": the guide video with each subject erased to gray and drawn as its DWPose skeleton (body, hands and
|
||||
face, with the eyes and the mouth opening) in the subject's color.
|
||||
- "translucent overlay guide": the source video with each subject's mask tinted in its color at the Picker's
|
||||
mark_opacity, so the original pose and motion remain visible under the tint.
|
||||
- "outline guide": the source video with a thin colored contour inside each subject's mask; the scene and held
|
||||
objects remain visible.
|
||||
The marked guides come from BFS Video Segment Picker (`hybrid` output). The Pictures are native references and the
|
||||
prompt is written from the mode and the number of Pictures (or prompt_override).
|
||||
"""
|
||||
@@ -17,8 +21,9 @@ import torch
|
||||
GRAY = 128.0 / 255.0 # background of the keyframe_video preview
|
||||
MAX_KEYFRAMES = 8
|
||||
MAX_PICTURES = 3
|
||||
PINS, NORMALS, POSE = "keyframe pins", "normal map guide", "DWPose guide"
|
||||
MODES = [PINS, NORMALS, POSE]
|
||||
PINS, NORMALS, POSE, OVERLAY, OUTLINE = ("keyframe pins", "normal map guide", "DWPose guide",
|
||||
"translucent overlay guide", "outline guide")
|
||||
MODES = [PINS, NORMALS, POSE, OVERLAY, OUTLINE]
|
||||
SUBJECT_COLORS = ["red", "blue", "green"]
|
||||
|
||||
|
||||
@@ -53,7 +58,46 @@ def normals_caption(k):
|
||||
'changes, not where the new look must end, and everything else in the guide is the real video.\n' + defs +
|
||||
f'\nsummary: [video editing + reference generation] The target video is the guide video with {who}, from the first frame '
|
||||
'to the last, following each mark\'s pose and motion. Every other person, the layout, the backgrounds, the camera and the '
|
||||
'motion stay exactly as in the guide video.\nretention_analysis:\n' + ret + '\nEveryone else, the layout and the backgrounds: fully_preserved.')
|
||||
'motion stay exactly as in the guide video. Keep held objects and foreground occluders in their original '
|
||||
'positions, shapes and colors; they are separate from the subject being replaced.\nretention_analysis:\n' + ret +
|
||||
'\nEveryone else, held objects, foreground occluders, the layout and the backgrounds: fully_preserved.')
|
||||
|
||||
|
||||
def overlay_caption(k):
|
||||
"""Prompt for the Picker's translucent color overlay, where source appearance is still visible underneath."""
|
||||
C = SUBJECT_COLORS
|
||||
defs = '\n'.join(f'<Subject {j}> is the person under the translucent {C[j - 1]} overlay in the guide video. '
|
||||
f'Replace that person with <Picture {j}> while keeping their position, pose, motion and timing.'
|
||||
for j in range(1, k + 1))
|
||||
ret = '\n'.join(f'<Subject {j}>: attribute_transfer - use the face, hair, body and full outfit of <Picture {j}>; '
|
||||
f'use the {C[j - 1]} marked person only for position, pose, motion and timing.'
|
||||
for j in range(1, k + 1))
|
||||
return ('character_swap:\nsubject_definitions:\nThe guide is the original video with a translucent color overlay on each '
|
||||
'person to replace. The overlay identifies the person; the original appearance visible beneath it is not the target '
|
||||
'appearance. The tint must disappear in the result.\n' + defs +
|
||||
'\nsummary: [video editing + reference generation] Replace every marked person throughout the video with their '
|
||||
'matching Picture. Keep their actions, camera motion, all unmarked people and the background as in the guide. '
|
||||
'Do not render the colored overlays.\nretention_analysis:\n' + ret +
|
||||
'\nUnmarked people and background: fully_preserved.')
|
||||
|
||||
|
||||
def outline_caption(k):
|
||||
"""Prompt for a person contour that leaves the original scene and interactions unobscured."""
|
||||
C = SUBJECT_COLORS
|
||||
defs = '\n'.join(f'<Subject {j}> is the person inside the {C[j - 1]} outline in the guide video. '
|
||||
f'Take their new appearance from <Picture {j}> while retaining their position, pose, motion and timing.'
|
||||
for j in range(1, k + 1))
|
||||
ret = '\n'.join(f'<Subject {j}>: attribute_transfer - use the face, hair, body and full outfit of <Picture {j}>; '
|
||||
f'keep the movement and interactions of the {C[j - 1]} outlined person.'
|
||||
for j in range(1, k + 1))
|
||||
return ('character_swap:\nsubject_definitions:\nThe guide is the original video with a thin colored outline on each '
|
||||
'person to replace. The outline is only an identity marker and must disappear in the result. The source person '
|
||||
'inside the outline provides pose and motion, while their new appearance comes from the matching Picture.\n' + defs +
|
||||
'\nsummary: [video editing + reference generation] Replace only the outlined people throughout the video. '
|
||||
'Preserve the background, other people, and all held or interacting objects with their original colors, positions, '
|
||||
'shapes and contact with the subjects. Preserve foreground occlusions and the camera motion. Do not render the '
|
||||
'colored outlines.\nretention_analysis:\n' + ret +
|
||||
'\nBackground, other people, held objects and occlusions: fully_preserved.')
|
||||
|
||||
|
||||
def _tail(k):
|
||||
@@ -108,8 +152,9 @@ class BFSH3KeyframeConditioning:
|
||||
"width": ("INT", {"forceInput": True}), "height": ("INT", {"forceInput": True}), "length": ("INT", {"forceInput": True}),
|
||||
"picture_1": ("IMAGE", {"tooltip": "Subject 1 (red mark in the guide modes)."}),
|
||||
"mode": (MODES, {"default": PINS, "tooltip": "How the subjects to replace are marked: edited keyframes pinned at their "
|
||||
"frames, or a guide with each subject drawn as a colored normal map or "
|
||||
"DWPose skeleton (BFS Video Segment Picker)."}),
|
||||
"frames, or a guide with each subject drawn as a colored normal map, "
|
||||
"DWPose skeleton, translucent color overlay, or colored outline "
|
||||
"(BFS Video Segment Picker)."}),
|
||||
"keyframe_until": ("FLOAT", {"default": 0.7, "min": 0.0, "max": 1.0, "step": 0.01,
|
||||
"tooltip": "keyframe pins only, on the returned model: the guides condition every step whose "
|
||||
"sigma is at least this; below it only the Pictures remain. 0.5 = more faithful to "
|
||||
@@ -128,8 +173,9 @@ class BFSH3KeyframeConditioning:
|
||||
FUNCTION = "encode"
|
||||
CATEGORY = "BFS/MiniMax H3"
|
||||
DESCRIPTION = ("MiniMax H3 Character Swap conditioning. keyframe pins: the source video plus one Add Guide per edited "
|
||||
"keyframe. normal map guide / DWPose guide: the marked guide from BFS Video Segment Picker, subjects 1-4 = "
|
||||
"red, blue, green, yellow = Picture 1-4. The Pictures as references and the matching prompt.")
|
||||
"keyframe. Guide modes use the BFS Video Segment Picker hybrid output: normal map, DWPose, translucent "
|
||||
"overlay or colored outline. Red, blue and green marks correspond to Pictures 1, 2 and 3. "
|
||||
"The node writes the matching prompt automatically.")
|
||||
|
||||
@staticmethod
|
||||
def _collect(kw, fps=24.0):
|
||||
@@ -290,7 +336,7 @@ class BFSH3KeyframeConditioning:
|
||||
pictures = [picture_1] + [kw[f"picture_{j}"] for j in range(2, MAX_PICTURES + 1) if kw.get(f"picture_{j}") is not None]
|
||||
kfs = self._collect(kw)
|
||||
if mode != PINS:
|
||||
return self._guide(clip, vae, guide_video, width, height, length, pictures, prompt_override, model, kw, pose=mode == POSE)
|
||||
return self._guide(clip, vae, guide_video, width, height, length, pictures, prompt_override, model, kw, mode=mode)
|
||||
if not kfs:
|
||||
raise ValueError("No keyframes: connect the Planner's keyframes + keyframe_spec, or keyframe_1 / time_1.")
|
||||
if len(guide_video) != length:
|
||||
@@ -322,13 +368,14 @@ class BFSH3KeyframeConditioning:
|
||||
return (self._until(model, keyframe_until, keyframe_until_drops.startswith("keyframes")), positive, latent, prompt, video,
|
||||
pictures)
|
||||
|
||||
def _guide(self, clip, vae, guide_video, width, height, length, pictures, prompt_override, model, kw, pose=False):
|
||||
"""Normal map / DWPose guide: the marked guide (subjects drawn as color-coded marks) as the only aligned guide, every
|
||||
def _guide(self, clip, vae, guide_video, width, height, length, pictures, prompt_override, model, kw, mode=NORMALS):
|
||||
"""Marked guide (subjects drawn as color-coded marks) as the only aligned guide, every
|
||||
step; the Pictures in color order; no keyframes."""
|
||||
from comfy_extras.nodes_minimax_h3 import MiniMaxH3AddGuide, MiniMaxH3ReferenceToVideo
|
||||
if len(guide_video) != length:
|
||||
raise ValueError(f"guide has {len(guide_video)} frames, length is {length}: connect Prepare's length.")
|
||||
prompt = prompt_override.strip() or (pose_caption if pose else normals_caption)(len(pictures))
|
||||
captions = {NORMALS: normals_caption, POSE: pose_caption, OVERLAY: overlay_caption, OUTLINE: outline_caption}
|
||||
prompt = prompt_override.strip() or captions[mode](len(pictures))
|
||||
print("[BFS H3 keyframe] guide prompt:\n" + prompt, flush=True)
|
||||
try:
|
||||
positive, latent = MiniMaxH3ReferenceToVideo.execute(
|
||||
|
||||
+18
-6
@@ -163,8 +163,11 @@ def detect_text(sam: tuple, frame: torch.Tensor, text: str, threshold: float, ma
|
||||
from comfy_extras.nodes_sam3 import SAM3_Detect
|
||||
model, clip = sam
|
||||
h, w = frame.shape[1:3]
|
||||
masks, boxes = SAM3_Detect.execute(model=model, image=frame, conditioning=_text_cond(clip, text, max_detections),
|
||||
threshold=float(threshold), refine_iterations=1, individual_masks=True).args
|
||||
# ComfyUI can materialize SAM weights as inference tensors. Its detector must then
|
||||
# run under inference_mode too; otherwise autograd asks those weights for a version counter.
|
||||
with torch.inference_mode():
|
||||
masks, boxes = SAM3_Detect.execute(model=model, image=frame, conditioning=_text_cond(clip, text, max_detections),
|
||||
threshold=float(threshold), refine_iterations=1, individual_masks=True).args
|
||||
masks = masks.cpu() > .5 if masks.numel() else torch.zeros(0, h, w, dtype=torch.bool)
|
||||
boxes = boxes[0] if boxes else []
|
||||
return [(m, {"score": round(float(b.get("score", 0)), 3)}) for m, b in zip(masks, boxes)]
|
||||
@@ -176,8 +179,9 @@ def detect_points(sam: tuple, frame: torch.Tensor, points: list[dict], threshold
|
||||
h, w = frame.shape[1:3]
|
||||
pos = [{"x": p["x"] * w, "y": p["y"] * h} for p in points if p["label"]]
|
||||
neg = [{"x": p["x"] * w, "y": p["y"] * h} for p in points if not p["label"]]
|
||||
masks = SAM3_Detect.execute(model=model, image=frame, positive_coords=json.dumps(pos), negative_coords=json.dumps(neg),
|
||||
threshold=float(threshold), refine_iterations=2).args[0]
|
||||
with torch.inference_mode():
|
||||
masks = SAM3_Detect.execute(model=model, image=frame, positive_coords=json.dumps(pos), negative_coords=json.dumps(neg),
|
||||
threshold=float(threshold), refine_iterations=2).args[0]
|
||||
return (masks[0].cpu() > .5) if masks.numel() else torch.zeros(h, w, dtype=torch.bool)
|
||||
|
||||
|
||||
@@ -211,8 +215,16 @@ def auto_objects(sam: tuple, frame: torch.Tensor, concepts: list[str], threshold
|
||||
found = []
|
||||
for concept in concepts:
|
||||
for m, info in detect_text(sam, frame, concept, threshold, max_per_concept):
|
||||
if int(m.sum()) >= max(64, min_area * h * w):
|
||||
found.append((m, {**info, "label": concept}))
|
||||
area = int(m.sum())
|
||||
if area < max(64, min_area * h * w):
|
||||
continue
|
||||
# Generic "objects" may segment the entire scene. Such a mask is
|
||||
# unsafe as an auto subject: it paints the background into the guide.
|
||||
if concept.strip().casefold() in {"object", "objects"} and area >= .25 * h * w:
|
||||
_, _, bw, bh = box_of(m)
|
||||
if bw >= .98 * w and bh >= .98 * h:
|
||||
continue
|
||||
found.append((m, {**info, "label": concept}))
|
||||
found = dedupe(found)
|
||||
found.sort(key=lambda f: box_of(f[0])[0]) # left to right, so "a1" reads naturally
|
||||
return [{"id": f"a{k + 1}", "label": info["label"], "source": "auto", "score": info["score"], "mask": m}
|
||||
|
||||
+13
-3
@@ -140,6 +140,8 @@ class BFSVideoCompare:
|
||||
"size) or a batch; up to 12."}),
|
||||
"label_guide": ("STRING", {"default": "guide"}),
|
||||
"label_target": ("STRING", {"default": "target"}),
|
||||
"free_memory_after": ("BOOLEAN", {"default": True, "tooltip":
|
||||
"After the workflow finishes, release cached node outputs and unloaded model weights from RAM. The next run reloads models."}),
|
||||
},
|
||||
}
|
||||
|
||||
@@ -157,10 +159,10 @@ class BFSVideoCompare:
|
||||
"where they differ, pixel by pixel.")
|
||||
|
||||
def compare(self, guide, target, fps=24.0, audio=None, gain=2.0, size="guide", preview_height=720,
|
||||
label_guide="guide", label_target="target", reference=None):
|
||||
label_guide="guide", label_target="target", reference=None, free_memory_after=True):
|
||||
one = lambda v: v[0] if isinstance(v, list) and v else v
|
||||
fps, audio, gain, size, preview_height, label_guide, label_target = map(
|
||||
one, (fps, audio, gain, size, preview_height, label_guide, label_target))
|
||||
fps, audio, gain, size, preview_height, label_guide, label_target, free_memory_after = map(
|
||||
one, (fps, audio, gain, size, preview_height, label_guide, label_target, free_memory_after))
|
||||
guide, target = video_batches(guide, "guide"), video_batches(target, "target")
|
||||
ng, nt = sum(x.shape[0] for x in guide), sum(x.shape[0] for x in target)
|
||||
if len(guide) > 1 or len(target) > 1:
|
||||
@@ -226,6 +228,14 @@ class BFSVideoCompare:
|
||||
"refs": refs,
|
||||
"stills": {k: {"filename": f, "subfolder": sub, "type": "temp"} for k, f in stills.items()},
|
||||
"report": report, "notes": notes}]}
|
||||
if free_memory_after:
|
||||
try:
|
||||
from server import PromptServer
|
||||
except ImportError: # the comparison can also run outside ComfyUI
|
||||
PromptServer = None
|
||||
queue = getattr(getattr(PromptServer, "instance", None), "prompt_queue", None)
|
||||
if queue is not None:
|
||||
queue.set_flag("free_memory", True)
|
||||
return {"ui": ui, "result": (diff_map, report, mean)}
|
||||
|
||||
|
||||
|
||||
+309
-79
@@ -34,10 +34,11 @@ def hybrid_tint(normal_rgb, color, alpha=HYBRID_TINT):
|
||||
return (detail * c * alpha + detail * (1 - alpha)).clamp(0, 1)
|
||||
|
||||
|
||||
FREE_MODES = ["unload all models from the GPU", "unload SAM 3 from the GPU", "unload all models + clear the picker caches",
|
||||
FREE_MODES = ["unload all models from the GPU", "unload all models + release Molmo2 RAM", "unload SAM 3 from the GPU", "unload all models + clear the picker caches",
|
||||
"keep everything loaded"]
|
||||
LEGACY_FREE = {"unload SAM 3 + clear the picker caches": "unload all models + clear the picker caches"}
|
||||
MARK_STYLES = ["normal map (needs normals)", "openpose + face (needs pose_keypoint)", "flat color", "shading (needs normals)", "blur",
|
||||
PROTECTED_NORMAL_STYLE = "normal map inside mask (needs normals)"
|
||||
MARK_STYLES = ["normal map (needs normals)", PROTECTED_NORMAL_STYLE, "openpose + face (needs pose_keypoint)", "flat color", "translucent overlay", "colored outline", "shading (needs normals)", "blur",
|
||||
"pixelate", "gray detail", "edges"]
|
||||
|
||||
|
||||
@@ -54,7 +55,7 @@ def mark_fill(style, frame, normal_rgb, mask, color, alpha=HYBRID_TINT):
|
||||
light = n.new_tensor([-0.35, -0.5, 0.8]); light = light / light.norm()
|
||||
lum = (0.30 + 0.70 * (n * light).sum(-1).clamp(0, 1))[..., None]
|
||||
return (lum * c).clamp(0, 1)
|
||||
if style == "flat color":
|
||||
if style in ("flat color", "translucent overlay", "colored outline"):
|
||||
return c.expand_as(frame).clone()
|
||||
gray = (frame[..., 0] * 0.299 + frame[..., 1] * 0.587 + frame[..., 2] * 0.114)
|
||||
ys, xs = torch.nonzero(mask, as_tuple=True)
|
||||
@@ -133,6 +134,7 @@ def person_at(sam, frame, x, y, threshold):
|
||||
|
||||
_MOLMO_CACHE = OrderedDict() # (video, subject, anchor frame, anchor mask) -> (description, points): Molmo2 runs once per pick
|
||||
_SHOT_CACHE = OrderedDict() # (video, subject, shot, its anchors + masks, threshold) -> (masks, report) of that shot
|
||||
_PICKER_MASK_CACHE_MAX_BYTES = 2 * 1024 ** 3 # keep picker mask caches bounded by RAM as well as entry count
|
||||
|
||||
|
||||
def mask_key(m):
|
||||
@@ -141,10 +143,28 @@ def mask_key(m):
|
||||
return hashlib.sha1(np.packbits((m > 0.5).cpu().numpy()).tobytes()).hexdigest()[:16]
|
||||
|
||||
|
||||
def _remember(cache, key, value, cap):
|
||||
def _cache_tensor_bytes(value, seen=None):
|
||||
"""Count unique tensor storages in a cache value; item-count caps alone are unsafe for full videos."""
|
||||
if seen is None:
|
||||
seen = set()
|
||||
if isinstance(value, torch.Tensor):
|
||||
storage = value.untyped_storage()
|
||||
storage_key = (str(value.device), storage.data_ptr(), storage.nbytes())
|
||||
if storage_key in seen:
|
||||
return 0
|
||||
seen.add(storage_key)
|
||||
return storage.nbytes()
|
||||
if isinstance(value, dict):
|
||||
return sum(_cache_tensor_bytes(v, seen) for v in value.values())
|
||||
if isinstance(value, (list, tuple)):
|
||||
return sum(_cache_tensor_bytes(v, seen) for v in value)
|
||||
return 0
|
||||
|
||||
|
||||
def _remember(cache, key, value, cap, max_bytes=None):
|
||||
cache[key] = value
|
||||
cache.move_to_end(key)
|
||||
while len(cache) > cap:
|
||||
while len(cache) > cap or (max_bytes is not None and sum(_cache_tensor_bytes(v) for v in cache.values()) > max_bytes):
|
||||
cache.popitem(last=False)
|
||||
|
||||
|
||||
@@ -498,9 +518,79 @@ def grow_masks(masks, rel):
|
||||
return F.max_pool2d(masks[:, None].float(), 2 * r + 1, 1, r)[:, 0]
|
||||
|
||||
|
||||
def feather_inside_mask(mask, radius):
|
||||
"""Soften a rendered mask edge without painting outside the selected pixels."""
|
||||
if radius <= 0:
|
||||
return mask.float()
|
||||
import cv2
|
||||
import numpy as np
|
||||
binary = mask.numpy().astype(np.uint8)
|
||||
distance = cv2.distanceTransform(binary, cv2.DIST_L2, cv2.DIST_MASK_PRECISE)
|
||||
alpha = np.clip(distance / radius, 0.0, 1.0)
|
||||
alpha = cv2.GaussianBlur(alpha, (0, 0), sigmaX=0.8)
|
||||
return torch.from_numpy(alpha * binary)
|
||||
|
||||
|
||||
def anchor_windows(anchors, start, end):
|
||||
"""Split a shot at nearest-anchor midpoints; ties stay with the earlier anchor."""
|
||||
anchors = sorted(a for a in set(anchors) if start <= a < end)
|
||||
for pos, anchor in enumerate(anchors):
|
||||
lo = start if pos == 0 else (anchors[pos - 1] + anchor) // 2 + 1
|
||||
hi = end if pos == len(anchors) - 1 else (anchor + anchors[pos + 1]) // 2 + 1
|
||||
yield anchor, lo, hi
|
||||
|
||||
|
||||
def consecutive_runs(indices):
|
||||
run = []
|
||||
for i in indices:
|
||||
if run and i != run[-1] + 1:
|
||||
yield run
|
||||
run = []
|
||||
run.append(i)
|
||||
if run:
|
||||
yield run
|
||||
|
||||
|
||||
def fill_empty_mask_runs(image, model, masks, report, shots, active_frame=-1):
|
||||
"""Track only empty stretches from the nearest existing mask in the same shot."""
|
||||
filled = 0
|
||||
for s0, s1 in shots:
|
||||
anchors = [i for i in range(s0, s1) if bool(masks[i].any())]
|
||||
if not anchors:
|
||||
continue
|
||||
empty = [i for i in range(s0, s1) if not bool(masks[i].any())]
|
||||
for run in consecutive_runs(empty):
|
||||
lo, hi = run[0], run[-1] + 1
|
||||
if all(report[i].get("source") == "before first pick" for i in run):
|
||||
continue
|
||||
anchor = min(anchors, key=lambda a: (min(abs(a - lo), abs(a - (hi - 1))), a != active_frame, a))
|
||||
begin, end = min(anchor, lo), max(anchor + 1, hi)
|
||||
tracked = BFSTrackFromFrame().track(image[begin:end, ..., :3], model,
|
||||
masks[anchor][None].float(), anchor - begin)[0]
|
||||
for i in run:
|
||||
new_mask = tracked[i - begin] > 0.5
|
||||
if bool(new_mask.any()):
|
||||
masks[i] = new_mask
|
||||
report[i] = {"frame": i, "source": f"filled from {anchor}"}
|
||||
filled += 1
|
||||
return filled
|
||||
|
||||
|
||||
from collections import OrderedDict
|
||||
_TRACK_CACHE = OrderedDict() # (video, subject, its picks, cuts, options) -> (masks, report), most recent last
|
||||
_LAST_TRACK_CACHE = OrderedDict() # (video, subject) -> last completed propagated masks/report
|
||||
_LOCK_CACHE = OrderedDict() # (video, locked objects and cuts) -> tracked protection masks
|
||||
|
||||
|
||||
def _cache_track_result(ckey, vkey, sj, masks, report, picks, keep_last=True):
|
||||
# Masks are binary after propagation. Keep one bool storage shared by both caches instead of
|
||||
# two full-resolution float32 clones (4x larger per frame).
|
||||
compact = masks > 0.5
|
||||
report_copy = [dict(r) for r in report]
|
||||
_remember(_TRACK_CACHE, ckey, (compact, report_copy), 24, _PICKER_MASK_CACHE_MAX_BYTES)
|
||||
if keep_last:
|
||||
_remember(_LAST_TRACK_CACHE, (vkey, sj), (compact, report_copy, picks), 24,
|
||||
_PICKER_MASK_CACHE_MAX_BYTES)
|
||||
|
||||
|
||||
def video_key(image):
|
||||
@@ -510,7 +600,7 @@ def video_key(image):
|
||||
return hashlib.sha1(smp.tobytes() + str(tuple(image.shape)).encode()).hexdigest()[:16]
|
||||
|
||||
|
||||
def timeline_strip(image, per_subject, width=256, max_frames=400):
|
||||
def timeline_strip(image, per_subject, protected=None, width=256, max_frames=400):
|
||||
"""Small JPEGs of every frame with each subject's mask tinted in its color, for the panel's whole-video scrubber
|
||||
(so a wrong mask can be found on any frame, not only on the strip)."""
|
||||
import os, uuid
|
||||
@@ -527,6 +617,9 @@ def timeline_strip(image, per_subject, width=256, max_frames=400):
|
||||
c = torch.tensor(HYBRID_COLORS[(int(sj) - 1) % len(HYBRID_COLORS)], dtype=fr.dtype) / 255.0
|
||||
mk = (m[i] > 0.5)[..., None].to(fr.dtype) * 0.55
|
||||
fr = fr * (1 - mk) + c * mk
|
||||
if protected is not None and bool(protected[i].any()):
|
||||
mk = protected[i][..., None].to(fr.dtype) * 0.55
|
||||
fr = fr * (1 - mk) + fr.new_tensor([1.0, 0.78, 0.18]) * mk
|
||||
arr = (fr.clamp(0, 1).cpu().numpy() * 255).astype(np.uint8)
|
||||
name = f"f{i:05d}.jpg"
|
||||
Image.fromarray(arr).resize((sw, sh), Image.BILINEAR).save(os.path.join(out_dir, name), quality=80)
|
||||
@@ -576,7 +669,7 @@ def parse_frames(text):
|
||||
continue
|
||||
out[str(k)] = {"1": v} if ("selected" in v or "added" in v) else {
|
||||
str(sk): sv for sk, sv in v.items()
|
||||
if isinstance(sv, dict) and str(sk).isdigit() and 1 <= int(sk) <= len(HYBRID_COLORS)
|
||||
if isinstance(sv, dict) and (str(sk) == "lock" or (str(sk).isdigit() and 1 <= int(sk) <= len(HYBRID_COLORS)))
|
||||
}
|
||||
return out
|
||||
|
||||
@@ -620,6 +713,8 @@ class BFSVideoSegmentPicker:
|
||||
t = sp.BFSSegmentPicker.INPUT_TYPES()
|
||||
req = dict(t["required"])
|
||||
req["image"] = ("IMAGE", {"tooltip": f"The frames: the Keyframe Planner's source_frames or a short clip (at most {MAX_FRAMES})."})
|
||||
req["mark_opacity"] = ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.05,
|
||||
"tooltip": "Opacity of the subject mark in both outputs: the marked preview and the H3 hybrid guide. Use 1 for a clearly visible normal map; lower values blend the original frame into both outputs."})
|
||||
req["selection"] = ("STRING", {"default": "{}", "multiline": True, "tooltip": "Set by the panel: the selection on each frame."})
|
||||
req["track"] = ("BOOLEAN", {"default": True, "tooltip": "Frames without a selection take the nearest selected frame's mask, tracked with SAM 3."})
|
||||
opt = {
|
||||
@@ -632,10 +727,16 @@ class BFSVideoSegmentPicker:
|
||||
"for mark_style 'openpose + face'."}),
|
||||
"molmo_clip": ("CLIP", {"lazy": True, "tooltip": "Optional: Molmo2 as H3 encoder. It sees the whole video, finds the picked person again after "
|
||||
"camera cuts and helps recover missed masks. In the default Molmo2 (classic) mode it describes the selected person, tracks that description through the video, and uses its points to guide SAM 3.1."}),
|
||||
"point_model": ("MOLMO_POINT_MODEL", {"lazy": True, "tooltip": "MolmoPoint-8B point tracker. Use with propagation_method 'MolmoPoint-8B'; its native point tokens guide SAM 3 without changing the H3 text encoder."}),
|
||||
"mark_style": (MARK_STYLES, {"default": MARK_STYLES[0],
|
||||
"tooltip": "How each subject is drawn in the hybrid output, always in its color: the normal map, the DWPose "
|
||||
"skeleton with the face, or texture-free alternatives that need no MoGe (flat, blur, pixelate, gray "
|
||||
"detail, edges). With a style that needs no normals, 'hybrid' works without the normals input."}),
|
||||
"detail, edges). 'normal map inside mask' hides source texture only inside the person: "
|
||||
"everything outside is protected automatically. It feathers the boundary and also "
|
||||
"keeps objects selected with the padlock or protect_mask unchanged inside the person mask. "
|
||||
"'translucent overlay' colors the exact mask with mark_opacity. "
|
||||
"'colored outline' draws only a thin border inside the mask, preserving frame detail "
|
||||
"and held objects. Both work without normals."}),
|
||||
"hybrid_grow": ("FLOAT", {"default": 0.08, "min": 0.0, "max": 0.5, "step": 0.01,
|
||||
"tooltip": "Margin around the subject in the hybrid guide, relative to its size (normal map: 0.04-0.12; "
|
||||
"openpose: 0.08-0.18 around the person's convex hull)."}),
|
||||
@@ -643,7 +744,7 @@ class BFSVideoSegmentPicker:
|
||||
"tooltip": "How many frames the panel shows to pick on (SAM segments each one): fewer is faster."}),
|
||||
"free_after": (FREE_MODES, {"default": FREE_MODES[0],
|
||||
"tooltip": "After the node: free the GPU for the next nodes (H3). 'all models' also unloads H3 / the text "
|
||||
"encoder left from the last generation (they load again when needed). Clearing the caches also "
|
||||
"encoder left from the last generation. 'release Molmo2 RAM' drops its CPU weights too, so the next Molmo2 run reloads them. Clearing the caches also "
|
||||
"frees RAM, but the next run segments and tracks again. Before MoGe / DWPose / Molmo2 run "
|
||||
"(only on Propagate) every model is unloaded anyway, so a second run does not run out of memory."}),
|
||||
"pose_region": (["person outline", "convex hull"], {"default": "person outline",
|
||||
@@ -659,10 +760,13 @@ class BFSVideoSegmentPicker:
|
||||
"tooltip": "Limit active subjects: 'auto' automatically detects connected picture inputs in downstream H3 conditioning (ignoring extra subjects like green if only 2 pictures are connected), or force 1, 2, or 3."}),
|
||||
"propagate_scope": (["all frames", "from first selected frame onward", "only this frame", "from active frame onward", "fill only empty masks"], {"default": "fill only empty masks",
|
||||
"tooltip": "Fill only empty masks keeps every existing non-empty frame unchanged and fills only frames whose mask is empty. Only this frame replaces one mask; from active frame onward recalculates from the open frame and preserves earlier results; all frames recomputes the clip."}),
|
||||
"propagation_method": (["Molmo2 (classic)", "SAM 3 (current)"], {"default": "Molmo2 (classic)",
|
||||
"tooltip": "Classic uses Molmo2 to describe the selected person, track it through the full video, and re-anchor weak or lost masks. SAM 3 (current) uses mask tracking with Molmo2-assisted recovery."}),
|
||||
"propagation_method": (["Molmo2 (classic)", "MolmoPoint-8B", "SAM 3 (current)"], {"default": "Molmo2 (classic)",
|
||||
"tooltip": "MolmoPoint-8B uses native point tokens to track and re-anchor the selected person; Molmo2 (classic) uses text coordinates. Both use SAM 3 for per-frame masks."}),
|
||||
"molmo_descriptions": ("STRING", {"multiline": False, "default": "",
|
||||
"tooltip": "Optional tracking description for subject 1. Leave blank to let Molmo2 describe the person you selected automatically. This only overrides the text query; the selected mask remains the visual anchor."}),
|
||||
"protect_mask": ("MASK", {"tooltip": "Optional external mask of objects to keep unchanged in 'normal map inside mask'. You can instead select protected objects with the padlock in this node. One mask or one per guide frame."}),
|
||||
"edge_feather": ("FLOAT", {"default": 0.01, "min": 0.0, "max": 0.1, "step": 0.005,
|
||||
"tooltip": "'normal map inside mask': soften the mark inward from the person and protected-object edges. Fraction of the shorter frame side."}),
|
||||
}
|
||||
return {"required": req, "optional": opt, "hidden": {**t.get("hidden", {}), "prompt": "PROMPT"}}
|
||||
|
||||
@@ -670,27 +774,29 @@ class BFSVideoSegmentPicker:
|
||||
RETURN_NAMES = ("masks", "marked", "overlay", "report", "hybrid")
|
||||
OUTPUT_TOOLTIPS = ("One mask per frame: the person to replace.", "Every frame with the person tinted (mark_color, mark_opacity).",
|
||||
"The strip frames with all objects found, numbered.", "JSON per frame: picked by hand, tracked from frame N, or empty.",
|
||||
"With normals: the frames (or hybrid_base) with this subject drawn as its tinted normal map; else hybrid_base or the frames.")
|
||||
"The H3 guide with each subject drawn in its color using mark_style and mark_opacity; 'normal map inside mask' keeps pixels outside the person and protect_mask unchanged.")
|
||||
FUNCTION = "pick"
|
||||
CATEGORY = "BFS/Mask"
|
||||
OUTPUT_NODE = True
|
||||
|
||||
def pick(self, image, sam3_model, sam3_clip, concepts, threshold, max_per_concept, min_area, mark_color, mark_opacity,
|
||||
selection, track=True, strip_frames=16, normals=None, normals_convention="normal_directx", hybrid_base=None, subject=None,
|
||||
mark_style=MARK_STYLES[0], molmo_clip=None, pose_keypoint=None,
|
||||
mark_style=MARK_STYLES[0], molmo_clip=None, point_model=None, pose_keypoint=None,
|
||||
hybrid_grow=0.08, free_after=FREE_MODES[0], pose_region="person outline",
|
||||
pose_gray_opacity=1.0, max_subjects="auto", molmo_descriptions="", propagate_scope="fill only empty masks",
|
||||
propagation_method="Molmo2 (classic)", unique_id=None, prompt=None, **kw):
|
||||
propagation_method="Molmo2 (classic)", unique_id=None, prompt=None, protect_mask=None, edge_feather=0.01, **kw):
|
||||
try:
|
||||
return self._pick(image, sam3_model, sam3_clip, concepts, threshold, max_per_concept, min_area, mark_color, mark_opacity,
|
||||
selection, track, strip_frames, normals, normals_convention, hybrid_base, subject, mark_style,
|
||||
molmo_clip, pose_keypoint, hybrid_grow, unique_id, pose_region, pose_gray_opacity,
|
||||
max_subjects, molmo_descriptions, propagate_scope, propagation_method, prompt)
|
||||
max_subjects, molmo_descriptions, propagate_scope, propagation_method, prompt,
|
||||
protect_mask=protect_mask, edge_feather=edge_feather, point_model=point_model)
|
||||
finally:
|
||||
self._free(sam3_model, free_after)
|
||||
self._free(sam3_model, free_after, point_model or molmo_clip)
|
||||
|
||||
def check_lazy_status(self, selection="{}", mark_style=MARK_STYLES[0], track=True, normals=None, pose_keypoint=None,
|
||||
molmo_clip=None, unique_id=None, prompt=None, **kw):
|
||||
molmo_clip=None, point_model=None, propagation_method="Molmo2 (classic)", propagate_scope="fill only empty masks",
|
||||
image=None, unique_id=None, prompt=None, **kw):
|
||||
"""The heavy optional inputs (MoGe normals, DWPose keypoints, Molmo2) are only needed when propagating, not while
|
||||
picking; before they run, every model leaves the GPU (e.g. H3 from the last generation), or MoGe runs out of memory."""
|
||||
try:
|
||||
@@ -700,12 +806,21 @@ class BFSVideoSegmentPicker:
|
||||
if segment_only:
|
||||
return []
|
||||
need = []
|
||||
if normals is None and not mark_style.startswith("openpose"):
|
||||
if normals is None and ("needs normals" in mark_style or mark_style == PROTECTED_NORMAL_STYLE):
|
||||
need.append("normals")
|
||||
if pose_keypoint is None and mark_style.startswith("openpose"):
|
||||
need.append("pose_keypoint")
|
||||
if molmo_clip is None and track:
|
||||
reuse_existing = False
|
||||
if propagate_scope == "fill only empty masks" and isinstance(image, torch.Tensor):
|
||||
states = parse_frames(selection)
|
||||
subjects = {int(sk) for frame_state in states.values() for sk, state in frame_state.items()
|
||||
if str(sk).isdigit() and state.get("selected")} or {1}
|
||||
vkey = video_key(image)
|
||||
reuse_existing = all((vkey, sj) in _LAST_TRACK_CACHE for sj in subjects)
|
||||
if molmo_clip is None and track and propagation_method == "Molmo2 (classic)" and not reuse_existing:
|
||||
need.append("molmo_clip")
|
||||
if point_model is None and track and propagation_method == "MolmoPoint-8B" and not reuse_existing:
|
||||
need.append("point_model")
|
||||
links = ((prompt or {}).get(str(unique_id)) or {}).get("inputs", {})
|
||||
need = [x for x in need if isinstance(links.get(x), list)] # only connected inputs can be requested
|
||||
if need:
|
||||
@@ -718,7 +833,7 @@ class BFSVideoSegmentPicker:
|
||||
return need
|
||||
|
||||
@staticmethod
|
||||
def _free(sam3_model, mode):
|
||||
def _free(sam3_model, mode, molmo_clip=None):
|
||||
mode = LEGACY_FREE.get(mode, mode)
|
||||
if mode.startswith("keep"):
|
||||
return
|
||||
@@ -731,6 +846,12 @@ class BFSVideoSegmentPicker:
|
||||
mm.unload_model_and_clones(sam3_model)
|
||||
if "caches" in mode:
|
||||
sp._SESSIONS.clear(); _TRACK_CACHE.clear(); _LAST_TRACK_CACHE.clear(); _SHOT_CACHE.clear(); _MOLMO_CACHE.clear()
|
||||
if "release Molmo2 RAM" in mode and molmo_clip is not None:
|
||||
runtime = getattr(molmo_clip, "runtime", None)
|
||||
if runtime is not None and hasattr(runtime, "release_ram"):
|
||||
runtime.release_ram()
|
||||
if hasattr(molmo_clip, "cache"):
|
||||
molmo_clip.cache.clear()
|
||||
gc.collect()
|
||||
mm.soft_empty_cache(True)
|
||||
print(f"[BFS Video Segment Picker] memory freed ({mode})", flush=True)
|
||||
@@ -741,7 +862,8 @@ class BFSVideoSegmentPicker:
|
||||
selection, track, strip_frames, normals, normals_convention, hybrid_base, subject, mark_style, molmo_clip,
|
||||
pose_keypoint, hybrid_grow, unique_id, pose_region="person outline", pose_gray_opacity=1.0,
|
||||
max_subjects="auto", molmo_descriptions="", propagate_scope="fill only empty masks",
|
||||
propagation_method="Molmo2 (classic)", prompt=None):
|
||||
propagation_method="Molmo2 (classic)", prompt=None, protect_mask=None, edge_feather=0.01,
|
||||
point_model=None):
|
||||
st = Status(unique_id)
|
||||
n = int(image.shape[0])
|
||||
st(f"start: {n} frames", 0.0, log=True)
|
||||
@@ -762,11 +884,14 @@ class BFSVideoSegmentPicker:
|
||||
segment_only = False
|
||||
active_frame = -1
|
||||
if segment_only:
|
||||
st("segment only (picking): no tracking, no Molmo2 - press Propagate to track", None, log=True)
|
||||
st("segment only (picking): no tracking or Molmo2; press Apply fix to track", None, log=True)
|
||||
if not segment_only and normals is None and (mark_style == PROTECTED_NORMAL_STYLE or "needs normals" in mark_style):
|
||||
raise ValueError("Connect MoGe normals to 'normals' for this mark style, or choose 'translucent overlay' to render without normals.")
|
||||
user_descs = [l.strip().strip('. ') for l in str(molmo_descriptions or '').splitlines()]
|
||||
while user_descs and not user_descs[-1]: user_descs.pop()
|
||||
|
||||
subjects = sorted({int(sk) for st in states.values() for sk, sv in st.items() if (sv.get("selected") or [])} or {1})
|
||||
subjects = sorted({int(sk) for frame_state in states.values() for sk, sv in frame_state.items()
|
||||
if str(sk).isdigit() and (sv.get("selected") or [])} or {1})
|
||||
max_allowed = None
|
||||
if str(max_subjects) in ("1", "2", "3"):
|
||||
max_allowed = int(max_subjects)
|
||||
@@ -782,6 +907,7 @@ class BFSVideoSegmentPicker:
|
||||
subjects = active
|
||||
panels, overlays = [], []
|
||||
own = {sj: {} for sj in subjects}
|
||||
own_locked = {}
|
||||
for k_strip, i in enumerate(strip):
|
||||
st(f"SAM 3: segmenting frame {i} ({k_strip + 1}/{len(strip)})", 0.02 + 0.28 * k_strip / max(1, len(strip)))
|
||||
frame = image[i:i + 1, ..., :3].float().cpu()
|
||||
@@ -797,6 +923,20 @@ class BFSVideoSegmentPicker:
|
||||
cut = torch.stack(cut).any(0) if cut else torch.zeros(h, w, dtype=torch.bool)
|
||||
mask, _ = sp.compose([o["mask"] & ~cut for o in chosen if not o.get("remove")], h, w)
|
||||
own[sj][i] = mask.reshape(h, w).float()
|
||||
lock_state = sp.parse_state(json.dumps(states.get(str(i), {}).get("lock", {})))
|
||||
lock_objects = sp.all_objects(sess, lock_state)
|
||||
lock_by_id = {o["id"]: o for o in lock_objects}
|
||||
lock_chosen = [lock_by_id[k] for k in lock_state["selected"] if k in lock_by_id]
|
||||
chosen_all |= {o["id"] for o in lock_chosen}
|
||||
lock_cut = [o["mask"] for o in lock_chosen if o.get("remove")]
|
||||
lock_cut = torch.stack(lock_cut).any(0) if lock_cut else torch.zeros(h, w, dtype=torch.bool)
|
||||
lock_mask, _ = sp.compose([o["mask"] & ~lock_cut for o in lock_chosen if not o.get("remove")], h, w)
|
||||
lock_mask = lock_mask.reshape(h, w) > 0.5
|
||||
selected_people = torch.stack([own[sj][i] > 0.5 for sj in subjects]).any(0)
|
||||
if bool(lock_mask.any()) and mask_overlap(lock_mask, selected_people)[0] >= 0.7:
|
||||
st(f"frame {i}: lock overlaps the selected subject; pick the object by text or add negative points", None, log=True)
|
||||
lock_mask = torch.zeros_like(lock_mask)
|
||||
own_locked[i] = lock_mask
|
||||
objects = sp.all_objects(sess, sp.parse_state(json.dumps({"added": [a for st in states.get(str(i), {}).values() for a in st.get("added", [])]})))
|
||||
panels.append(sp.panel_data(sid, sess, objects))
|
||||
overlays.append(sp.overlay(frame, objects, chosen_all))
|
||||
@@ -805,10 +945,37 @@ class BFSVideoSegmentPicker:
|
||||
st(f"camera cuts at frames {cuts}", 0.3, log=True)
|
||||
per_subject, report_s = {}, {}
|
||||
vkey = video_key(image)
|
||||
lock_picks = {k: frame_state["lock"] for k, frame_state in sorted(states.items()) if frame_state.get("lock")}
|
||||
lock_anchors = sorted(i for i in strip if bool(own_locked[i].any()))
|
||||
locked = None
|
||||
if lock_anchors:
|
||||
lock_key = (vkey, json.dumps(spec, sort_keys=True), json.dumps(lock_picks, sort_keys=True),
|
||||
tuple(cuts), bool(track), bool(segment_only))
|
||||
cached_lock = _LOCK_CACHE.get(lock_key)
|
||||
if cached_lock is not None:
|
||||
locked = cached_lock
|
||||
_LOCK_CACHE.move_to_end(lock_key)
|
||||
else:
|
||||
locked = torch.zeros((n, h, w), dtype=torch.bool)
|
||||
if track and not segment_only and n > 1:
|
||||
for s0, s1 in shots_of(n, cuts):
|
||||
shot_anchors = [a for a in lock_anchors if s0 <= a < s1]
|
||||
for a in shot_anchors:
|
||||
st(f"SAM 3: tracking protected object from frame {a} through shot {s0}-{s1 - 1}", 0.3)
|
||||
tracked = BFSTrackFromFrame().track(
|
||||
image[s0:s1, ..., :3], sam3_model, own_locked[a][None].float(), a - s0
|
||||
)[0].detach().cpu() > 0.5
|
||||
locked[s0:s1] |= tracked
|
||||
for a in lock_anchors:
|
||||
locked[a] = own_locked[a]
|
||||
_remember(_LOCK_CACHE, lock_key, locked, 16, _PICKER_MASK_CACHE_MAX_BYTES)
|
||||
for sj in subjects:
|
||||
mine = {k: v[str(sj)] for k, v in sorted(states.items()) if v.get(str(sj))} # only this subject's picks
|
||||
previous = _LAST_TRACK_CACHE.get((vkey, sj))
|
||||
ckey = ("v4", vkey, sj, json.dumps(mine, sort_keys=True), tuple(cuts), bool(track), molmo_clip is not None, float(threshold), propagate_scope, propagation_method, active_frame)
|
||||
ckey = ("v4", vkey, sj, json.dumps(mine, sort_keys=True), tuple(cuts), bool(track),
|
||||
(point_model if propagation_method == "MolmoPoint-8B" else molmo_clip) is not None,
|
||||
float(threshold),
|
||||
propagate_scope, propagation_method, active_frame)
|
||||
if (not segment_only and propagate_scope not in ("all frames", "fill only empty masks") and previous is not None
|
||||
and len(previous) > 2 and previous[2] == mine):
|
||||
per_subject[sj], report_s[sj] = previous[0].clone(), [dict(r) for r in previous[1]]
|
||||
@@ -833,30 +1000,46 @@ class BFSVideoSegmentPicker:
|
||||
min_valid_frame = (first_pick if propagate_scope in ("from first selected frame onward", "from active frame onward")
|
||||
else (0 if first_pick == 0 else prev_strip))
|
||||
if segment_only: # picking only: the picked frames' masks (the last propagated masks are kept by the cache)
|
||||
m = previous[0].clone() if previous is not None else torch.zeros(n, h, w)
|
||||
m = previous[0].clone() if previous is not None else torch.zeros(n, h, w, dtype=torch.bool)
|
||||
rp_prev = [dict(r) for r in previous[1]] if previous is not None else [
|
||||
{"frame": i, "source": "empty"} for i in range(n)]
|
||||
for i in user_anchors:
|
||||
m[i] = own[sj][i]
|
||||
m[i] = own[sj][i] > 0.5
|
||||
rp_prev[i] = {"frame": i, "source": "picked (pending propagate)"}
|
||||
per_subject[sj] = m
|
||||
report_s[sj] = rp_prev
|
||||
continue
|
||||
if propagate_scope == "fill only empty masks" and previous is not None:
|
||||
# A local correction must not trigger Molmo2 or retrack good frames.
|
||||
# Apply changed picks first, then use nearby existing masks only for gaps.
|
||||
m = previous[0].clone()
|
||||
rep = [dict(r) for r in previous[1]]
|
||||
old_picks = previous[2] if len(previous) > 2 else {}
|
||||
edited = {a for a in user_anchors if mine.get(str(a)) != old_picks.get(str(a))}
|
||||
if fix_frame in user_anchors:
|
||||
edited.add(fix_frame)
|
||||
for a in sorted(edited):
|
||||
m[a] = own[sj][a] > 0.5
|
||||
rep[a] = {"frame": a, "source": "picked (frame fix)"}
|
||||
filled = fill_empty_mask_runs(image, sam3_model, m, rep, shots_of(n, cuts), fix_frame) if track else 0
|
||||
st(f"subject {sj}: {len(edited)} picked frame(s) applied, {filled} empty frame(s) filled; "
|
||||
"existing masks kept", None, log=True)
|
||||
per_subject[sj] = m > 0.5; report_s[sj] = rep
|
||||
_cache_track_result(ckey, vkey, sj, m, rep, mine)
|
||||
continue
|
||||
m = torch.zeros(n, h, w)
|
||||
for i in anchors:
|
||||
m[i] = own[sj][i]
|
||||
rep = [{"frame": i, "source": "picked" if i in anchors else "empty"} for i in range(n)]
|
||||
if propagate_scope == "only this frame":
|
||||
m = previous[0].clone() if previous is not None else torch.zeros(n, h, w)
|
||||
m = previous[0].clone() if previous is not None else torch.zeros(n, h, w, dtype=torch.bool)
|
||||
rep = [dict(r) for r in previous[1]] if previous is not None else [
|
||||
{"frame": i, "source": "empty"} for i in range(n)]
|
||||
if 0 <= fix_frame < n and bool(own[sj].get(fix_frame, torch.zeros(h, w)).any()):
|
||||
m[fix_frame] = own[sj][fix_frame]
|
||||
m[fix_frame] = own[sj][fix_frame] > 0.5
|
||||
rep[fix_frame] = {"frame": fix_frame, "source": "picked (frame fix)"}
|
||||
per_subject[sj] = m; report_s[sj] = rep
|
||||
_TRACK_CACHE[ckey] = (m.clone(), [dict(r) for r in rep])
|
||||
_LAST_TRACK_CACHE[(vkey, sj)] = (m.clone(), [dict(r) for r in rep], mine)
|
||||
_LAST_TRACK_CACHE.move_to_end((vkey, sj))
|
||||
_cache_track_result(ckey, vkey, sj, m, rep, mine)
|
||||
continue
|
||||
if anchors and track and n > 1:
|
||||
def claimed_at(f):
|
||||
@@ -869,7 +1052,11 @@ class BFSVideoSegmentPicker:
|
||||
shots = shots_of(n, cuts)
|
||||
shot_of = lambda f: next(k for k, (s0, s1) in enumerate(shots) if s0 <= f < s1)
|
||||
has = [any(s0 <= a < s1 for a in anchors) for s0, s1 in shots]
|
||||
use_molmo = molmo_clip is not None and hasattr(molmo_clip, "runtime")
|
||||
tracker = point_model if propagation_method == "MolmoPoint-8B" else molmo_clip
|
||||
use_molmo = (propagation_method in ("Molmo2 (classic)", "MolmoPoint-8B")
|
||||
and tracker is not None and hasattr(tracker, "runtime"))
|
||||
if propagation_method == "MolmoPoint-8B" and not use_molmo:
|
||||
raise ValueError("Connect Auto Download & Load MolmoPoint-8B to point_model.")
|
||||
pts = []
|
||||
if use_molmo:
|
||||
st(f"Molmo2: describing and tracking subject {sj} from frame {first_pick} forward", 0.3 + 0.4 * subjects.index(sj) / len(subjects))
|
||||
@@ -878,11 +1065,12 @@ class BFSVideoSegmentPicker:
|
||||
desc_override = user_descs[sj - 1] if 0 <= sj - 1 < len(user_descs) and user_descs[sj - 1] else None
|
||||
if desc_override:
|
||||
st(f"Molmo2 subject {sj}: using explicit text description '{desc_override}'", None, log=True)
|
||||
relocate = molmo_relocate_legacy if propagation_method == "Molmo2 (classic)" else molmo_relocate
|
||||
if propagation_method == "Molmo2 (classic)":
|
||||
relocate = molmo_relocate_legacy if propagation_method in ("Molmo2 (classic)", "MolmoPoint-8B") else molmo_relocate
|
||||
if propagation_method in ("Molmo2 (classic)", "MolmoPoint-8B"):
|
||||
extra, note, pts = relocate(
|
||||
molmo_clip, (sam3_model, sam3_clip), image, own[sj][first_pick], first_pick,
|
||||
shots, has, float(threshold), memo=(vkey, sj, float(threshold), desc_override or ""),
|
||||
tracker, (sam3_model, sam3_clip), image, own[sj][first_pick], first_pick,
|
||||
shots, has, float(threshold), memo=(vkey, sj, propagation_method, id(tracker.runtime),
|
||||
float(threshold), desc_override or ""),
|
||||
text_override=desc_override
|
||||
)
|
||||
else:
|
||||
@@ -905,7 +1093,7 @@ class BFSVideoSegmentPicker:
|
||||
out[i] = torch.zeros(h, w); rp[i] = "before first pick"
|
||||
continue
|
||||
sa = [a for a in anchors if s0 <= a < s1]
|
||||
skey = ("v2", vkey, sj, s0, s1, min_valid_frame, tuple((a, mask_key(own[sj][a])) for a in sa), float(threshold))
|
||||
skey = ("v3", vkey, sj, s0, s1, min_valid_frame, tuple((a, mask_key(own[sj][a])) for a in sa), float(threshold))
|
||||
if skey in _SHOT_CACHE and propagate_scope != "fill only empty masks": # fill mode retries shots with empty masks instead of reusing a failed track
|
||||
cm, cr = _SHOT_CACHE[skey]
|
||||
_SHOT_CACHE.move_to_end(skey)
|
||||
@@ -915,24 +1103,25 @@ class BFSVideoSegmentPicker:
|
||||
continue
|
||||
st(f"SAM 3: tracking subject {sj}, shot {si_ + 1}/{len(shots)} (frames {s0}-{s1 - 1})",
|
||||
0.3 + 0.4 * (subjects.index(sj) + si_ / max(1, len(shots))) / len(subjects))
|
||||
if not sa:
|
||||
seg_s0 = max(s0, min_valid_frame)
|
||||
windows = list(anchor_windows(sa, seg_s0, s1))
|
||||
if not windows:
|
||||
for i in range(s0, s1):
|
||||
out[i] = torch.zeros(h, w)
|
||||
rp[i] = "before first pick" if i < first_pick else "lost (no anchor in this shot)"
|
||||
continue
|
||||
seg_s0 = max(s0, min_valid_frame)
|
||||
seg = image[seg_s0:s1, ..., :3]
|
||||
tr = {a: (BFSTrackFromFrame().track(seg, sam3_model, own[sj][a][None], a - seg_s0)[0] if s1 - seg_s0 > 1
|
||||
else own[sj][a][None]) for a in sa if a >= seg_s0}
|
||||
for i in range(s0, seg_s0):
|
||||
out[i] = torch.zeros(h, w); rp[i] = "before first pick"
|
||||
for i in range(seg_s0, s1):
|
||||
if i in sa:
|
||||
out[i] = own[sj][i]; rp[i] = "picked" if i in user_anchors else "Molmo2 + SAM"; continue
|
||||
a = min(tr, key=lambda k: (abs(k - i), k))
|
||||
out[i] = (tr[a][i - seg_s0] > 0.5).float()
|
||||
rp[i] = f"tracked from {a}" if bool(out[i].any()) else (
|
||||
"before first pick" if i < first_pick else f"lost (tracked from {a})")
|
||||
for a, lo, hi in windows:
|
||||
seg = image[lo:hi, ..., :3]
|
||||
tr = (BFSTrackFromFrame().track(seg, sam3_model, own[sj][a][None], a - lo)[0]
|
||||
if hi - lo > 1 else own[sj][a][None])
|
||||
for i in range(lo, hi):
|
||||
if i in sa:
|
||||
out[i] = own[sj][i]; rp[i] = "picked" if i in user_anchors else "Molmo2 + SAM"; continue
|
||||
out[i] = (tr[i - lo] > 0.5).float()
|
||||
rp[i] = f"tracked from {a}" if bool(out[i].any()) else (
|
||||
"before first pick" if i < first_pick else f"lost (tracked from {a})")
|
||||
if first_pick > seg_s0 and seg_s0 <= first_pick < s1:
|
||||
for i in range(first_pick - 1, seg_s0 - 1, -1):
|
||||
ok_bwd = (
|
||||
@@ -955,13 +1144,13 @@ class BFSVideoSegmentPicker:
|
||||
rp[i] = "before first pick" if i < first_pick else rp[i].replace("tracked from", "lost (tracked from") + (")" if not rp[i].endswith(")") else "")
|
||||
if sa:
|
||||
_remember(_SHOT_CACHE, skey, (torch.stack([out[i] > 0.5 for i in range(s0, s1)]),
|
||||
[rp[i] for i in range(s0, s1)]), 64)
|
||||
[rp[i] for i in range(s0, s1)]), 64, _PICKER_MASK_CACHE_MAX_BYTES)
|
||||
if kept:
|
||||
st(f"subject {sj}: {kept}/{len(shots)} shots unchanged, reused", None, log=True)
|
||||
return out, rp
|
||||
|
||||
out, rp = track_shots(anchors)
|
||||
if pts and propagation_method == "Molmo2 (classic)":
|
||||
if pts and propagation_method in ("Molmo2 (classic)", "MolmoPoint-8B"):
|
||||
# Classic behavior: Molmo points audit the full SAM track; two nearby misses indicate a lost stretch.
|
||||
off = set()
|
||||
for t_, _o, x_, y_ in pts:
|
||||
@@ -994,7 +1183,7 @@ class BFSVideoSegmentPicker:
|
||||
if added:
|
||||
st(f"subject {sj}: {added} lost stretches re-anchored by Molmo2 (classic mode), tracking again", None, log=True)
|
||||
out, rp = track_shots(anchors)
|
||||
if pts and propagation_method != "Molmo2 (classic)": # current mode recovery: verify Molmo points against the selected person's appearance
|
||||
if pts and propagation_method not in ("Molmo2 (classic)", "MolmoPoint-8B"): # current mode recovery
|
||||
ref_crop = person_crop(image[first_pick, ..., :3], own[sj][first_pick])
|
||||
off = set()
|
||||
for t_, _o, x_, y_ in pts:
|
||||
@@ -1046,24 +1235,11 @@ class BFSVideoSegmentPicker:
|
||||
for i in range(n):
|
||||
m[i] = out[i]; rep[i] = {"frame": i, "source": rp[i]}
|
||||
if propagate_scope == "from active frame onward" and previous is not None:
|
||||
m[:fix_frame] = previous[0][:fix_frame]
|
||||
m[:fix_frame] = previous[0][:fix_frame].to(m.dtype)
|
||||
rep[:fix_frame] = [dict(r) for r in previous[1][:fix_frame]]
|
||||
if propagate_scope == "fill only empty masks" and previous is not None:
|
||||
keep = previous[0] > 0.5
|
||||
m[keep] = previous[0][keep]
|
||||
for i in range(n):
|
||||
if bool(keep[i].any()):
|
||||
rep[i] = dict(previous[1][i])
|
||||
per_subject[sj] = m; report_s[sj] = rep
|
||||
_TRACK_CACHE[ckey] = (m.clone(), [dict(r) for r in rep])
|
||||
if not segment_only:
|
||||
_LAST_TRACK_CACHE[(vkey, sj)] = (m.clone(), [dict(r) for r in rep], mine)
|
||||
_LAST_TRACK_CACHE.move_to_end((vkey, sj))
|
||||
while len(_LAST_TRACK_CACHE) > 24:
|
||||
_LAST_TRACK_CACHE.popitem(last=False)
|
||||
while len(_TRACK_CACHE) > 24:
|
||||
_TRACK_CACHE.popitem(last=False)
|
||||
masks = torch.stack(list(per_subject.values())).amax(0)
|
||||
per_subject[sj] = m > 0.5; report_s[sj] = rep
|
||||
_cache_track_result(ckey, vkey, sj, m, rep, mine, keep_last=not segment_only)
|
||||
masks = torch.stack(list(per_subject.values())).amax(0).float()
|
||||
anchors = sorted({i for sj in subjects for i in strip if bool(own[sj][i].any())})
|
||||
report = [{"frame": i, "subjects": {str(sj): report_s[sj][i]["source"] for sj in subjects}} for i in range(n)]
|
||||
if not anchors:
|
||||
@@ -1071,9 +1247,17 @@ class BFSVideoSegmentPicker:
|
||||
color = torch.tensor(sp.parse_color(mark_color), dtype=image.dtype) / 255.0
|
||||
a = (masks.to(image.dtype) * float(mark_opacity))[..., None]
|
||||
marked = image[..., :3] * (1 - a) + color * a
|
||||
base = (hybrid_base if hybrid_base is not None else image)[..., :3].clone()
|
||||
if mark_style == PROTECTED_NORMAL_STYLE and edge_feather > 0:
|
||||
# The marked output should show the same soft boundary as the H3 guide.
|
||||
feather_px = max(1, round(min(h, w) * float(edge_feather)))
|
||||
for i in range(n):
|
||||
alpha = feather_inside_mask(masks[i] > 0.5, feather_px).to(image.dtype)
|
||||
alpha = (alpha * float(mark_opacity))[..., None]
|
||||
marked[i] = image[i, ..., :3] * (1 - alpha) + color * alpha
|
||||
base_source = (hybrid_base if hybrid_base is not None else image)[..., :3]
|
||||
base = base_source.clone()
|
||||
if mark_style.startswith("openpose") and pose_keypoint is None and segment_only:
|
||||
st("picking: DWPose runs on Propagate, the hybrid shows the marks then", None, log=True)
|
||||
st("picking: DWPose runs on Apply fix, then the hybrid shows the marks", None, log=True)
|
||||
elif mark_style.startswith("openpose"):
|
||||
if pose_keypoint is None:
|
||||
raise ValueError("mark_style 'openpose + face' needs the pose_keypoint input (DWPose Estimator on the same frames).")
|
||||
@@ -1138,26 +1322,69 @@ class BFSVideoSegmentPicker:
|
||||
drawn = cv.amax(-1) > 0
|
||||
base[i][drawn] = cv[drawn] # the skeleton may leave the region: the body is free
|
||||
needs_normals = "needs normals" in mark_style and not mark_style.startswith("openpose")
|
||||
if mark_style == PROTECTED_NORMAL_STYLE and normals is None and not segment_only:
|
||||
raise ValueError("'normal map inside mask' needs the normals input from MoGe.")
|
||||
if not mark_style.startswith("openpose") and (normals is not None or not needs_normals):
|
||||
import torch.nn.functional as F
|
||||
if normals is not None:
|
||||
protected = None
|
||||
if mark_style == PROTECTED_NORMAL_STYLE and protect_mask is not None:
|
||||
protected = protect_mask.detach().float().cpu()
|
||||
if protected.ndim == 2:
|
||||
protected = protected[None]
|
||||
if protected.ndim != 3 or protected.shape[0] not in (1, n):
|
||||
raise ValueError("protect_mask must be one mask or one mask per guide frame.")
|
||||
if protected.shape[1:] != (h, w):
|
||||
protected = F.interpolate(protected[:, None], size=(h, w), mode="nearest")[:, 0]
|
||||
if protected.shape[0] == 1:
|
||||
protected = protected.expand(n, -1, -1)
|
||||
protected = protected > 0.5
|
||||
if mark_style == PROTECTED_NORMAL_STYLE and locked is not None:
|
||||
protected = locked if protected is None else (protected | locked)
|
||||
if needs_normals and normals is not None:
|
||||
if normals.shape[0] != n:
|
||||
raise ValueError(f"{normals.shape[0]} normal maps for {n} frames: run MoGe on the same frames.")
|
||||
nm = normals[..., :3].float()
|
||||
if nm.shape[1:3] != (h, w):
|
||||
nm = F.interpolate(nm.movedim(-1, 1), size=(h, w), mode="bilinear", align_corners=False).movedim(1, -1)
|
||||
nm = to_opencv_normals(nm, normals_convention)
|
||||
elif needs_normals:
|
||||
nm = None
|
||||
else:
|
||||
nm = torch.full((n, h, w, 3), 0.5)
|
||||
nm = None
|
||||
for sj, m in per_subject.items(): # subject j: the j-th color (= Picture j)
|
||||
hm = grow_masks((m > 0.5).float(), float(hybrid_grow)) > 0.5
|
||||
selected = m > 0.5
|
||||
if mark_style in ("translucent overlay", PROTECTED_NORMAL_STYLE):
|
||||
hm = selected
|
||||
elif mark_style == "colored outline":
|
||||
# Draw inside the selected person only. Keep the scene, held props and
|
||||
# occluding objects visible instead of painting over the whole mask.
|
||||
radius = max(1, round(min(h, w) * 0.005))
|
||||
eroded = 1 - F.max_pool2d((~selected)[:, None].float(), 2 * radius + 1, 1, radius)[:, 0]
|
||||
hm = selected & (eroded < 0.5)
|
||||
else:
|
||||
hm = grow_masks(selected.float(), float(hybrid_grow)) > 0.5
|
||||
col = HYBRID_COLORS[(int(sj) - 1) % len(HYBRID_COLORS)]
|
||||
for i in range(n):
|
||||
if bool(hm[i].any()):
|
||||
fill = mark_fill(mark_style, image[i, ..., :3].float(), nm[i], hm[i], col)
|
||||
base[i][hm[i]] = fill.to(base.dtype)[hm[i]]
|
||||
fill = mark_fill(mark_style, image[i, ..., :3].float(), nm[i] if nm is not None else None, hm[i], col)
|
||||
if mark_style == PROTECTED_NORMAL_STYLE:
|
||||
import cv2
|
||||
import numpy as np
|
||||
target = hm[i]
|
||||
if protected is not None:
|
||||
keep = cv2.dilate(protected[i].numpy().astype(np.uint8),
|
||||
np.ones((5, 5), np.uint8)) > 0
|
||||
target = target & ~torch.from_numpy(keep)
|
||||
if bool(target.any()):
|
||||
feather_px = round(min(h, w) * max(0.0, float(edge_feather)))
|
||||
alpha = feather_inside_mask(target, feather_px).to(base.dtype)
|
||||
alpha = alpha[..., None]
|
||||
base[i] = base[i] * (1 - alpha) + fill.to(base.dtype) * alpha
|
||||
else:
|
||||
base[i][hm[i]] = fill.to(base.dtype)[hm[i]]
|
||||
# Apply the same opacity control to the guide sent to H3. Blending against the
|
||||
# original hybrid base preserves stacked pickers and fades both fills and DWPose marks.
|
||||
opacity = min(1.0, max(0.0, float(mark_opacity)))
|
||||
if opacity < 1.0:
|
||||
base = base_source + (base - base_source) * opacity
|
||||
lost_runs = [] # stretches where a picked subject has no mask after its first appearance: the panel offers them to add to the strip
|
||||
for sj in subjects:
|
||||
if not any(r["source"] == "picked" for r in report_s[sj]):
|
||||
@@ -1171,7 +1398,10 @@ class BFSVideoSegmentPicker:
|
||||
elif run is not None: lost_runs.append({"subject": sj, "start": run[0], "end": run[1]}); run = None
|
||||
if run is not None: lost_runs.append({"subject": sj, "start": run[0], "end": run[1]})
|
||||
ui = {"bfs_vsegpick": [{"frames": panels, "indices": strip, "report": [report[i] for i in strip], "subjects": subjects,
|
||||
"count": n, "lost": lost_runs, "timeline": (st("building the timeline previews", 0.9) or timeline_strip(image, per_subject))}]}
|
||||
"protect_anchors": lock_anchors,
|
||||
"protected_frames": [bool(locked[i].any()) if locked is not None else False for i in strip],
|
||||
"count": n, "lost": lost_runs,
|
||||
"timeline": (st("building the timeline previews", 0.9) or timeline_strip(image, per_subject, locked))}]}
|
||||
lost_frames = sum(r["end"] - r["start"] + 1 for r in lost_runs)
|
||||
st(f"done: {len(subjects)} subject(s), {lost_frames} frames without a mask", 1.0, log=True)
|
||||
return {"ui": ui, "result": (masks, marked, torch.cat(overlays), json.dumps({"frames": report, "anchors": anchors, "subjects": subjects}), base)}
|
||||
|
||||
@@ -420,3 +420,7 @@ NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
'BFSMiniMaxH3GuideTarget': 'MiniMax-H3 Guide Target — Image / Video (BFS)',
|
||||
'BFSMiniMaxH3IdentityReference': 'MiniMax-H3 Identity Reference + RoPE (BFS)',
|
||||
}
|
||||
|
||||
from .bfs_h3_image_guide import BFSH3ImageAddGuide
|
||||
NODE_CLASS_MAPPINGS['BFSH3ImageAddGuide'] = BFSH3ImageAddGuide
|
||||
NODE_DISPLAY_NAME_MAPPINGS['BFSH3ImageAddGuide'] = 'Add Guide for MiniMax H3 Image (BFS)'
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
[project]
|
||||
name = "bfsnodes"
|
||||
description = "ComfyUI nodes for video workflows: SAM 3 segmentation and Molmo2 tracking; MiniMax H3 character swapping and sampling; shot planning and looping; LTX video controls and identity transfer; model loaders, LoRA tools, and image/video utilities."
|
||||
version = "1.119.5"
|
||||
version = "1.120.0"
|
||||
license = {file = "LICENSE"}
|
||||
dependencies = [
|
||||
"torch",
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
import importlib.util
|
||||
import sys
|
||||
import os
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
import torch
|
||||
if os.environ.get('COMFYUI_PATH'):
|
||||
sys.path.insert(0, os.environ['COMFYUI_PATH'])
|
||||
from comfy.ldm.minimax.model import PackedLayout
|
||||
spec = importlib.util.spec_from_file_location('image_guide', Path(__file__).resolve().parents[1] / 'bfs_h3_image_guide.py')
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
class ImageGuideTest(unittest.TestCase):
|
||||
def test_only_guide_positions_are_aligned_and_inputs_stay_unchanged(self):
|
||||
refs = [{'kind': 'image', 'latent_h': 8, 'latent_w': 8},
|
||||
{'kind': 'image', 'latent_h': 8, 'latent_w': 8, module.MARKER: True}]
|
||||
original = PackedLayout(4, 1, 8, 8, 0, refs=refs)
|
||||
positions = original.position_ids.clone()
|
||||
result = module.align_guide_layout(original, refs)
|
||||
ref_segments = [(a,b) for a,b,k in result.segments if k == 'ref_img']
|
||||
a,b = ref_segments[1]
|
||||
ta,tb = next((a,b) for a,b,k in result.segments if k == 'video')
|
||||
self.assertTrue(torch.equal(result.position_ids[a:b], result.position_ids[ta:tb]))
|
||||
a,b = ref_segments[0]
|
||||
self.assertTrue(torch.equal(result.position_ids[a:b], positions[a:b]))
|
||||
self.assertTrue(torch.equal(original.position_ids, positions))
|
||||
self.assertTrue(torch.equal(result.img_update, original.img_update))
|
||||
|
||||
def test_incompatible_guide_grid_is_rejected(self):
|
||||
refs = [{'kind': 'image', 'latent_h': 4, 'latent_w': 4, module.MARKER: True}]
|
||||
layout = PackedLayout(4, 1, 8, 8, 0, refs=refs)
|
||||
with self.assertRaisesRegex(ValueError, 'match the target'):
|
||||
module.align_guide_layout(layout, refs)
|
||||
@@ -0,0 +1,60 @@
|
||||
import importlib.util
|
||||
import sys
|
||||
import os
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
import torch
|
||||
|
||||
if os.environ.get('COMFYUI_PATH'):
|
||||
sys.path.insert(0, os.environ['COMFYUI_PATH'])
|
||||
spec = importlib.util.spec_from_file_location('image_swap', Path(__file__).resolve().parents[1] / 'bfs_h3_image_swap_conditioning.py')
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
|
||||
class Clip:
|
||||
def tokenize(self, prompt, **kwargs):
|
||||
self.prompt, self.images = prompt, kwargs['images']
|
||||
return {'text': prompt}
|
||||
|
||||
def encode_from_tokens_scheduled(self, tokens):
|
||||
return [[torch.zeros(1, 4, 8), {}]]
|
||||
|
||||
|
||||
class Vae:
|
||||
def encode(self, pixels):
|
||||
return torch.zeros(1, 24, 1, pixels.shape[1] // 16, pixels.shape[2] // 16)
|
||||
|
||||
|
||||
class ImageSwapTest(unittest.TestCase):
|
||||
def test_modes_use_still_image_language_and_shifted_picture_numbers(self):
|
||||
for mode in module.MODES:
|
||||
text = module.image_swap_prompt(3, mode)
|
||||
for word in ('video', 'motion', 'frame', 'timing'):
|
||||
self.assertNotIn(word, text.lower())
|
||||
for j, color in enumerate(module.COLORS, 1):
|
||||
self.assertIn(f'<Subject {j}> is the subject marked by the {color}', text)
|
||||
self.assertIn(f'appearance comes from <Picture {j+1}>', text)
|
||||
self.assertIn('foreground occluders', text)
|
||||
|
||||
def test_native_conditioning_order_and_image_latent(self):
|
||||
clip = Clip()
|
||||
guide = torch.full((1, 32, 32, 3), .1)
|
||||
red = torch.full_like(guide, .2)
|
||||
blue = torch.full_like(guide, .3)
|
||||
result = module.BFSH3ImageCharacterSwapConditioning.execute(
|
||||
clip, Vae(), guide, {'reference_2': blue, 'reference_0': red}, width=128, height=96)
|
||||
positive, latent, prompt = result.args
|
||||
self.assertEqual(tuple(latent['samples'].shape), (1, 24, 1, 6, 8))
|
||||
self.assertEqual(len(positive[0][1]['minimax_refs']), 3)
|
||||
self.assertEqual([round(float(i.mean()), 1) for i in clip.images], [.1, .2, .3])
|
||||
self.assertIn('red subject replaced by <Picture 2>', prompt)
|
||||
self.assertIn('blue subject replaced by <Picture 3>', prompt)
|
||||
|
||||
def test_override_preserved(self):
|
||||
clip = Clip()
|
||||
image = torch.zeros(1, 32, 32, 3)
|
||||
result = module.BFSH3ImageCharacterSwapConditioning.execute(
|
||||
clip, Vae(), image, {'reference_0': image}, width=64, height=64, prompt_override='Custom edit')
|
||||
self.assertEqual(result.args[2], 'Custom edit')
|
||||
self.assertEqual(clip.prompt, 'Custom edit')
|
||||
@@ -11,7 +11,7 @@ NODE = SC.BFSH3KeyframeConditioning
|
||||
|
||||
class InputsTest(unittest.TestCase):
|
||||
def test_modes(self):
|
||||
self.assertEqual(SC.MODES, ["keyframe pins", "normal map guide", "DWPose guide"])
|
||||
self.assertEqual(SC.MODES, ["keyframe pins", "normal map guide", "DWPose guide", "translucent overlay guide", "outline guide"])
|
||||
|
||||
def test_pictures_grouped_first(self):
|
||||
t = NODE.INPUT_TYPES()
|
||||
@@ -46,6 +46,27 @@ class PromptTest(unittest.TestCase):
|
||||
self.assertIn("<Picture 2>", p)
|
||||
self.assertIn("1 keyframe of the target video is pinned at 2.0s", SC.pins_caption(1, [2.0]))
|
||||
|
||||
def test_overlay_prompt(self):
|
||||
prompt = SC.overlay_caption(2)
|
||||
self.assertIn("translucent red overlay", prompt)
|
||||
self.assertIn("translucent blue overlay", prompt)
|
||||
self.assertIn("original appearance visible beneath it is not the target appearance", prompt)
|
||||
self.assertIn("The tint must disappear in the result", prompt)
|
||||
self.assertNotIn("<Picture 3>", prompt)
|
||||
|
||||
def test_outline_prompt_preserves_props(self):
|
||||
prompt = SC.outline_caption(2)
|
||||
self.assertIn("red outline", prompt)
|
||||
self.assertIn("blue outline", prompt)
|
||||
self.assertIn("held or interacting objects", prompt)
|
||||
self.assertIn("foreground occlusions", prompt)
|
||||
self.assertNotIn("<Picture 3>", prompt)
|
||||
|
||||
def test_normal_map_prompt_preserves_occluders(self):
|
||||
prompt = SC.normals_caption(1)
|
||||
self.assertIn("held objects and foreground occluders", prompt)
|
||||
self.assertIn("original positions, shapes and colors", prompt)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
import types
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
|
||||
@@ -9,6 +12,8 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
SPEC = importlib.util.spec_from_file_location("bfs_segment_picker", ROOT / "bfs_segment_picker.py")
|
||||
SP = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(SP)
|
||||
ORIGINAL_DETECT_TEXT = SP.detect_text
|
||||
ORIGINAL_DETECT_POINTS = SP.detect_points
|
||||
|
||||
|
||||
def rect(h, w, x0, y0, x1, y1):
|
||||
@@ -77,6 +82,49 @@ class GeometryTest(unittest.TestCase):
|
||||
self.assertEqual(SP.parse_color("nope"), (255, 0, 255))
|
||||
|
||||
|
||||
class InferenceModeTest(unittest.TestCase):
|
||||
def test_text_and_point_sam_calls_run_in_inference_mode(self):
|
||||
calls = []
|
||||
|
||||
class FakeDetect:
|
||||
@staticmethod
|
||||
def execute(**kwargs):
|
||||
calls.append(torch.is_inference_mode_enabled())
|
||||
return types.SimpleNamespace(args=(torch.ones(1, 4, 4), [[{"score": 0.9}]]))
|
||||
|
||||
parent = types.ModuleType("comfy_extras")
|
||||
parent.__path__ = []
|
||||
child = types.ModuleType("comfy_extras.nodes_sam3")
|
||||
child.SAM3_Detect = FakeDetect
|
||||
frame = torch.zeros(1, 4, 4, 3)
|
||||
with patch.dict(sys.modules, {"comfy_extras": parent, "comfy_extras.nodes_sam3": child}), \
|
||||
patch.object(SP, "_text_cond", return_value=[]):
|
||||
self.assertEqual(len(ORIGINAL_DETECT_TEXT((None, None), frame, "person", 0.4, 1)), 1)
|
||||
self.assertTrue(ORIGINAL_DETECT_POINTS((None, None), frame, [{"x": 0.5, "y": 0.5, "label": 1}], 0.4).any())
|
||||
self.assertEqual(calls, [True, True])
|
||||
|
||||
|
||||
class AutoObjectsTest(unittest.TestCase):
|
||||
def test_rejects_scene_sized_generic_object_but_keeps_person(self):
|
||||
h, w = 20, 40
|
||||
scene = torch.zeros(h, w, dtype=torch.bool)
|
||||
scene[::2, :] = True
|
||||
scene[-1, :] = True
|
||||
person = rect(h, w, 10, 2, 28, 18)
|
||||
|
||||
def detect_text(sam, frame, concept, threshold, max_detections):
|
||||
mask = person if concept == "person" else scene
|
||||
return [(mask, {"score": .9})]
|
||||
|
||||
frame = torch.zeros(1, h, w, 3)
|
||||
with patch.object(SP, "detect_text", side_effect=detect_text):
|
||||
objects = SP.auto_objects((None, None), frame, ["objects", "person"], .4, 8, 0.)
|
||||
with patch.object(SP, "detect_text", return_value=[(scene, {"score": .9})]):
|
||||
large_person = SP.auto_objects((None, None), frame, ["person"], .4, 8, 0.)
|
||||
self.assertEqual([(o["id"], o["label"]) for o in objects], [("a1", "person")])
|
||||
self.assertEqual(len(large_person), 1)
|
||||
|
||||
|
||||
class SessionTest(unittest.TestCase):
|
||||
"""The node with SAM replaced: what is segmented once, what is segmented again, what the outputs hold."""
|
||||
|
||||
|
||||
@@ -3,6 +3,9 @@ import sys
|
||||
import types
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
pkg = types.ModuleType("bfsnodes")
|
||||
@@ -12,6 +15,59 @@ sys.modules["bfsnodes"] = pkg
|
||||
VSP = importlib.import_module("bfsnodes.bfs_video_segment_picker")
|
||||
|
||||
|
||||
class MaskQualityTest(unittest.TestCase):
|
||||
def test_rendered_edge_is_soft_and_stays_inside_selection(self):
|
||||
mask = torch.zeros(20, 20, dtype=torch.bool)
|
||||
mask[3:17, 3:17] = True
|
||||
alpha = VSP.feather_inside_mask(mask, 4)
|
||||
self.assertEqual(float(alpha[2, 10]), 0.0)
|
||||
self.assertGreater(float(alpha[10, 10]), 0.95)
|
||||
self.assertGreater(float(alpha[3, 10]), 0.0)
|
||||
self.assertLess(float(alpha[3, 10]), 0.5)
|
||||
|
||||
def test_anchor_windows_cover_each_frame_once_and_keep_earlier_ties(self):
|
||||
windows = list(VSP.anchor_windows([0, 53, 69, 125], 0, 209))
|
||||
assigned = {i: a for a, lo, hi in windows for i in range(lo, hi)}
|
||||
self.assertEqual(len(assigned), 209)
|
||||
self.assertEqual(assigned[26], 0)
|
||||
self.assertEqual(assigned[27], 53)
|
||||
self.assertEqual(assigned[61], 53)
|
||||
self.assertEqual(assigned[62], 69)
|
||||
|
||||
def test_fill_empty_tracks_only_gaps_and_preserves_existing_masks(self):
|
||||
frames = torch.zeros(8, 4, 4, 3)
|
||||
masks = torch.zeros(8, 4, 4, dtype=torch.bool)
|
||||
masks[[0, 3, 7], 1, 1] = True
|
||||
report = [{"frame": i, "source": "empty"} for i in range(8)]
|
||||
calls = []
|
||||
|
||||
class FakeTracker:
|
||||
def track(self, video, model, mask, anchor_frame):
|
||||
calls.append((len(video), anchor_frame))
|
||||
out = torch.zeros(len(video), 4, 4)
|
||||
out[:, 0, 0] = 1
|
||||
return out, anchor_frame
|
||||
|
||||
with patch.object(VSP, "BFSTrackFromFrame", FakeTracker):
|
||||
filled = VSP.fill_empty_mask_runs(frames, None, masks, report, [(0, 8)], active_frame=3)
|
||||
self.assertEqual(filled, 5)
|
||||
self.assertEqual(calls, [(3, 2), (4, 0)])
|
||||
self.assertTrue(bool(masks[3, 1, 1]))
|
||||
self.assertFalse(bool(masks[3, 0, 0]))
|
||||
self.assertTrue(bool(masks[5, 0, 0]))
|
||||
|
||||
def test_fill_scope_does_not_request_molmo_when_masks_are_cached(self):
|
||||
selection = '{"frames":{"1":{"1":{"selected":["a1"]}}}}'
|
||||
prompt = {"7": {"inputs": {"molmo_clip": ["6", 0]}}}
|
||||
with patch.object(VSP, "video_key", return_value="video"), \
|
||||
patch.object(VSP, "_LAST_TRACK_CACHE", {("video", 1): (None, None, None)}):
|
||||
need = VSP.BFSVideoSegmentPicker().check_lazy_status(
|
||||
selection=selection, mark_style="flat color", track=True, image=torch.zeros(2, 4, 4, 3),
|
||||
molmo_clip=None, propagation_method="Molmo2 (classic)",
|
||||
propagate_scope="fill only empty masks", unique_id="7", prompt=prompt)
|
||||
self.assertEqual(need, [])
|
||||
|
||||
|
||||
class AutoUnmarkingTest(unittest.TestCase):
|
||||
def test_find_connected_picture_count(self):
|
||||
# 1. 2 pictures connected downstream
|
||||
@@ -58,7 +114,18 @@ class AutoUnmarkingTest(unittest.TestCase):
|
||||
|
||||
self.assertIn("molmo_descriptions", opt)
|
||||
self.assertEqual(opt["molmo_descriptions"][0], "STRING")
|
||||
self.assertTrue(opt["molmo_descriptions"][1].get("multiline"))
|
||||
self.assertFalse(opt["molmo_descriptions"][1].get("multiline"))
|
||||
self.assertEqual(opt["protect_mask"][0], "MASK")
|
||||
self.assertIn(VSP.PROTECTED_NORMAL_STYLE, opt["mark_style"][0])
|
||||
|
||||
def test_locked_objects_survive_selection_round_trip(self):
|
||||
import json
|
||||
state = {"frames": {"7": {"1": {"selected": ["a1"]},
|
||||
"lock": {"selected": ["lock_p1"], "added": [
|
||||
{"id": "lock_p1", "points": [{"x": 0.4, "y": 0.5, "label": 1}]}]}}}}
|
||||
frames = VSP.parse_frames(json.dumps(state))
|
||||
self.assertEqual(frames["7"]["1"]["selected"], ["a1"])
|
||||
self.assertEqual(frames["7"]["lock"]["selected"], ["lock_p1"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -103,6 +103,7 @@ function loadMask(o, color) {
|
||||
|
||||
export function Panel(io, options = {}) { // exported: BFS Video Segment Picker shows one per frame
|
||||
const showEraser = options.showEraser !== false;
|
||||
const showRemove = options.showRemove !== false;
|
||||
const state = reactive(parseState(io.get()));
|
||||
const data = ref(null); // {session, width, height, image, objects} from the last run
|
||||
const loaded = new Map(); // object id -> loadMask result (not reactive: canvases and big arrays)
|
||||
@@ -119,6 +120,7 @@ export function Panel(io, options = {}) { // exported: BFS Video Segment Picke
|
||||
const fit = reactive({ w: 0, h: 0 });
|
||||
|
||||
const save = () => io.set(JSON.stringify({ selected: state.selected, added: state.added, next: state.next }));
|
||||
const entryId = kind => `${options.entryPrefix || ""}${kind}${state.next}`;
|
||||
const objects = computed(() => data.value?.objects || []);
|
||||
const colorOf = id => {
|
||||
const i = Math.max(0, objects.value.findIndex(o => o.id === id));
|
||||
@@ -138,7 +140,7 @@ export function Panel(io, options = {}) { // exported: BFS Video Segment Picke
|
||||
await Promise.all(objs.map(async (o, i) => {
|
||||
const key = `${o.id}|${o.mask.filename}`;
|
||||
if (loaded.get(o.id)?.key === key) return;
|
||||
try { loaded.set(o.id, { key, ...(await loadMask(o, o.remove ? CUT : PALETTE[i % PALETTE.length])) }); } catch { /* shown as missing */ }
|
||||
try { loaded.set(o.id, { key, ...(await loadMask(o, o.remove ? CUT : (options.maskColor || PALETTE[i % PALETTE.length]))) }); } catch { /* shown as missing */ }
|
||||
}));
|
||||
ready.value++;
|
||||
}
|
||||
@@ -222,7 +224,9 @@ export function Panel(io, options = {}) { // exported: BFS Video Segment Picke
|
||||
return;
|
||||
}
|
||||
const o = objectAt(p);
|
||||
if (o) toggle(o.id);
|
||||
if (options.autoPointOnClick && (!o || !o.entry)) {
|
||||
add({ id: entryId("p"), points: [{ x: +p.x.toFixed(4), y: +p.y.toFixed(4), label: 1 }] });
|
||||
} else if (o) toggle(o.id);
|
||||
}
|
||||
function onMove(e) {
|
||||
if (data.value && brushing()) {
|
||||
@@ -247,7 +251,7 @@ export function Panel(io, options = {}) { // exported: BFS Video Segment Picke
|
||||
if (!r.ok || j.error) throw new Error(j.error || r.statusText);
|
||||
if (!j.objects.length) { error.value = entry.text ? `Nothing found for “${entry.text}”: try other words or a lower threshold.` : "The points gave an empty mask."; return; }
|
||||
state.added = [...state.added.filter(a => a.id !== entry.id), entry];
|
||||
state.next = Math.max(state.next, parseInt(entry.id.slice(1)) + 1);
|
||||
state.next = Math.max(state.next, Number(entry.id.match(/(\d+)$/)?.[1] || 0) + 1);
|
||||
data.value = { ...data.value, objects: [...data.value.objects.filter(o => o.entry !== entry.id), ...j.objects] };
|
||||
state.selected = [...state.selected, ...j.objects.map(o => o.id).filter(id => !isSel(id))]; // what you add is what you want
|
||||
save();
|
||||
@@ -256,16 +260,16 @@ export function Panel(io, options = {}) { // exported: BFS Video Segment Picke
|
||||
error.value = String(err.message || err);
|
||||
} finally { busy.value = ""; }
|
||||
}
|
||||
const addText = () => { const t = text.value.trim(); if (t) { add({ id: `t${state.next}`, text: t }); text.value = ""; } };
|
||||
const addText = () => { const t = text.value.trim(); if (t) { add({ id: entryId("t"), text: t }); text.value = ""; } };
|
||||
const addPoints = async () => {
|
||||
if (!points.value.some(p => p.label)) return;
|
||||
await add(cutting() ? { id: `r${state.next}`, remove: true, points: points.value } : { id: `p${state.next}`, points: points.value });
|
||||
await add(cutting() ? { id: entryId("r"), remove: true, points: points.value } : { id: entryId("p"), points: points.value });
|
||||
if (!error.value) setMode("select");
|
||||
};
|
||||
const addBrush = async () => {
|
||||
if (!strokes.value.length) return;
|
||||
const stroke = { image: data.value.image_key, strokes: strokes.value };
|
||||
await add(cutting() ? { id: `e${state.next}`, remove: true, ...stroke } : { id: `b${state.next}`, ...stroke });
|
||||
await add(cutting() ? { id: entryId("e"), remove: true, ...stroke } : { id: entryId("b"), ...stroke });
|
||||
if (!error.value) setMode("select");
|
||||
};
|
||||
function removeEntry(entryId) {
|
||||
@@ -328,8 +332,8 @@ export function Panel(io, options = {}) { // exported: BFS Video Segment Picke
|
||||
title: "Place points on something the automatic pass missed" }, "📍 Points"),
|
||||
h("button", { class: mode.value === "brush" ? "on" : "", disabled: !d, onClick: () => setMode("brush"),
|
||||
title: "Paint extra area into the mark by hand (e.g. hair SAM left out)" }, "🖌 Brush"),
|
||||
h("button", { class: mode.value === "remove" ? "on" : "", disabled: !d, onClick: () => setMode("remove"),
|
||||
title: "Place points on something to cut out of the mark (SAM segments it)" }, "➖ Remove"),
|
||||
showRemove ? h("button", { class: mode.value === "remove" ? "on" : "", disabled: !d, onClick: () => setMode("remove"),
|
||||
title: "Place points on something to cut out of the mark (SAM segments it)" }, "➖ Remove") : null,
|
||||
showEraser ? h("button", { class: mode.value === "eraser" ? "on" : "", disabled: !d, onClick: () => setMode("eraser"),
|
||||
title: "Paint an area to subtract manually from the selected mask" }, "🧽 Eraser") : null,
|
||||
h("span", { class: "hint" }, d ? `${objs.length} objects · ${state.selected.length} selected` : ""),
|
||||
|
||||
@@ -510,6 +510,11 @@ app.registerExtension({
|
||||
const r = created?.apply(this, arguments);
|
||||
styles();
|
||||
const host = el("div", { style: "width:100%;height:100%;min-height:320px" });
|
||||
// Keep clicks/drags inside the interactive player from reaching LiteGraph. Otherwise a
|
||||
// pointer event on a video/control can be interpreted as a canvas gesture and the node
|
||||
// may recalculate its DOM-widget height, making the compare panel collapse after use.
|
||||
for (const type of ["pointerdown", "pointerup", "pointermove", "click", "dblclick", "contextmenu"])
|
||||
host.addEventListener(type, e => e.stopPropagation());
|
||||
this._bfsCompare = buildPlayer(host);
|
||||
wheelToCanvas(host); // the wheel zooms the canvas over the player too
|
||||
this.addDOMWidget("bfs_video_compare", "div", host, { serialize: false, hideOnZoom: false, getMinHeight: () => 320 });
|
||||
|
||||
@@ -20,7 +20,7 @@ function parse(text) { // {frame: {subject: selection}}; a bare selection (old
|
||||
for (const [k, v] of Object.entries(f)) {
|
||||
if (v && (v.selected || v.added)) out[k] = { "1": v };
|
||||
else if (v && typeof v === "object") {
|
||||
out[k] = Object.fromEntries(Object.entries(v).filter(([sk]) => +sk >= 1 && +sk <= SUBJECTS.length));
|
||||
out[k] = Object.fromEntries(Object.entries(v).filter(([sk]) => sk === "lock" || (+sk >= 1 && +sk <= SUBJECTS.length)));
|
||||
} else out[k] = {};
|
||||
}
|
||||
return out;
|
||||
@@ -49,6 +49,7 @@ function styles() {
|
||||
.bvsp .subj .t .clr:hover{background:#f43f5e;color:#fff}
|
||||
.bvsp .subj .t i{width:10px;height:10px;border-radius:50%;display:block}
|
||||
.bvsp .subj .t.on{color:#fff;border-color:currentColor;background:#2b2f37}
|
||||
.bvsp .subj .t.lock.on{color:#fbbf24;background:#44351b}
|
||||
.bvsp .stat{display:flex;align-items:center;gap:8px;font-size:11px;color:#cfd6e0}
|
||||
.bvsp .stat .pb{flex:0 0 110px;height:5px;background:#2a2d34;border-radius:3px;overflow:hidden}
|
||||
.bvsp .stat .pb i{display:block;height:100%}
|
||||
@@ -175,7 +176,8 @@ app.registerExtension({
|
||||
if (JSON.stringify(frames.v[String(i)]?.[sj]?.selected || []) !== before) autoPropagate();
|
||||
},
|
||||
expose: o => Object.assign(handle, o),
|
||||
}, { showEraser: false }) });
|
||||
}, { showEraser: false, showRemove: false, autoPointOnClick: sj === "lock", entryPrefix: sj === "lock" ? "lock_" : "",
|
||||
maskColor: sj === "lock" ? [251, 191, 36] : null }) });
|
||||
panelApp.mount(el);
|
||||
const d = run.data?.frames?.[active.value];
|
||||
if (d) setTimeout(() => handle.executed?.(d), 0);
|
||||
@@ -184,7 +186,9 @@ app.registerExtension({
|
||||
const pickedSubjects = i => Object.entries(frames.v[String(run.data?.indices?.[i] ?? i)] || {}).filter(([, v]) => (v?.selected || []).length).map(([k]) => k);
|
||||
const status = i => {
|
||||
const own = (frames.v[String(run.data?.indices?.[i] ?? i)]?.[subject.value]?.selected || []).length > 0;
|
||||
const r = run.data?.report?.[i]?.subjects?.[subject.value] || run.data?.report?.[i]?.source || "";
|
||||
const r = subject.value === "lock"
|
||||
? (run.data?.protected_frames?.[i] ? "tracked protected" : "")
|
||||
: (run.data?.report?.[i]?.subjects?.[subject.value] || run.data?.report?.[i]?.source || "");
|
||||
if (own) return ["picked", "✓"];
|
||||
if (r.startsWith("tracked")) return ["tracked", "↔"];
|
||||
if (r.startsWith("lost")) return ["lost", "⚠"];
|
||||
@@ -257,7 +261,7 @@ app.registerExtension({
|
||||
setup: () => () => h("div", { class: "bvsp" }, [
|
||||
h("div", { class: "card" }, [
|
||||
h("div", { class: "row" }, [
|
||||
h("div", { class: "subj" }, SUBJECTS.map(([k, col, name]) => h("span", {
|
||||
h("div", { class: "subj" }, [...SUBJECTS.map(([k, col, name]) => h("span", {
|
||||
class: ["t", subject.value === k ? "on" : ""], style: subject.value === k ? { color: col } : null,
|
||||
title: `Subject ${k} (${name}) = Picture ${k}: clicks select this subject`,
|
||||
onClick: () => { if (subject.value !== k) { subject.value = k; mountPanel(); } } },
|
||||
@@ -269,7 +273,17 @@ app.registerExtension({
|
||||
title: `Clear all picks for Subject ${k} (${name})`,
|
||||
onClick: (e) => { e.stopPropagation(); clearSubject(k); }
|
||||
}, "✕") : null,
|
||||
]))),
|
||||
])), h("span", {
|
||||
class: ["t", "lock", subject.value === "lock" ? "on" : ""],
|
||||
title: "Everything outside the selected subject is already protected. Use this padlock only for an object inside that subject's mask; click it to segment with SAM 3.",
|
||||
onClick: () => { if (subject.value !== "lock") { subject.value = "lock"; mountPanel(); } },
|
||||
}, [
|
||||
h("span", null, "🔒"),
|
||||
hasPicks("lock") ? h("span", {
|
||||
class: "clr", title: "Clear all protected-object picks",
|
||||
onClick: e => { e.stopPropagation(); clearSubject("lock"); },
|
||||
}, "✕") : null,
|
||||
])]),
|
||||
h("span", { class: "sp" }),
|
||||
h("select", { value: scope.value, title: "Choose how a new frame selection changes the propagated masks",
|
||||
style: "max-width:190px;background:#23262c;color:#e3e6eb;border:1px solid #3a3e47;border-radius:6px;padding:4px;font:inherit",
|
||||
@@ -288,7 +302,9 @@ app.registerExtension({
|
||||
h("span", { class: "txt", title: live.text }, busy.value && !live.running ? busy.value
|
||||
: `${live.running ? "⏳" : "✓"} ${live.text} · ${live.elapsed}s`),
|
||||
h("span", { class: "tog", onClick: () => { live.open = !live.open; } }, live.open ? "▾ log" : "▸ log"),
|
||||
]) : h("div", { class: "muted" }, "Pick a subject, click a frame below, then click its person. Subject N = Picture N."),
|
||||
]) : h("div", { class: "muted" }, subject.value === "lock"
|
||||
? "🔒 Outside the subject is already protected. Click an object inside the subject mask to keep it unchanged, then Apply fix."
|
||||
: "Pick a subject, click a frame below, then click its person. Subject N = Picture N."),
|
||||
live.open ? h("div", { class: "log" }, live.log.length ? live.log.map((l, i) => h("div", { key: i }, l)) : "no log yet") : null,
|
||||
]),
|
||||
h("div", { class: "card" }, [
|
||||
@@ -306,7 +322,9 @@ app.registerExtension({
|
||||
style: { backgroundImage: `url("${fileUrl(d.image)}")` },
|
||||
onClick: () => { if (active.value !== i) { active.value = i; focusFrame = run.data.indices?.[i] ?? i; save(); mountPanel(); } } },
|
||||
[h("span", { class: "b" }, `f${run.data.indices?.[i] ?? i}`), mark ? h("span", { class: ["s", cls] }, mark) : null,
|
||||
h("span", { class: "dots" }, pickedSubjects(i).map(k => h("i", { style: { background: SUBJECTS[(+k - 1) % SUBJECTS.length][1] } })))]);
|
||||
h("span", { class: "dots" }, pickedSubjects(i).map(k => k === "lock"
|
||||
? h("span", { title: "Protected object picked", style: "color:#fbbf24;background:#181818;border-radius:3px;padding:0 2px" }, "🔒")
|
||||
: h("i", { style: { background: SUBJECTS[(+k - 1) % SUBJECTS.length][1] } })))]);
|
||||
}))
|
||||
: h("div", { class: "empty" }, "Run the node once: the frames appear here. Click a frame, then the person."),
|
||||
]),
|
||||
|
||||
Reference in New Issue
Block a user