Vendor the crop planner so the node stands alone (1.32.0)
The sampler no longer reaches into a sibling pack at runtime: the crop path planner and its helpers now live here as bfs_crop_planner.py and bfs_subject_crop.py, copied from drozbay/MaskVidExperiments (d98cc89, v0.2.0). Both packs are GPL-3.0, which is what makes the copy legal; attribution is in NOTICE and in each file's header. Only the ComfyUI node classes were dropped -- they stay in the original pack, where they belong. tracked/zoomed therefore work out of the box now, and the per-frame paste-back is ours rather than a call into their uncrop node. Also adds the node's help page (docs/BFSHeadSwapMaskedSampler.md, the '?' on the node): what each crop mode does, why the mask is native inpainting rather than a LoRA feature, why latent_mask is the output to check first, how to size the latent to the crop, and the order to switch things on when a run looks wrong. scipy is now a declared dependency (the planner uses ndimage).
This commit is contained in:
alisson-anjos
@@ -0,0 +1,8 @@
|
||||
|
||||
Portions of this pack are vendored from third-party GPL-3.0 projects:
|
||||
|
||||
bfs_crop_planner.py, bfs_subject_crop.py
|
||||
from drozbay/MaskVidExperiments (https://github.com/drozbay/MaskVidExperiments),
|
||||
commit d98cc89 (v0.2.0), Copyright (c) drozbay, GPL-3.0.
|
||||
Vendored so the head-swap sampler needs no runtime dependency on that pack;
|
||||
behaviour unchanged, ComfyUI node classes dropped.
|
||||
@@ -0,0 +1,528 @@
|
||||
"""Crop-path planner, vendored from drozbay/MaskVidExperiments.
|
||||
|
||||
Source: https://github.com/drozbay/MaskVidExperiments (commit d98cc89, v0.2.0)
|
||||
Copyright (c) drozbay. Licensed GPL-3.0, the same licence this pack carries,
|
||||
which is what makes vendoring it legal here.
|
||||
|
||||
Vendored rather than imported so the head-swap sampler has no runtime
|
||||
dependency on that pack being installed. Behaviour is unchanged; only the
|
||||
ComfyUI node classes were dropped (they stay in the original pack, where they
|
||||
belong) and the planner import was repointed.
|
||||
|
||||
Why this code and not a crop of my own: naive per-frame crops around a mask
|
||||
jitter in position and size, and a video model reads that jitter as camera
|
||||
motion. These boxes are planned over the whole clip, so they hold still through
|
||||
mask noise and occlusion and follow only sustained movement.
|
||||
"""
|
||||
|
||||
"""Subject crop planner: shape as an optimization output.
|
||||
|
||||
Standalone numpy/scipy planner solved against binary masks; produces the
|
||||
per-frame crop boxes only.
|
||||
|
||||
Structure per mode:
|
||||
tracked -- one LP: scalar W, H + per-frame positions. Shape emerges.
|
||||
zoomed -- outer sweep over constant aspect candidates, inner LP per
|
||||
candidate: per-frame height (width slaved to aspect) +
|
||||
positions. Rent is charged against the tightest
|
||||
aspect-respecting padded box, never per axis.
|
||||
combined -- closed form: padded union box (also the movement_cost -> inf
|
||||
limit of the other two).
|
||||
|
||||
Dials (p):
|
||||
crop_scale padding promise as a multiple of the subject
|
||||
min_padding_allowed fraction of the promise enforced as hard floors
|
||||
min_padding_allowed_window odd median window defining sustained motion
|
||||
pad_deficit_tol shortfall above the floor costs 1/this per px-frame
|
||||
pad_surplus_tol size beyond the padded need costs 1/this per px-frame
|
||||
resize_cost cost per pixel of box size change (zoomed)
|
||||
movement_cost cost per pixel of box movement
|
||||
center_pull tie-break pull toward the subject center
|
||||
end_tightening fraction of rent charged on the phantoms
|
||||
end_tightening_window phantom frames held at each end
|
||||
zoom_step >1 quantizes sizes to geometric levels (zoomed)
|
||||
max_zoom_rate >1 caps per-frame size ratio change (zoomed), 0/1 off
|
||||
aspect_ratio >0 fixes the shape and skips the sweep
|
||||
seamless_loop periodic plan (positions and sizes wrap)
|
||||
"""
|
||||
|
||||
import math
|
||||
|
||||
import numpy as np
|
||||
from scipy import ndimage, sparse
|
||||
from scipy.optimize import linprog
|
||||
|
||||
|
||||
def _extent_tracks(binary):
|
||||
"""Per-frame subject extents, NaN gaps interpolated and edge-held."""
|
||||
n = binary.shape[0]
|
||||
bx0 = np.full(n, np.nan)
|
||||
bx1 = np.full(n, np.nan)
|
||||
by0 = np.full(n, np.nan)
|
||||
by1 = np.full(n, np.nan)
|
||||
for i, frame in enumerate(binary):
|
||||
ys, xs = frame.nonzero()
|
||||
if len(xs):
|
||||
bx0[i], bx1[i] = xs.min(), xs.max()
|
||||
by0[i], by1[i] = ys.min(), ys.max()
|
||||
known = ~np.isnan(bx0)
|
||||
idx = np.arange(n)
|
||||
out = [np.interp(idx, idx[known], t[known]) for t in (bx0, bx1, by0, by1)]
|
||||
return out, known
|
||||
|
||||
|
||||
def _median_tracks(tracks, window):
|
||||
g = max(int(window), 1) | 1
|
||||
return [ndimage.median_filter(t, size=g, mode="nearest") for t in tracks]
|
||||
|
||||
|
||||
def _round_corridor(t, lo, hi):
|
||||
ilo = np.ceil(lo - 1e-6)
|
||||
ihi = np.floor(hi + 1e-6)
|
||||
x = np.clip(np.round(t), ilo, ihi)
|
||||
return np.where(ilo > ihi, np.round((lo + hi) / 2), x).astype(int)
|
||||
|
||||
|
||||
class _LP:
|
||||
"""Row-wise builder for sparse linprog problems."""
|
||||
|
||||
def __init__(self):
|
||||
self.blocks = [] # (name, size)
|
||||
self.off = {}
|
||||
self.nv = 0
|
||||
self.ri, self.ci, self.vv, self.rhs = [], [], [], []
|
||||
self.c = []
|
||||
self.lo, self.hi = [], []
|
||||
|
||||
def var(self, name, size, cost=0.0, lo=0.0, hi=None):
|
||||
self.off[name] = self.nv
|
||||
self.nv += size
|
||||
cost = np.broadcast_to(np.asarray(cost, dtype=float), (size,))
|
||||
lo = np.broadcast_to(np.asarray(lo, dtype=float), (size,))
|
||||
self.c.extend(cost)
|
||||
self.lo.extend(lo)
|
||||
if hi is None:
|
||||
self.hi.extend([None] * size)
|
||||
else:
|
||||
hi_arr = np.broadcast_to(np.asarray(hi, dtype=float), (size,))
|
||||
self.hi.extend(hi_arr)
|
||||
return self.off[name]
|
||||
|
||||
def row(self, cols, vals, b):
|
||||
r = len(self.rhs)
|
||||
self.ri.extend([r] * len(cols))
|
||||
self.ci.extend(cols)
|
||||
self.vv.extend(vals)
|
||||
self.rhs.append(b)
|
||||
|
||||
def solve(self, eq=None):
|
||||
A = sparse.csr_matrix((self.vv, (self.ri, self.ci)),
|
||||
shape=(len(self.rhs), self.nv))
|
||||
bounds = [(l, h) for l, h in zip(self.lo, self.hi)]
|
||||
kw = {}
|
||||
if eq is not None:
|
||||
er, ec, ev, eb = eq
|
||||
kw["A_eq"] = sparse.csr_matrix((ev, (er, ec)),
|
||||
shape=(max(er) + 1, self.nv))
|
||||
kw["b_eq"] = np.asarray(eb, dtype=float)
|
||||
res = linprog(np.array(self.c), A_ub=A,
|
||||
b_ub=np.array(self.rhs, dtype=float),
|
||||
bounds=bounds, method="highs", **kw)
|
||||
if not res.success:
|
||||
raise RuntimeError(f"crop planner solve failed: {res.message}")
|
||||
return res
|
||||
|
||||
|
||||
def _prep(binary, img_w, img_h, p):
|
||||
"""Shared preprocessing: extent tracks, phantoms, pads, floors."""
|
||||
n = binary.shape[0]
|
||||
(bx0, bx1, by0, by1), known = _extent_tracks(binary)
|
||||
loop = bool(p.get("seamless_loop")) and n > 1
|
||||
s0 = int(p.get("end_tightening_window", 0)) if not loop else 0
|
||||
if s0:
|
||||
hold = lambda a: np.concatenate(
|
||||
[np.full(s0, a[0]), a, np.full(s0, a[-1])])
|
||||
bx0, bx1, by0, by1 = hold(bx0), hold(bx1), hold(by0), hold(by1)
|
||||
m = len(bx0)
|
||||
scale = p["crop_scale"]
|
||||
padx = (scale - 1) / 2 * (bx1 - bx0 + 1)
|
||||
pady = (scale - 1) / 2 * (by1 - by0 + 1)
|
||||
|
||||
floor = float(p.get("min_padding_allowed", 0.0))
|
||||
sx0, sx1, sy0, sy1 = _median_tracks([bx0, bx1, by0, by1],
|
||||
p.get("min_padding_allowed_window", 16))
|
||||
pad_fx = floor * (scale - 1) / 2 * (sx1 - sx0 + 1)
|
||||
pad_fy = floor * (scale - 1) / 2 * (sy1 - sy0 + 1)
|
||||
# floor datums, clamped to the image (a side pinned at the image edge
|
||||
# is exempt from its promise)
|
||||
flo_x = np.minimum(sx1 + 1 + pad_fx, img_w)
|
||||
fhi_x = np.maximum(sx0 - pad_fx, 0)
|
||||
flo_y = np.minimum(sy1 + 1 + pad_fy, img_h)
|
||||
fhi_y = np.maximum(sy0 - pad_fy, 0)
|
||||
|
||||
# soft targets: the full promise, clamped to the image
|
||||
lx = np.maximum(bx0 - padx, 0)
|
||||
rx = np.minimum(bx1 + 1 + padx, img_w)
|
||||
ly = np.maximum(by0 - pady, 0)
|
||||
ry = np.minimum(by1 + 1 + pady, img_h)
|
||||
|
||||
rent_scale = np.ones(m)
|
||||
if s0:
|
||||
rent_scale[:s0] = p.get("end_tightening", 0.0)
|
||||
rent_scale[m - s0:] = p.get("end_tightening", 0.0)
|
||||
|
||||
return {
|
||||
"n": n, "m": m, "s0": s0, "loop": loop, "known": known,
|
||||
"bx0": bx0, "bx1": bx1, "by0": by0, "by1": by1,
|
||||
"padx": padx, "pady": pady, "floor": floor,
|
||||
"flo_x": flo_x, "fhi_x": fhi_x, "flo_y": flo_y, "fhi_y": fhi_y,
|
||||
"lx": lx, "rx": rx, "ly": ly, "ry": ry, "rent_scale": rent_scale,
|
||||
}
|
||||
|
||||
|
||||
def _tv_rows(lp, var, tv, m, loop):
|
||||
nt = m if loop else m - 1
|
||||
for i in range(nt):
|
||||
a, b = lp.off[var] + i, lp.off[var] + (i + 1) % m
|
||||
t = lp.off[tv] + i
|
||||
lp.row([b, a, t], [1, -1, -1], 0)
|
||||
lp.row([b, a, t], [-1, 1, -1], 0)
|
||||
|
||||
|
||||
def _solve_tracked(pre, img_w, img_h, p):
|
||||
"""Scalar-size LP: W, H are single variables, shape emerges from the
|
||||
movement/rent balance. Manual aspect adds one equality."""
|
||||
m, loop = pre["m"], pre["loop"]
|
||||
nt = m if loop else m - 1
|
||||
bx0, bx1, by0, by1 = pre["bx0"], pre["bx1"], pre["by0"], pre["by1"]
|
||||
padx, pady = pre["padx"], pre["pady"]
|
||||
soft = 1.0 / max(p.get("pad_deficit_tol", 16), 1e-6)
|
||||
rent = pre["rent_scale"] / max(p.get("pad_surplus_tol", 16), 1e-6)
|
||||
mc = p.get("movement_cost", 1.0)
|
||||
eps = p.get("center_pull", 1e-4)
|
||||
extw = bx1 - bx0 + 1
|
||||
exth = by1 - by0 + 1
|
||||
|
||||
lp = _LP()
|
||||
lp.var("W", 1, lo=max(extw.max(), 1), hi=img_w)
|
||||
lp.var("H", 1, lo=max(exth.max(), 1), hi=img_h)
|
||||
ub_x = np.maximum(bx0, 0)
|
||||
ub_y = np.maximum(by0, 0)
|
||||
if pre["floor"] > 0:
|
||||
ub_x = np.minimum(ub_x, pre["fhi_x"])
|
||||
ub_y = np.minimum(ub_y, pre["fhi_y"])
|
||||
lp.var("x", m, lo=0.0, hi=ub_x)
|
||||
lp.var("y", m, lo=0.0, hi=ub_y)
|
||||
lp.var("tx", nt, cost=mc)
|
||||
lp.var("ty", nt, cost=mc)
|
||||
lp.var("vx", m, cost=soft)
|
||||
lp.var("vy", m, cost=soft)
|
||||
lp.var("rx", m, cost=rent)
|
||||
lp.var("ry", m, cost=rent)
|
||||
lp.var("ux", m, cost=eps)
|
||||
lp.var("uy", m, cost=eps)
|
||||
|
||||
_tv_rows(lp, "x", "tx", m, loop)
|
||||
_tv_rows(lp, "y", "ty", m, loop)
|
||||
W, H = lp.off["W"], lp.off["H"]
|
||||
for i in range(m):
|
||||
x, y = lp.off["x"] + i, lp.off["y"] + i
|
||||
vx, vy = lp.off["vx"] + i, lp.off["vy"] + i
|
||||
rx, ry = lp.off["rx"] + i, lp.off["ry"] + i
|
||||
ux, uy = lp.off["ux"] + i, lp.off["uy"] + i
|
||||
lp.row([x, W], [-1, -1], -(bx1[i] + 1)) # containment right
|
||||
lp.row([x, W], [1, 1], img_w) # image right
|
||||
lp.row([y, H], [-1, -1], -(by1[i] + 1))
|
||||
lp.row([y, H], [1, 1], img_h)
|
||||
lp.row([x, vx], [1, -1], pre["lx"][i]) # soft pad, left
|
||||
lp.row([x, W, vx], [-1, -1, -1], -pre["rx"][i]) # soft pad, right
|
||||
lp.row([y, vy], [1, -1], pre["ly"][i])
|
||||
lp.row([y, H, vy], [-1, -1, -1], -pre["ry"][i])
|
||||
lp.row([W, rx], [1, -1], min(extw[i] + 2 * padx[i], img_w))
|
||||
lp.row([H, ry], [1, -1], min(exth[i] + 2 * pady[i], img_h))
|
||||
if pre["floor"] > 0:
|
||||
lp.row([x, W], [-1, -1], -pre["flo_x"][i])
|
||||
lp.row([y, H], [-1, -1], -pre["flo_y"][i])
|
||||
cx = (bx0[i] + bx1[i] + 1) / 2
|
||||
cy = (by0[i] + by1[i] + 1) / 2
|
||||
lp.row([x, W, ux], [1, 0.5, -1], cx)
|
||||
lp.row([x, W, ux], [-1, -0.5, -1], -cx)
|
||||
lp.row([y, H, uy], [1, 0.5, -1], cy)
|
||||
lp.row([y, H, uy], [-1, -0.5, -1], -cy)
|
||||
|
||||
eq = None
|
||||
ar = p.get("aspect_ratio", 0.0)
|
||||
if ar > 0:
|
||||
eq = ([0, 0], [W, H], [1.0, -ar], [0.0])
|
||||
res = lp.solve(eq=eq)
|
||||
o = lp.off
|
||||
return (res.x[W], res.x[H],
|
||||
res.x[o["x"]:o["x"] + m], res.x[o["y"]:o["y"] + m])
|
||||
|
||||
|
||||
def _yielded_floors(pre, ar, img_w, img_h):
|
||||
"""Relax the floor datums per side wherever containment, the image, or
|
||||
the aspect lock leaves no room."""
|
||||
bx0, bx1, by0, by1 = pre["bx0"], pre["bx1"], pre["by0"], pre["by1"]
|
||||
wmax = min(img_w, ar * img_h)
|
||||
hmax = min(img_h, img_w / ar)
|
||||
flo_x, fhi_x = pre["flo_x"].copy(), pre["fhi_x"].copy()
|
||||
flo_y, fhi_y = pre["flo_y"].copy(), pre["fhi_y"].copy()
|
||||
ex = np.maximum(flo_x - fhi_x - wmax, 0)
|
||||
flo_x = np.maximum(flo_x - ex / 2, bx1 + 1)
|
||||
fhi_x = np.minimum(fhi_x + ex / 2, np.maximum(bx0, 0))
|
||||
flo_x = np.minimum(flo_x, np.maximum(bx0, 0) + wmax)
|
||||
fhi_x = np.maximum(fhi_x, bx1 + 1 - wmax)
|
||||
ey = np.maximum(flo_y - fhi_y - hmax, 0)
|
||||
flo_y = np.maximum(flo_y - ey / 2, by1 + 1)
|
||||
fhi_y = np.minimum(fhi_y + ey / 2, np.maximum(by0, 0))
|
||||
flo_y = np.minimum(flo_y, np.maximum(by0, 0) + hmax)
|
||||
fhi_y = np.maximum(fhi_y, by1 + 1 - hmax)
|
||||
return flo_x, fhi_x, flo_y, fhi_y
|
||||
|
||||
|
||||
def _solve_zoomed_at(pre, ar, img_w, img_h, p):
|
||||
"""Inner LP for one aspect candidate. Width is ar*h throughout; rent
|
||||
is charged on h beyond the tightest aspect-respecting padded need, so
|
||||
width the ratio forces is never billed."""
|
||||
m, loop = pre["m"], pre["loop"]
|
||||
nt = m if loop else m - 1
|
||||
bx0, bx1, by0, by1 = pre["bx0"], pre["bx1"], pre["by0"], pre["by1"]
|
||||
padx, pady = pre["padx"], pre["pady"]
|
||||
extw = bx1 - bx0 + 1
|
||||
exth = by1 - by0 + 1
|
||||
soft = 1.0 / max(p.get("pad_deficit_tol", 16), 1e-6)
|
||||
# need is aspect-aware (width the ratio forces is never billed), but
|
||||
# the excess is billed at its (1+ar) box magnitude so wide and narrow
|
||||
# candidates pay the same for the same true slack
|
||||
rent = (1.0 + ar) * pre["rent_scale"] / max(p.get("pad_surplus_tol", 16), 1e-6)
|
||||
mc = p.get("movement_cost", 1.0)
|
||||
sc = p.get("resize_cost", 2.0) * (1.0 + ar)
|
||||
eps = p.get("center_pull", 1e-4)
|
||||
|
||||
hmax = min(img_h, img_w / ar)
|
||||
hmin = np.maximum(np.maximum(exth, extw / ar), 1.0)
|
||||
if hmin.max() > hmax + 1e-6:
|
||||
return None # this aspect cannot contain the subject in-image
|
||||
need_h = np.minimum(
|
||||
np.maximum(exth + 2 * pady, (extw + 2 * padx) / ar), hmax)
|
||||
|
||||
floors = pre["floor"] > 0
|
||||
if floors:
|
||||
flo_x, fhi_x, flo_y, fhi_y = _yielded_floors(pre, ar, img_w, img_h)
|
||||
|
||||
lp = _LP()
|
||||
ub_x = np.maximum(bx0, 0)
|
||||
ub_y = np.maximum(by0, 0)
|
||||
if floors:
|
||||
ub_x = np.minimum(ub_x, fhi_x)
|
||||
ub_y = np.minimum(ub_y, fhi_y)
|
||||
lp.var("x", m, lo=0.0, hi=ub_x)
|
||||
lp.var("y", m, lo=0.0, hi=ub_y)
|
||||
lp.var("h", m, lo=hmin, hi=hmax)
|
||||
lp.var("tx", nt, cost=mc)
|
||||
lp.var("ty", nt, cost=mc)
|
||||
lp.var("th", nt, cost=sc)
|
||||
lp.var("vx", m, cost=soft)
|
||||
lp.var("vy", m, cost=soft)
|
||||
lp.var("r", m, cost=rent)
|
||||
lp.var("ux", m, cost=eps)
|
||||
lp.var("uy", m, cost=eps)
|
||||
|
||||
_tv_rows(lp, "x", "tx", m, loop)
|
||||
_tv_rows(lp, "y", "ty", m, loop)
|
||||
_tv_rows(lp, "h", "th", m, loop)
|
||||
|
||||
rate = p.get("max_zoom_rate", 0.0)
|
||||
if rate and rate > 1.0:
|
||||
for i in range(nt):
|
||||
a, b = lp.off["h"] + i, lp.off["h"] + (i + 1) % m
|
||||
lp.row([b, a], [1, -rate], 0)
|
||||
lp.row([a, b], [1, -rate], 0)
|
||||
|
||||
for i in range(m):
|
||||
x, y, h = lp.off["x"] + i, lp.off["y"] + i, lp.off["h"] + i
|
||||
vx, vy = lp.off["vx"] + i, lp.off["vy"] + i
|
||||
r = lp.off["r"] + i
|
||||
ux, uy = lp.off["ux"] + i, lp.off["uy"] + i
|
||||
lp.row([x, h], [-1, -ar], -(bx1[i] + 1)) # containment right
|
||||
lp.row([x, h], [1, ar], img_w) # image right
|
||||
lp.row([y, h], [-1, -1], -(by1[i] + 1))
|
||||
lp.row([y, h], [1, 1], img_h)
|
||||
lp.row([x, vx], [1, -1], pre["lx"][i])
|
||||
lp.row([x, h, vx], [-1, -ar, -1], -pre["rx"][i])
|
||||
lp.row([y, vy], [1, -1], pre["ly"][i])
|
||||
lp.row([y, h, vy], [-1, -1, -1], -pre["ry"][i])
|
||||
lp.row([h, r], [1, -1], need_h[i])
|
||||
if floors:
|
||||
lp.row([x, h], [-1, -ar], -flo_x[i])
|
||||
lp.row([y, h], [-1, -1], -flo_y[i])
|
||||
cx = (bx0[i] + bx1[i] + 1) / 2
|
||||
cy = (by0[i] + by1[i] + 1) / 2
|
||||
lp.row([x, h, ux], [1, 0.5 * ar, -1], cx)
|
||||
lp.row([x, h, ux], [-1, -0.5 * ar, -1], -cx)
|
||||
lp.row([y, h, uy], [1, 0.5, -1], cy)
|
||||
lp.row([y, h, uy], [-1, -0.5, -1], -cy)
|
||||
|
||||
res = lp.solve()
|
||||
o = lp.off
|
||||
# the objective is the sweep score: floors carry the padding tier as
|
||||
# hard rows, so candidates compare on how cheaply they serve the
|
||||
# cell's own priced trade above them
|
||||
viol = res.x[o["vx"]:o["vx"] + m].sum() + res.x[o["vy"]:o["vy"] + m].sum()
|
||||
h_t = res.x[o["h"]:o["h"] + m]
|
||||
# dead width -- width this ratio forces beyond the frame's padded
|
||||
# horizontal need -- is billed in the score only, at the same rent as
|
||||
# height slack. Inside the LP that width stays free (billing it there
|
||||
# collapses manual wide aspects), but across candidates it
|
||||
# must cost, or the sweep drifts to the widest feasible shape whenever
|
||||
# the vertical need pins the height. Capped at need_h: excess above
|
||||
# need_h already carries its width via the (1+ar) rent.
|
||||
w_need = np.minimum(extw + 2 * padx, img_w)
|
||||
dead_w = np.maximum(ar * np.minimum(h_t, need_h) - w_need, 0)
|
||||
dead = ((pre["rent_scale"] * dead_w).sum()
|
||||
/ max(p.get("pad_surplus_tol", 16), 1e-6))
|
||||
return {
|
||||
"ar": ar, "score": res.fun + dead, "viol": viol,
|
||||
"x": res.x[o["x"]:o["x"] + m], "y": res.x[o["y"]:o["y"] + m],
|
||||
"h": h_t,
|
||||
}
|
||||
|
||||
|
||||
def _aspect_candidates(pre, img_w, img_h, n_known):
|
||||
"""Hull of the per-frame shape quantiles and the padded union aspect:
|
||||
the winner lives between what the subject looks like and where it
|
||||
goes."""
|
||||
bx0, bx1, by0, by1 = pre["bx0"], pre["bx1"], pre["by0"], pre["by1"]
|
||||
padx, pady = pre["padx"], pre["pady"]
|
||||
frame_ar = (bx1 - bx0 + 1) / (by1 - by0 + 1)
|
||||
qs = np.quantile(frame_ar, [0.05, 0.25, 0.5, 0.75, 0.95])
|
||||
ux0 = (bx0 - padx).min()
|
||||
ux1 = (bx1 + 1 + padx).max()
|
||||
uy0 = (by0 - pady).min()
|
||||
uy1 = (by1 + 1 + pady).max()
|
||||
union_ar = max(ux1 - ux0, 1.0) / max(uy1 - uy0, 1.0)
|
||||
lo = min(qs.min(), union_ar)
|
||||
hi = max(qs.max(), union_ar)
|
||||
grid = np.geomspace(lo, hi, 7) if hi > lo * 1.02 else np.array([lo])
|
||||
cands = np.concatenate([grid, qs, [union_ar]])
|
||||
# feasibility band: the box must contain every frame inside the image
|
||||
extw = bx1 - bx0 + 1
|
||||
exth = by1 - by0 + 1
|
||||
band_lo = extw.max() / img_h
|
||||
band_hi = img_w / exth.max()
|
||||
cands = np.clip(cands, band_lo, band_hi)
|
||||
cands = np.unique(np.round(cands, 3))
|
||||
keep = [cands[0]]
|
||||
for a in cands[1:]:
|
||||
if a > keep[-1] * 1.02:
|
||||
keep.append(a)
|
||||
return keep
|
||||
|
||||
|
||||
def _quantize_steps(v, step, upper):
|
||||
base = v.min()
|
||||
k = np.ceil(np.log(v / base) / np.log(step) - 1e-9)
|
||||
return np.minimum(base * step ** k, upper)
|
||||
|
||||
|
||||
def plan(binary, sel, p, divisible_by=16):
|
||||
"""binary: (n, H, W) bool mask stack. Returns (boxes, info)."""
|
||||
n, img_h, img_w = binary.shape
|
||||
if not binary.any():
|
||||
raise ValueError("all masks are empty, nothing to plan")
|
||||
scale = p["crop_scale"]
|
||||
|
||||
if sel == "combined":
|
||||
ys, xs = np.nonzero(binary.any(axis=0))
|
||||
w0 = (xs.max() - xs.min() + 1) * scale
|
||||
h0 = (ys.max() - ys.min() + 1) * scale
|
||||
ar = p.get("aspect_ratio", 0.0)
|
||||
if ar > 0:
|
||||
if w0 < h0 * ar:
|
||||
w0 = h0 * ar
|
||||
else:
|
||||
h0 = w0 / ar
|
||||
w = min(math.ceil(w0 / divisible_by) * divisible_by,
|
||||
(img_w // divisible_by) * divisible_by or img_w)
|
||||
h = min(math.ceil(h0 / divisible_by) * divisible_by,
|
||||
(img_h // divisible_by) * divisible_by or img_h)
|
||||
x = min(max(0, round((xs.min() + xs.max() + 1) / 2 - w / 2)), img_w - w)
|
||||
y = min(max(0, round((ys.min() + ys.max() + 1) / 2 - h / 2)), img_h - h)
|
||||
box = {"x": int(x), "y": int(y), "width": int(w), "height": int(h)}
|
||||
return [box] * n, {"aspect": w / h}
|
||||
|
||||
pre = _prep(binary, img_w, img_h, p)
|
||||
m, s0 = pre["m"], pre["s0"]
|
||||
bx0, bx1, by0, by1 = pre["bx0"], pre["bx1"], pre["by0"], pre["by1"]
|
||||
|
||||
if sel == "tracked":
|
||||
Wf, Hf, xt, yt = _solve_tracked(pre, img_w, img_h, p)
|
||||
w = min(math.ceil(Wf / divisible_by) * divisible_by,
|
||||
(img_w // divisible_by) * divisible_by or img_w)
|
||||
h = min(math.ceil(Hf / divisible_by) * divisible_by,
|
||||
(img_h // divisible_by) * divisible_by or img_h)
|
||||
lo_x = np.maximum(bx1 + 1 - w, 0)
|
||||
hi_x = np.minimum(np.maximum(bx0, 0), img_w - w)
|
||||
lo_y = np.maximum(by1 + 1 - h, 0)
|
||||
hi_y = np.minimum(np.maximum(by0, 0), img_h - h)
|
||||
if pre["floor"] > 0:
|
||||
lo_x = np.maximum(lo_x, np.minimum(pre["flo_x"] - w, hi_x))
|
||||
hi_x = np.minimum(hi_x, np.maximum(pre["fhi_x"], lo_x))
|
||||
lo_y = np.maximum(lo_y, np.minimum(pre["flo_y"] - h, hi_y))
|
||||
hi_y = np.minimum(hi_y, np.maximum(pre["fhi_y"], lo_y))
|
||||
xi = _round_corridor(xt, lo_x, hi_x)[s0:s0 + n]
|
||||
yi = _round_corridor(yt, lo_y, hi_y)[s0:s0 + n]
|
||||
boxes = [{"x": int(xi[i]), "y": int(yi[i]),
|
||||
"width": int(w), "height": int(h)} for i in range(n)]
|
||||
return boxes, {"aspect": w / h, "W": Wf, "H": Hf}
|
||||
|
||||
# zoomed
|
||||
ar_manual = p.get("aspect_ratio", 0.0)
|
||||
if ar_manual > 0:
|
||||
extw = bx1 - bx0 + 1
|
||||
exth = by1 - by0 + 1
|
||||
ar = min(max(ar_manual, extw.max() / img_h), img_w / exth.max())
|
||||
best = _solve_zoomed_at(pre, ar, img_w, img_h, p)
|
||||
if best is None:
|
||||
raise RuntimeError("requested aspect_ratio cannot contain the subject")
|
||||
tried = [best]
|
||||
else:
|
||||
tried = []
|
||||
for ar in _aspect_candidates(pre, img_w, img_h, pre["known"].sum()):
|
||||
sol = _solve_zoomed_at(pre, ar, img_w, img_h, p)
|
||||
if sol is not None:
|
||||
tried.append(sol)
|
||||
if not tried:
|
||||
raise RuntimeError("no feasible aspect candidate")
|
||||
best = min(tried, key=lambda s: s["score"])
|
||||
for ar in (best["ar"] * 0.93, best["ar"] * 1.075):
|
||||
sol = _solve_zoomed_at(pre, ar, img_w, img_h, p)
|
||||
if sol is not None and sol["score"] < best["score"]:
|
||||
best = sol
|
||||
|
||||
ar = best["ar"]
|
||||
h = best["h"]
|
||||
if p.get("zoom_step", 1.0) > 1.0:
|
||||
h = _quantize_steps(h, p["zoom_step"], min(img_h, img_w / ar))
|
||||
w = np.minimum(h * ar, img_w)
|
||||
bw = np.minimum(np.ceil(w - 1e-6), img_w).astype(int)
|
||||
bh = np.minimum(np.ceil(h - 1e-6), img_h).astype(int)
|
||||
lo_x = np.maximum(bx1 + 1 - bw, 0)
|
||||
hi_x = np.minimum(np.maximum(bx0, 0), img_w - bw)
|
||||
lo_y = np.maximum(by1 + 1 - bh, 0)
|
||||
hi_y = np.minimum(np.maximum(by0, 0), img_h - bh)
|
||||
if pre["floor"] > 0:
|
||||
flo_x, fhi_x, flo_y, fhi_y = _yielded_floors(pre, ar, img_w, img_h)
|
||||
lo_x = np.maximum(lo_x, np.minimum(flo_x - bw, hi_x))
|
||||
hi_x = np.minimum(hi_x, np.maximum(fhi_x, lo_x))
|
||||
lo_y = np.maximum(lo_y, np.minimum(flo_y - bh, hi_y))
|
||||
hi_y = np.minimum(hi_y, np.maximum(fhi_y, lo_y))
|
||||
xi = _round_corridor(best["x"], lo_x, hi_x)[s0:s0 + n]
|
||||
yi = _round_corridor(best["y"], lo_y, hi_y)[s0:s0 + n]
|
||||
bw, bh = bw[s0:s0 + n], bh[s0:s0 + n]
|
||||
boxes = [{"x": int(min(max(xi[i], 0), img_w - bw[i])),
|
||||
"y": int(min(max(yi[i], 0), img_h - bh[i])),
|
||||
"width": int(bw[i]), "height": int(bh[i])} for i in range(n)]
|
||||
return boxes, {"aspect": ar, "swept": ar_manual <= 0,
|
||||
"candidates": len(tried)}
|
||||
@@ -0,0 +1,291 @@
|
||||
"""Stable subject-crop planning, vendored from drozbay/MaskVidExperiments.
|
||||
|
||||
Source: https://github.com/drozbay/MaskVidExperiments (commit d98cc89, v0.2.0)
|
||||
Copyright (c) drozbay. Licensed GPL-3.0, the same licence this pack carries,
|
||||
which is what makes vendoring it legal here.
|
||||
|
||||
Vendored rather than imported so the head-swap sampler has no runtime
|
||||
dependency on that pack being installed. Behaviour is unchanged; only the
|
||||
ComfyUI node classes were dropped (they stay in the original pack, where they
|
||||
belong) and the planner import was repointed.
|
||||
|
||||
Why this code and not a crop of my own: naive per-frame crops around a mask
|
||||
jitter in position and size, and a video model reads that jitter as camera
|
||||
motion. These boxes are planned over the whole clip, so they hold still through
|
||||
mask noise and occlusion and follow only sustained movement.
|
||||
"""
|
||||
|
||||
"""Subject-tracked batch crop/uncrop for video mask workflows. The crop
|
||||
boxes come from planner.py, which plans position, size, and shape jointly
|
||||
over the whole clip."""
|
||||
|
||||
import logging
|
||||
import math
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from scipy import ndimage
|
||||
|
||||
import comfy.utils
|
||||
|
||||
from . import bfs_crop_planner as planner
|
||||
|
||||
CATEGORY = "MaskVidExperiments"
|
||||
|
||||
|
||||
def _clean_mask_stack(masks, value_thresh, min_pixels, min_frames):
|
||||
"""Binarize a mask batch and drop noise blobs.
|
||||
|
||||
Connected components are labelled in 3D (time + space). A blob survives if
|
||||
its peak single-frame area clears min_pixels and it is either subject-sized
|
||||
(peak >= 50% of the largest blob, so fast motion that splits the subject
|
||||
into per-frame components doesn't delete it) or persists min_frames frames.
|
||||
"""
|
||||
binary = masks.detach().cpu().numpy() > value_thresh
|
||||
if not binary.any():
|
||||
return binary
|
||||
|
||||
labels, count = ndimage.label(binary, structure=np.ones((3, 3, 3), dtype=bool))
|
||||
if count == 0:
|
||||
return binary
|
||||
|
||||
span = np.zeros(count + 1, dtype=np.int64)
|
||||
for label, location in enumerate(ndimage.find_objects(labels), start=1):
|
||||
if location is not None:
|
||||
span[label] = location[0].stop - location[0].start
|
||||
|
||||
peak = np.zeros(count + 1, dtype=np.int64)
|
||||
for frame in labels:
|
||||
peak = np.maximum(peak, np.bincount(frame.ravel(), minlength=count + 1))
|
||||
|
||||
span[0] = 0
|
||||
peak[0] = 0
|
||||
subject = peak >= 0.5 * peak.max() if peak.max() > 0 else np.zeros_like(peak, dtype=bool)
|
||||
keep = (peak >= min_pixels) & (subject | (span >= min_frames))
|
||||
keep[0] = False
|
||||
return keep[labels]
|
||||
|
||||
|
||||
def _open_reconstruct(masks, value_thresh, radius, min_frames):
|
||||
"""Morphological opening by reconstruction on a mask batch.
|
||||
|
||||
Erodes each frame spatially so blobs thinner than ~2*radius vanish, then
|
||||
instead of dilating back, keeps the original shape of every blob with a
|
||||
surviving core, so the subject's silhouette (thin protrusions included) is
|
||||
untouched. Blobs are 3D components (time + space): a core surviving in any
|
||||
frame keeps the whole track. border_value=1 so subjects cut off by the
|
||||
frame edge are not eroded from that side.
|
||||
"""
|
||||
binary = masks.detach().cpu().numpy() > value_thresh
|
||||
if not binary.any():
|
||||
return binary
|
||||
|
||||
spatial = np.zeros((3, 3, 3), dtype=bool)
|
||||
spatial[1] = [[0, 1, 0], [1, 1, 1], [0, 1, 0]]
|
||||
eroded = ndimage.binary_erosion(binary, structure=spatial, iterations=radius, border_value=1)
|
||||
|
||||
labels, count = ndimage.label(binary, structure=np.ones((3, 3, 3), dtype=bool))
|
||||
if count == 0:
|
||||
return binary
|
||||
|
||||
keep = np.zeros(count + 1, dtype=bool)
|
||||
keep[np.unique(labels[eroded])] = True
|
||||
if min_frames > 1:
|
||||
for label, location in enumerate(ndimage.find_objects(labels), start=1):
|
||||
if location is not None and location[0].stop - location[0].start < min_frames:
|
||||
keep[label] = False
|
||||
keep[0] = False
|
||||
return keep[labels]
|
||||
|
||||
|
||||
def _resize_image(img_hwc, w, h, method="lanczos"):
|
||||
if img_hwc.shape[0] == h and img_hwc.shape[1] == w:
|
||||
return img_hwc
|
||||
return comfy.utils.common_upscale(
|
||||
img_hwc.movedim(-1, 0).unsqueeze(0), w, h, method, "disabled"
|
||||
).squeeze(0).movedim(0, -1)
|
||||
|
||||
|
||||
def _resize_mask(mask_hw, w, h, method="bilinear"):
|
||||
if mask_hw.shape[0] == h and mask_hw.shape[1] == w:
|
||||
return mask_hw
|
||||
return comfy.utils.common_upscale(
|
||||
mask_hw[None, None], w, h, method, "disabled"
|
||||
)[0, 0]
|
||||
|
||||
|
||||
def _upscale_size(w, h, megapixels, divisible_by):
|
||||
"""Size scaling a w x h crop to about abs(megapixels) pixels on the
|
||||
divisible_by grid. A positive value is a floor: None when already at or
|
||||
above it. A negative value is a target, reached by downscaling too."""
|
||||
scale = math.sqrt(abs(megapixels) * 1024 * 1024 / (w * h))
|
||||
if megapixels > 0 and scale <= 1.0:
|
||||
return None
|
||||
return (max(1, round(w * scale / divisible_by)) * divisible_by,
|
||||
max(1, round(h * scale / divisible_by)) * divisible_by)
|
||||
|
||||
|
||||
def _cell_params(mode):
|
||||
"""Raw planner dials for the standard node's (padding, prefer) cell.
|
||||
The advanced node feeds the same dials directly. guaranteed floors the
|
||||
full promise against every raw frame (floor window 1): a true
|
||||
guarantee, so mask noise must be cleaned upstream. firm floors 70%
|
||||
against the sustained tracks; flexible has no floor. In tracked mode
|
||||
the crop never rescales, so prefer differentiates through the oversize
|
||||
rent instead: stillness holds a bigger box to move less. In zoomed
|
||||
mode the rent is never scaled by prefer (cheapening it tips whole
|
||||
clips into one constant maximum size, killing zoom-follow); prefer
|
||||
differentiates through the resize cost."""
|
||||
sel = mode["mode"]
|
||||
pad_level = mode.get("padding", "firm")
|
||||
prefer = mode.get("prefer", "stillness")
|
||||
floor = {"guaranteed": 1.0, "firm": 0.7, "flexible": 0.0}[pad_level]
|
||||
still = prefer == "stillness"
|
||||
if sel == "tracked":
|
||||
oversize = 32.0 if still else 8.0
|
||||
else:
|
||||
oversize = float(mode.get("pad_surplus_tol", 16))
|
||||
return {
|
||||
"crop_scale": mode["crop_scale"],
|
||||
"min_padding_allowed": floor,
|
||||
"min_padding_allowed_window": 1 if pad_level == "guaranteed" else 16,
|
||||
"pad_deficit_tol": 16.0,
|
||||
"pad_surplus_tol": oversize,
|
||||
"resize_cost": 2.0 if still else 1.0,
|
||||
"movement_cost": 1.0,
|
||||
"center_pull": 1e-4,
|
||||
"end_tightening": 0.0,
|
||||
"end_tightening_window": 80 if floor > 0 else 0,
|
||||
"zoom_step": mode.get("zoom_step", 1.0),
|
||||
"max_zoom_rate": 0.0,
|
||||
"aspect_ratio": mode["aspect_ratio"],
|
||||
"seamless_loop": mode.get("seamless_loop", False),
|
||||
}
|
||||
|
||||
|
||||
def _debug_summary(binary, boxes, scale, sel, info, output_size):
|
||||
"""Brief per-run report: what shape the planner chose and how well the
|
||||
plan delivered the padding promise."""
|
||||
n = len(boxes)
|
||||
worst = []
|
||||
img_h, img_w = binary.shape[1:]
|
||||
for i in range(n):
|
||||
ys, xs = binary[i].nonzero()
|
||||
if len(xs) == 0:
|
||||
worst.append(np.nan)
|
||||
continue
|
||||
b = boxes[i]
|
||||
bx1 = b["x"] + b["width"] - 1
|
||||
by1 = b["y"] + b["height"] - 1
|
||||
wx = (scale - 1) / 2 * (xs.max() - xs.min() + 1)
|
||||
wy = (scale - 1) / 2 * (ys.max() - ys.min() + 1)
|
||||
fr = []
|
||||
if b["x"] > 0: fr.append((xs.min() - b["x"]) / max(wx, 1e-6))
|
||||
if bx1 < img_w - 1: fr.append((bx1 - xs.max()) / max(wx, 1e-6))
|
||||
if b["y"] > 0: fr.append((ys.min() - b["y"]) / max(wy, 1e-6))
|
||||
if by1 < img_h - 1: fr.append((by1 - ys.max()) / max(wy, 1e-6))
|
||||
worst.append(min(fr) if fr else np.nan)
|
||||
worst = np.array(worst)
|
||||
v = worst[~np.isnan(worst)]
|
||||
cx = np.array([b["x"] + b["width"] / 2 for b in boxes])
|
||||
cy = np.array([b["y"] + b["height"] / 2 for b in boxes])
|
||||
travel = np.hypot(np.diff(cx), np.diff(cy)).sum()
|
||||
|
||||
ar = info["aspect"]
|
||||
if info.get("swept"):
|
||||
shape = f"{ar:.2f} (chosen from {info['candidates']} candidates)"
|
||||
elif sel == "zoomed":
|
||||
shape = f"{ar:.2f} (manual)"
|
||||
else:
|
||||
shape = f"{ar:.2f}"
|
||||
ws = [b["width"] for b in boxes]
|
||||
hs = [b["height"] for b in boxes]
|
||||
lines = [
|
||||
f"mode: {sel}",
|
||||
f"frames: {n} ({img_w}x{img_h} source)",
|
||||
f"aspect_ratio: {shape}",
|
||||
f"smallest_box: {min(ws)}x{min(hs)}",
|
||||
f"largest_box: {max(ws)}x{max(hs)}",
|
||||
]
|
||||
if sel == "zoomed":
|
||||
lines.append(f"zoom_ratio: {max(hs) / max(min(hs), 1):.2f} "
|
||||
"(largest box / smallest box)")
|
||||
lines.append(f"crop region resizing: {max(ws)} x {max(hs)} -> "
|
||||
f"{output_size[0]} x {output_size[1]}")
|
||||
lines.append(f"movement: {travel:.0f}px "
|
||||
"(total distance the box center travels)")
|
||||
if len(v) and scale > 1.0:
|
||||
lines.append(f"padding_promised: {(scale - 1) / 2:.0%} of the "
|
||||
f"subject's size on each side (crop_scale {scale:.2f})")
|
||||
lines.append(f"padding_worst: {max(v.min(), 0):.0%} of promised")
|
||||
lines.append(f"padding_typical: {max(np.median(v), 0):.0%} of promised")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _plan_and_crop(original_images, masks, sel, p, divisible_by,
|
||||
mask_threshold, upscale_megapixels=0.0):
|
||||
if original_images.shape[0] != masks.shape[0]:
|
||||
raise ValueError(f"original_images ({original_images.shape[0]}) and masks ({masks.shape[0]}) must have the same frame count")
|
||||
if original_images.shape[1:3] != masks.shape[1:3]:
|
||||
raise ValueError(f"original_images ({original_images.shape[2]}x{original_images.shape[1]}) and masks ({masks.shape[2]}x{masks.shape[1]}) must have the same dimensions")
|
||||
|
||||
if 0.0 < p["crop_scale"] < 1.0:
|
||||
raise ValueError("crop_scale must be 0 (full frame) or at least 1.0")
|
||||
full_frame = p["crop_scale"] <= 0.0
|
||||
|
||||
binary = masks.detach().cpu().numpy() > mask_threshold
|
||||
if full_frame:
|
||||
img_h, img_w = masks.shape[1], masks.shape[2]
|
||||
boxes = [{"x": 0, "y": 0, "width": img_w, "height": img_h}
|
||||
for _ in range(masks.shape[0])]
|
||||
info = {"aspect": img_w / img_h}
|
||||
else:
|
||||
if not binary.any():
|
||||
raise ValueError("all masks are empty, nothing to crop")
|
||||
boxes, info = planner.plan(binary, sel, p, divisible_by)
|
||||
|
||||
bw = max(b["width"] for b in boxes)
|
||||
bh = max(b["height"] for b in boxes)
|
||||
size = None
|
||||
img_method, mask_method = "lanczos", "bilinear"
|
||||
if full_frame:
|
||||
gw = math.ceil(bw / divisible_by) * divisible_by
|
||||
gh = math.ceil(bh / divisible_by) * divisible_by
|
||||
if (gw, gh) != (bw, bh):
|
||||
size = (gw, gh)
|
||||
elif sel == "zoomed":
|
||||
# Target sized to the largest planned box: that stretch is ~1:1
|
||||
# and everything else upscales rather than losing detail.
|
||||
th = math.ceil(bh / divisible_by) * divisible_by
|
||||
size = (math.ceil(th * info["aspect"] / divisible_by) * divisible_by, th)
|
||||
if upscale_megapixels != 0:
|
||||
up = _upscale_size(*(size or (bw, bh)), upscale_megapixels, divisible_by)
|
||||
if up is not None:
|
||||
size = up
|
||||
img_method, mask_method = "bicubic", "nearest-exact"
|
||||
|
||||
debug = _debug_summary(binary, boxes, p["crop_scale"],
|
||||
f"{sel} (full frame)" if full_frame else sel,
|
||||
info, size or (bw, bh))
|
||||
|
||||
if size is not None:
|
||||
tw, th = size
|
||||
cropped_images = torch.stack([
|
||||
_resize_image(original_images[i, b["y"]:b["y"] + b["height"], b["x"]:b["x"] + b["width"], :], tw, th, img_method)
|
||||
for i, b in enumerate(boxes)
|
||||
])
|
||||
cropped_masks = torch.stack([
|
||||
_resize_mask(masks[i, b["y"]:b["y"] + b["height"], b["x"]:b["x"] + b["width"]], tw, th, mask_method)
|
||||
for i, b in enumerate(boxes)
|
||||
])
|
||||
else:
|
||||
cropped_images = torch.stack([
|
||||
original_images[i, b["y"]:b["y"] + b["height"], b["x"]:b["x"] + b["width"], :]
|
||||
for i, b in enumerate(boxes)
|
||||
])
|
||||
cropped_masks = torch.stack([
|
||||
masks[i, b["y"]:b["y"] + b["height"], b["x"]:b["x"] + b["width"]]
|
||||
for i, b in enumerate(boxes)
|
||||
])
|
||||
return (cropped_images, cropped_masks,
|
||||
[[b] for b in boxes], debug)
|
||||
@@ -0,0 +1,110 @@
|
||||
# BFS Head Swap Sampler (crop · mask · loop)
|
||||
|
||||
Runs an LTX head-swap LoRA and adds three things around it, each optional.
|
||||
Connect only what you need — the node degrades to whatever is wired.
|
||||
|
||||
| what you connect | what you get |
|
||||
|---|---|
|
||||
| `guide_video` + `identity_image` | plain head swap, one pass |
|
||||
| `+ subject_mask` | only the masked region is denoised |
|
||||
| `+ crop_mode` | the swap runs inside a box around the subject, then is pasted back |
|
||||
| `+ temporal_tile_size` | long clips sampled in chunks with overlap |
|
||||
|
||||
The LoRA is never asked to understand a mask or a crop. It keeps doing its one
|
||||
job: the guide rides on `source_id 1`, the identity on `source_id 2`. Everything
|
||||
else here is plumbing around it.
|
||||
|
||||
## The mask is native inpainting, not a LoRA feature
|
||||
|
||||
The mask becomes `latent["noise_mask"]`, which the guider consumes as
|
||||
`denoise_mask` — ComfyUI's own inpainting path. Pixels outside the mask keep the
|
||||
guide's own content, so **the original face stays visible to the model**. This is
|
||||
deliberately different from a masked-hole recipe, where the region is painted out
|
||||
before the model sees it.
|
||||
|
||||
Masks are reduced to the latent grid with **max**, not with ComfyUI's trilinear
|
||||
resize. Trilinear blurs a mask across frames and lets the original content bleed
|
||||
through the edit. Max keeps a latent cell that any masked pixel touches fully
|
||||
editable.
|
||||
|
||||
**Watch the `latent_mask` output.** It has one frame per *latent* frame, not per
|
||||
video frame, and that is the real resolution of the edit. A tight outline around
|
||||
a face can nearly vanish at that scale — and then nothing changes, no matter how
|
||||
good the LoRA is. If the mask looks thin there, raise `mask_grow`.
|
||||
|
||||
## Why crop, and which mode
|
||||
|
||||
At 512×288 a person filling a fifth of the frame leaves a face about 25 px tall.
|
||||
No LoRA recovers identity from 25 px — it is missing pixels, not missing
|
||||
training. Cropping the head region and sampling it full-frame gives that same
|
||||
face 200–300 px, and the result is feathered back into untouched frames.
|
||||
|
||||
| `crop_mode` | behaviour |
|
||||
|---|---|
|
||||
| `off` | the whole frame is sampled. Use it first, to confirm the swap works at all. |
|
||||
| `combined` | one static box around the subject's whole travel, held for the clip. No dependency. |
|
||||
| `tracked` | constant-size box that stays still until the subject would leave it, then moves as little as possible. |
|
||||
| `zoomed` | the box also follows the subject's size, planned over the whole clip, every crop resampled to one output size. |
|
||||
|
||||
`tracked` and `zoomed` use a planner vendored from
|
||||
[drozbay/MaskVidExperiments](https://github.com/drozbay/MaskVidExperiments)
|
||||
(GPL-3.0, the same licence this pack carries) — nothing extra to install. Those
|
||||
boxes hold still through mask noise and occlusion, and **a jittering crop reads
|
||||
to a video model as camera motion**, which is what a naive per-frame crop around
|
||||
a mask produces.
|
||||
|
||||
`crop_scale` is the box as a multiple of the subject. At 1.5 the subject occupies
|
||||
two thirds and the rest is margin. Keep neck and shoulders inside: a face-tight
|
||||
crop is a framing the LoRA never saw in training, and it will show.
|
||||
|
||||
## Sizing, which is the one fiddly part
|
||||
|
||||
The connected `latent` sets the sampled size. When you crop, the box is smaller
|
||||
than the frame, so the latent should match the box — the `debug` output prints
|
||||
the size to use:
|
||||
|
||||
```
|
||||
sample size 416x768 (connect EmptyLTXVLatentVideo at this size to avoid a resize)
|
||||
```
|
||||
|
||||
If they disagree the node resamples the crop on the way back, which costs exactly
|
||||
the sharpness the crop was there to buy. Run once, read `debug`, set
|
||||
`EmptyLTXVLatentVideo`, run again.
|
||||
|
||||
**The latent must come from the graph.** LTX‑2.5 latents are AV — video and audio
|
||||
nested together — and the node cannot fabricate that structure. Without one it
|
||||
builds a plain video latent, warns, and the guide may not reach the model at all.
|
||||
|
||||
## Chunking
|
||||
|
||||
`temporal_tile_size` at 0 samples the clip in one pass. Set it to the length the
|
||||
LoRA trained at (73 for the LTX head-swap recipe) for longer clips, and the node
|
||||
slices the guide, the mask and the overlap together per chunk.
|
||||
|
||||
Chunked sampling with an AV latent is refused rather than silently corrupted —
|
||||
slicing a nested latent in time is not a thing the node does yet.
|
||||
|
||||
## Reading the outputs
|
||||
|
||||
| output | what it answers |
|
||||
|---|---|
|
||||
| `images` | the result, with the crop pasted back |
|
||||
| `mask_over_source` | is the mask where I think it is? Red over the original frames, green box for the crop |
|
||||
| `cropped_guide` | exactly what the model was fed. If this is wrong, nothing downstream can be right |
|
||||
| `crop_mask` | the mask after grow/blur, in the crop's pixel space |
|
||||
| `latent_mask` | the mask as the sampler sees it — where a thin mask disappears |
|
||||
| `latent` | the sampled latent |
|
||||
| `debug` | crop mode and box, mask ops, tiling, the size to build the latent at |
|
||||
|
||||
## Suggested order when something looks wrong
|
||||
|
||||
1. `crop_mode: off`, no mask, `temporal_tile_size: 0`. This is the plain swap; if
|
||||
it fails, the problem is the recipe or the LoRA, not this node.
|
||||
2. Add the mask. Check `latent_mask` before judging the result.
|
||||
3. Add the crop. Check `mask_over_source` for the box, and set the latent to the
|
||||
size `debug` reports.
|
||||
4. Only then raise the frame count and turn on chunking.
|
||||
|
||||
Sampling settings follow the LoRA's recipe: CFG 3–5 with a real negative, and no
|
||||
LightX2V (CFG 1.0 means no amplification, and the reference stops being
|
||||
amplified).
|
||||
@@ -21,11 +21,11 @@ about 25 px tall. No LoRA recovers identity from that. Cropping the head region
|
||||
and sampling it full-frame gives the same face 200-300 px, then the result is
|
||||
feathered back into the untouched frames.
|
||||
|
||||
The crop planner comes from drozbay's MaskVidExperiments when that pack is
|
||||
installed (https://github.com/drozbay/MaskVidExperiments) — its boxes hold still
|
||||
through mask noise and occlusion, which naive per-frame crops do not, and a
|
||||
jittering crop reads to a video model as camera motion. Without the pack the node
|
||||
falls back to one static box around the subject's whole travel.
|
||||
The crop planner is vendored from drozbay's MaskVidExperiments (GPL-3.0, same as
|
||||
this pack) in `bfs_subject_crop.py`, so nothing here depends on that pack being
|
||||
installed. Its boxes hold still through mask noise and occlusion, which naive
|
||||
per-frame crops do not, and a jittering crop reads to a video model as camera
|
||||
motion.
|
||||
"""
|
||||
|
||||
import logging
|
||||
@@ -41,31 +41,32 @@ CATEGORY = "BFS/video"
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# optional dependency: drozbay/MaskVidExperiments crop planner
|
||||
# crop planning (vendored from drozbay/MaskVidExperiments, GPL-3.0)
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def _mvex_planner():
|
||||
"""`_plan_and_crop` from MaskVidExperiments, or None when it is not installed."""
|
||||
import importlib
|
||||
import os
|
||||
import sys
|
||||
def _planner():
|
||||
"""The vendored crop planner (bfs_subject_crop), or None if it fails to import."""
|
||||
try:
|
||||
from .bfs_subject_crop import _plan_and_crop
|
||||
return _plan_and_crop
|
||||
except Exception as exc: # pragma: no cover
|
||||
log.warning("vendored crop planner unavailable (%s); using the static box", exc)
|
||||
return None
|
||||
|
||||
import folder_paths
|
||||
|
||||
for root in folder_paths.get_folder_paths("custom_nodes"):
|
||||
path = os.path.join(root, "MaskVidExperiments", "nodes_subject_crop.py")
|
||||
if not os.path.isfile(path):
|
||||
continue
|
||||
try:
|
||||
spec = importlib.util.spec_from_file_location("_bfs_mvex_crop", path)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
sys.modules["_bfs_mvex_crop"] = mod
|
||||
spec.loader.exec_module(mod)
|
||||
return mod._plan_and_crop
|
||||
except Exception as exc: # pragma: no cover - depends on their internals
|
||||
log.warning("MaskVidExperiments found but not usable (%s); using the static box", exc)
|
||||
return None
|
||||
return None
|
||||
def _feather_ramp(h, w, feather, device):
|
||||
"""Blend weights that fade in from the crop border, as (1,h,w,1)."""
|
||||
if feather <= 0:
|
||||
return torch.ones(1, h, w, 1, device=device)
|
||||
ramp = torch.ones(h, w, device=device)
|
||||
f = min(int(feather), h // 2, w // 2)
|
||||
if f > 0:
|
||||
edge = torch.linspace(0, 1, f, device=device)
|
||||
ramp[:f, :] *= edge[:, None]
|
||||
ramp[-f:, :] *= edge.flip(0)[:, None]
|
||||
ramp[:, :f] *= edge[None, :]
|
||||
ramp[:, -f:] *= edge.flip(0)[None, :]
|
||||
return ramp[None, :, :, None]
|
||||
|
||||
|
||||
def _static_box(masks, images, crop_scale, divisible_by, thresh=0.1):
|
||||
@@ -109,7 +110,7 @@ def _mask_to_latent(masks, vae, latent_t, latent_h, latent_w):
|
||||
|
||||
ComfyUI would trilinearly resize the pixel mask instead, which blurs it
|
||||
across frames and lets the original content bleed through the edit -- the
|
||||
failure MaskVidExperiments' Mask To Latent Space node was written to fix.
|
||||
failure drozbay's Mask To Latent Space node was written to fix.
|
||||
Max keeps a latent cell that any masked pixel touches fully editable.
|
||||
"""
|
||||
m = masks.unsqueeze(1).float() # (N,1,H,W)
|
||||
@@ -195,8 +196,8 @@ class BFSHeadSwapMaskedSampler:
|
||||
"crop_mode": (["off", "combined", "tracked", "zoomed"], {"default": "off", "tooltip":
|
||||
"Sample inside a box around the subject instead of the whole frame -- the fix for faces "
|
||||
"that are too small to carry identity. combined: one static box for the clip. "
|
||||
"tracked/zoomed: MaskVidExperiments' planner (installed separately); without it these "
|
||||
"fall back to combined."}),
|
||||
"tracked: a constant-size box that stays still until the subject would leave it. "
|
||||
"zoomed: the box follows the subject's size too, planned over the whole clip."}),
|
||||
"crop_scale": ("FLOAT", {"default": 1.5, "min": 1.0, "max": 4.0, "step": 0.05, "tooltip":
|
||||
"Box size as a multiple of the subject. 1.5 leaves a third as margin. Keep neck and "
|
||||
"shoulders in: a face-tight crop is a framing the LoRA never saw in training."}),
|
||||
@@ -251,7 +252,7 @@ class BFSHeadSwapMaskedSampler:
|
||||
if mode == "off" or masks is None:
|
||||
return guide, masks, None, "crop: off"
|
||||
|
||||
planner = _mvex_planner() if mode in ("tracked", "zoomed") else None
|
||||
planner = _planner() if mode in ("tracked", "zoomed") else None
|
||||
if planner is not None:
|
||||
try:
|
||||
p = {"crop_scale": scale, "aspect_ratio": 0.0, "padding": "firm",
|
||||
@@ -259,9 +260,9 @@ class BFSHeadSwapMaskedSampler:
|
||||
"pad_surplus_tol": 16, "zoom_step": 1.0}
|
||||
out = planner(guide, masks, mode, p, div, 0.1, 0.0)
|
||||
cropped, cropped_masks, bboxes = out[0], out[1], out[2]
|
||||
return cropped, cropped_masks, ("mvex", bboxes), f"crop: mvex/{mode}"
|
||||
return cropped, cropped_masks, ("planned", bboxes), f"crop: planned/{mode}"
|
||||
except Exception as exc:
|
||||
log.warning("MaskVidExperiments planner failed (%s); using the static box", exc)
|
||||
log.warning("crop planner failed (%s); using the static box", exc)
|
||||
|
||||
x0, y0, w, h = _static_box(masks, guide, scale, div)
|
||||
cropped = guide[:, y0:y0 + h, x0:x0 + w, :]
|
||||
@@ -272,37 +273,33 @@ class BFSHeadSwapMaskedSampler:
|
||||
if ctx is None:
|
||||
return result
|
||||
kind, box = ctx
|
||||
if kind == "mvex":
|
||||
try:
|
||||
import importlib
|
||||
mod = importlib.import_module("_bfs_mvex_crop")
|
||||
return mod.MVEx_SubjectUncropNode.execute(result, original, box, feather).result[0]
|
||||
except Exception as exc:
|
||||
log.warning("MaskVidExperiments uncrop failed (%s); pasting the static way", exc)
|
||||
return result
|
||||
if kind == "planned":
|
||||
# one box per frame: paste each crop into its own box
|
||||
out = original.clone()
|
||||
n = min(result.shape[0], out.shape[0], len(box))
|
||||
for i in range(n):
|
||||
b = box[i][0] if isinstance(box[i], list) else box[i]
|
||||
x0, y0, w, h = (int(b[0]), int(b[1]), int(b[2]), int(b[3]))
|
||||
patch = result[i:i + 1]
|
||||
if patch.shape[1] != h or patch.shape[2] != w:
|
||||
patch = comfy.utils.common_upscale(
|
||||
patch.movedim(-1, 1), w, h, "lanczos", "disabled").movedim(1, -1)
|
||||
a = _feather_ramp(h, w, feather, patch.device)
|
||||
out[i:i + 1, y0:y0 + h, x0:x0 + w, :] = (
|
||||
a * patch + (1 - a) * out[i:i + 1, y0:y0 + h, x0:x0 + w, :])
|
||||
return out
|
||||
x0, y0, w, h = box
|
||||
out = original.clone()[: result.shape[0]]
|
||||
patch = result[: out.shape[0]]
|
||||
# The sampled crop does not have to match the box: the connected latent
|
||||
# sets the sampled size, and MaskVidExperiments' zoomed mode rescales
|
||||
# sets the sampled size, and the zoomed crop mode rescales
|
||||
# crops by design. Bring it back to the box before blending.
|
||||
if patch.shape[1] != h or patch.shape[2] != w:
|
||||
log.info("paste-back: resizing the sampled crop %dx%d -> box %dx%d",
|
||||
patch.shape[2], patch.shape[1], w, h)
|
||||
patch = comfy.utils.common_upscale(
|
||||
patch.movedim(-1, 1), w, h, "lanczos", "disabled").movedim(1, -1)
|
||||
if feather > 0:
|
||||
ramp = torch.ones(h, w, device=patch.device)
|
||||
f = min(feather, h // 2, w // 2)
|
||||
if f > 0:
|
||||
edge = torch.linspace(0, 1, f, device=patch.device)
|
||||
ramp[:f, :] *= edge[:, None]
|
||||
ramp[-f:, :] *= edge.flip(0)[:, None]
|
||||
ramp[:, :f] *= edge[None, :]
|
||||
ramp[:, -f:] *= edge.flip(0)[None, :]
|
||||
a = ramp[None, :, :, None]
|
||||
else:
|
||||
a = 1.0
|
||||
a = _feather_ramp(h, w, feather, patch.device)
|
||||
out[:, y0:y0 + h, x0:x0 + w, :] = (
|
||||
a * patch + (1 - a) * out[:, y0:y0 + h, x0:x0 + w, :]
|
||||
)
|
||||
|
||||
+2
-1
@@ -1,7 +1,7 @@
|
||||
[project]
|
||||
name = "bfsnodes"
|
||||
description = "Custom ComfyUI nodes including LTXV Edit Anything, Head-Swap/Anime2Real Bernini conditioning, the AMV Guide Builder, LTX Identity Transfer (IPT2V reference + source-phase RoPE), and MiniMax-H3 single-frame VAE loaders with tile control."
|
||||
version = "1.31.1"
|
||||
version = "1.32.0"
|
||||
license = {file = "LICENSE"}
|
||||
dependencies = [
|
||||
"torch",
|
||||
@@ -13,6 +13,7 @@ dependencies = [
|
||||
"insightface==0.7.3",
|
||||
"onnxruntime",
|
||||
"transformers>=4.55.0",
|
||||
"scipy",
|
||||
]
|
||||
# classifiers = [
|
||||
# # For OS-independent nodes (works on all operating systems)
|
||||
|
||||
Reference in New Issue
Block a user