diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..af23f00 --- /dev/null +++ b/NOTICE @@ -0,0 +1,8 @@ + +Portions of this pack are vendored from third-party GPL-3.0 projects: + + bfs_crop_planner.py, bfs_subject_crop.py + from drozbay/MaskVidExperiments (https://github.com/drozbay/MaskVidExperiments), + commit d98cc89 (v0.2.0), Copyright (c) drozbay, GPL-3.0. + Vendored so the head-swap sampler needs no runtime dependency on that pack; + behaviour unchanged, ComfyUI node classes dropped. diff --git a/bfs_crop_planner.py b/bfs_crop_planner.py new file mode 100644 index 0000000..9b2cc7d --- /dev/null +++ b/bfs_crop_planner.py @@ -0,0 +1,528 @@ +"""Crop-path planner, vendored from drozbay/MaskVidExperiments. + +Source: https://github.com/drozbay/MaskVidExperiments (commit d98cc89, v0.2.0) +Copyright (c) drozbay. Licensed GPL-3.0, the same licence this pack carries, +which is what makes vendoring it legal here. + +Vendored rather than imported so the head-swap sampler has no runtime +dependency on that pack being installed. Behaviour is unchanged; only the +ComfyUI node classes were dropped (they stay in the original pack, where they +belong) and the planner import was repointed. + +Why this code and not a crop of my own: naive per-frame crops around a mask +jitter in position and size, and a video model reads that jitter as camera +motion. These boxes are planned over the whole clip, so they hold still through +mask noise and occlusion and follow only sustained movement. +""" + +"""Subject crop planner: shape as an optimization output. + +Standalone numpy/scipy planner solved against binary masks; produces the +per-frame crop boxes only. + +Structure per mode: + tracked -- one LP: scalar W, H + per-frame positions. Shape emerges. + zoomed -- outer sweep over constant aspect candidates, inner LP per + candidate: per-frame height (width slaved to aspect) + + positions. Rent is charged against the tightest + aspect-respecting padded box, never per axis. + combined -- closed form: padded union box (also the movement_cost -> inf + limit of the other two). + +Dials (p): + crop_scale padding promise as a multiple of the subject + min_padding_allowed fraction of the promise enforced as hard floors + min_padding_allowed_window odd median window defining sustained motion + pad_deficit_tol shortfall above the floor costs 1/this per px-frame + pad_surplus_tol size beyond the padded need costs 1/this per px-frame + resize_cost cost per pixel of box size change (zoomed) + movement_cost cost per pixel of box movement + center_pull tie-break pull toward the subject center + end_tightening fraction of rent charged on the phantoms + end_tightening_window phantom frames held at each end + zoom_step >1 quantizes sizes to geometric levels (zoomed) + max_zoom_rate >1 caps per-frame size ratio change (zoomed), 0/1 off + aspect_ratio >0 fixes the shape and skips the sweep + seamless_loop periodic plan (positions and sizes wrap) +""" + +import math + +import numpy as np +from scipy import ndimage, sparse +from scipy.optimize import linprog + + +def _extent_tracks(binary): + """Per-frame subject extents, NaN gaps interpolated and edge-held.""" + n = binary.shape[0] + bx0 = np.full(n, np.nan) + bx1 = np.full(n, np.nan) + by0 = np.full(n, np.nan) + by1 = np.full(n, np.nan) + for i, frame in enumerate(binary): + ys, xs = frame.nonzero() + if len(xs): + bx0[i], bx1[i] = xs.min(), xs.max() + by0[i], by1[i] = ys.min(), ys.max() + known = ~np.isnan(bx0) + idx = np.arange(n) + out = [np.interp(idx, idx[known], t[known]) for t in (bx0, bx1, by0, by1)] + return out, known + + +def _median_tracks(tracks, window): + g = max(int(window), 1) | 1 + return [ndimage.median_filter(t, size=g, mode="nearest") for t in tracks] + + +def _round_corridor(t, lo, hi): + ilo = np.ceil(lo - 1e-6) + ihi = np.floor(hi + 1e-6) + x = np.clip(np.round(t), ilo, ihi) + return np.where(ilo > ihi, np.round((lo + hi) / 2), x).astype(int) + + +class _LP: + """Row-wise builder for sparse linprog problems.""" + + def __init__(self): + self.blocks = [] # (name, size) + self.off = {} + self.nv = 0 + self.ri, self.ci, self.vv, self.rhs = [], [], [], [] + self.c = [] + self.lo, self.hi = [], [] + + def var(self, name, size, cost=0.0, lo=0.0, hi=None): + self.off[name] = self.nv + self.nv += size + cost = np.broadcast_to(np.asarray(cost, dtype=float), (size,)) + lo = np.broadcast_to(np.asarray(lo, dtype=float), (size,)) + self.c.extend(cost) + self.lo.extend(lo) + if hi is None: + self.hi.extend([None] * size) + else: + hi_arr = np.broadcast_to(np.asarray(hi, dtype=float), (size,)) + self.hi.extend(hi_arr) + return self.off[name] + + def row(self, cols, vals, b): + r = len(self.rhs) + self.ri.extend([r] * len(cols)) + self.ci.extend(cols) + self.vv.extend(vals) + self.rhs.append(b) + + def solve(self, eq=None): + A = sparse.csr_matrix((self.vv, (self.ri, self.ci)), + shape=(len(self.rhs), self.nv)) + bounds = [(l, h) for l, h in zip(self.lo, self.hi)] + kw = {} + if eq is not None: + er, ec, ev, eb = eq + kw["A_eq"] = sparse.csr_matrix((ev, (er, ec)), + shape=(max(er) + 1, self.nv)) + kw["b_eq"] = np.asarray(eb, dtype=float) + res = linprog(np.array(self.c), A_ub=A, + b_ub=np.array(self.rhs, dtype=float), + bounds=bounds, method="highs", **kw) + if not res.success: + raise RuntimeError(f"crop planner solve failed: {res.message}") + return res + + +def _prep(binary, img_w, img_h, p): + """Shared preprocessing: extent tracks, phantoms, pads, floors.""" + n = binary.shape[0] + (bx0, bx1, by0, by1), known = _extent_tracks(binary) + loop = bool(p.get("seamless_loop")) and n > 1 + s0 = int(p.get("end_tightening_window", 0)) if not loop else 0 + if s0: + hold = lambda a: np.concatenate( + [np.full(s0, a[0]), a, np.full(s0, a[-1])]) + bx0, bx1, by0, by1 = hold(bx0), hold(bx1), hold(by0), hold(by1) + m = len(bx0) + scale = p["crop_scale"] + padx = (scale - 1) / 2 * (bx1 - bx0 + 1) + pady = (scale - 1) / 2 * (by1 - by0 + 1) + + floor = float(p.get("min_padding_allowed", 0.0)) + sx0, sx1, sy0, sy1 = _median_tracks([bx0, bx1, by0, by1], + p.get("min_padding_allowed_window", 16)) + pad_fx = floor * (scale - 1) / 2 * (sx1 - sx0 + 1) + pad_fy = floor * (scale - 1) / 2 * (sy1 - sy0 + 1) + # floor datums, clamped to the image (a side pinned at the image edge + # is exempt from its promise) + flo_x = np.minimum(sx1 + 1 + pad_fx, img_w) + fhi_x = np.maximum(sx0 - pad_fx, 0) + flo_y = np.minimum(sy1 + 1 + pad_fy, img_h) + fhi_y = np.maximum(sy0 - pad_fy, 0) + + # soft targets: the full promise, clamped to the image + lx = np.maximum(bx0 - padx, 0) + rx = np.minimum(bx1 + 1 + padx, img_w) + ly = np.maximum(by0 - pady, 0) + ry = np.minimum(by1 + 1 + pady, img_h) + + rent_scale = np.ones(m) + if s0: + rent_scale[:s0] = p.get("end_tightening", 0.0) + rent_scale[m - s0:] = p.get("end_tightening", 0.0) + + return { + "n": n, "m": m, "s0": s0, "loop": loop, "known": known, + "bx0": bx0, "bx1": bx1, "by0": by0, "by1": by1, + "padx": padx, "pady": pady, "floor": floor, + "flo_x": flo_x, "fhi_x": fhi_x, "flo_y": flo_y, "fhi_y": fhi_y, + "lx": lx, "rx": rx, "ly": ly, "ry": ry, "rent_scale": rent_scale, + } + + +def _tv_rows(lp, var, tv, m, loop): + nt = m if loop else m - 1 + for i in range(nt): + a, b = lp.off[var] + i, lp.off[var] + (i + 1) % m + t = lp.off[tv] + i + lp.row([b, a, t], [1, -1, -1], 0) + lp.row([b, a, t], [-1, 1, -1], 0) + + +def _solve_tracked(pre, img_w, img_h, p): + """Scalar-size LP: W, H are single variables, shape emerges from the + movement/rent balance. Manual aspect adds one equality.""" + m, loop = pre["m"], pre["loop"] + nt = m if loop else m - 1 + bx0, bx1, by0, by1 = pre["bx0"], pre["bx1"], pre["by0"], pre["by1"] + padx, pady = pre["padx"], pre["pady"] + soft = 1.0 / max(p.get("pad_deficit_tol", 16), 1e-6) + rent = pre["rent_scale"] / max(p.get("pad_surplus_tol", 16), 1e-6) + mc = p.get("movement_cost", 1.0) + eps = p.get("center_pull", 1e-4) + extw = bx1 - bx0 + 1 + exth = by1 - by0 + 1 + + lp = _LP() + lp.var("W", 1, lo=max(extw.max(), 1), hi=img_w) + lp.var("H", 1, lo=max(exth.max(), 1), hi=img_h) + ub_x = np.maximum(bx0, 0) + ub_y = np.maximum(by0, 0) + if pre["floor"] > 0: + ub_x = np.minimum(ub_x, pre["fhi_x"]) + ub_y = np.minimum(ub_y, pre["fhi_y"]) + lp.var("x", m, lo=0.0, hi=ub_x) + lp.var("y", m, lo=0.0, hi=ub_y) + lp.var("tx", nt, cost=mc) + lp.var("ty", nt, cost=mc) + lp.var("vx", m, cost=soft) + lp.var("vy", m, cost=soft) + lp.var("rx", m, cost=rent) + lp.var("ry", m, cost=rent) + lp.var("ux", m, cost=eps) + lp.var("uy", m, cost=eps) + + _tv_rows(lp, "x", "tx", m, loop) + _tv_rows(lp, "y", "ty", m, loop) + W, H = lp.off["W"], lp.off["H"] + for i in range(m): + x, y = lp.off["x"] + i, lp.off["y"] + i + vx, vy = lp.off["vx"] + i, lp.off["vy"] + i + rx, ry = lp.off["rx"] + i, lp.off["ry"] + i + ux, uy = lp.off["ux"] + i, lp.off["uy"] + i + lp.row([x, W], [-1, -1], -(bx1[i] + 1)) # containment right + lp.row([x, W], [1, 1], img_w) # image right + lp.row([y, H], [-1, -1], -(by1[i] + 1)) + lp.row([y, H], [1, 1], img_h) + lp.row([x, vx], [1, -1], pre["lx"][i]) # soft pad, left + lp.row([x, W, vx], [-1, -1, -1], -pre["rx"][i]) # soft pad, right + lp.row([y, vy], [1, -1], pre["ly"][i]) + lp.row([y, H, vy], [-1, -1, -1], -pre["ry"][i]) + lp.row([W, rx], [1, -1], min(extw[i] + 2 * padx[i], img_w)) + lp.row([H, ry], [1, -1], min(exth[i] + 2 * pady[i], img_h)) + if pre["floor"] > 0: + lp.row([x, W], [-1, -1], -pre["flo_x"][i]) + lp.row([y, H], [-1, -1], -pre["flo_y"][i]) + cx = (bx0[i] + bx1[i] + 1) / 2 + cy = (by0[i] + by1[i] + 1) / 2 + lp.row([x, W, ux], [1, 0.5, -1], cx) + lp.row([x, W, ux], [-1, -0.5, -1], -cx) + lp.row([y, H, uy], [1, 0.5, -1], cy) + lp.row([y, H, uy], [-1, -0.5, -1], -cy) + + eq = None + ar = p.get("aspect_ratio", 0.0) + if ar > 0: + eq = ([0, 0], [W, H], [1.0, -ar], [0.0]) + res = lp.solve(eq=eq) + o = lp.off + return (res.x[W], res.x[H], + res.x[o["x"]:o["x"] + m], res.x[o["y"]:o["y"] + m]) + + +def _yielded_floors(pre, ar, img_w, img_h): + """Relax the floor datums per side wherever containment, the image, or + the aspect lock leaves no room.""" + bx0, bx1, by0, by1 = pre["bx0"], pre["bx1"], pre["by0"], pre["by1"] + wmax = min(img_w, ar * img_h) + hmax = min(img_h, img_w / ar) + flo_x, fhi_x = pre["flo_x"].copy(), pre["fhi_x"].copy() + flo_y, fhi_y = pre["flo_y"].copy(), pre["fhi_y"].copy() + ex = np.maximum(flo_x - fhi_x - wmax, 0) + flo_x = np.maximum(flo_x - ex / 2, bx1 + 1) + fhi_x = np.minimum(fhi_x + ex / 2, np.maximum(bx0, 0)) + flo_x = np.minimum(flo_x, np.maximum(bx0, 0) + wmax) + fhi_x = np.maximum(fhi_x, bx1 + 1 - wmax) + ey = np.maximum(flo_y - fhi_y - hmax, 0) + flo_y = np.maximum(flo_y - ey / 2, by1 + 1) + fhi_y = np.minimum(fhi_y + ey / 2, np.maximum(by0, 0)) + flo_y = np.minimum(flo_y, np.maximum(by0, 0) + hmax) + fhi_y = np.maximum(fhi_y, by1 + 1 - hmax) + return flo_x, fhi_x, flo_y, fhi_y + + +def _solve_zoomed_at(pre, ar, img_w, img_h, p): + """Inner LP for one aspect candidate. Width is ar*h throughout; rent + is charged on h beyond the tightest aspect-respecting padded need, so + width the ratio forces is never billed.""" + m, loop = pre["m"], pre["loop"] + nt = m if loop else m - 1 + bx0, bx1, by0, by1 = pre["bx0"], pre["bx1"], pre["by0"], pre["by1"] + padx, pady = pre["padx"], pre["pady"] + extw = bx1 - bx0 + 1 + exth = by1 - by0 + 1 + soft = 1.0 / max(p.get("pad_deficit_tol", 16), 1e-6) + # need is aspect-aware (width the ratio forces is never billed), but + # the excess is billed at its (1+ar) box magnitude so wide and narrow + # candidates pay the same for the same true slack + rent = (1.0 + ar) * pre["rent_scale"] / max(p.get("pad_surplus_tol", 16), 1e-6) + mc = p.get("movement_cost", 1.0) + sc = p.get("resize_cost", 2.0) * (1.0 + ar) + eps = p.get("center_pull", 1e-4) + + hmax = min(img_h, img_w / ar) + hmin = np.maximum(np.maximum(exth, extw / ar), 1.0) + if hmin.max() > hmax + 1e-6: + return None # this aspect cannot contain the subject in-image + need_h = np.minimum( + np.maximum(exth + 2 * pady, (extw + 2 * padx) / ar), hmax) + + floors = pre["floor"] > 0 + if floors: + flo_x, fhi_x, flo_y, fhi_y = _yielded_floors(pre, ar, img_w, img_h) + + lp = _LP() + ub_x = np.maximum(bx0, 0) + ub_y = np.maximum(by0, 0) + if floors: + ub_x = np.minimum(ub_x, fhi_x) + ub_y = np.minimum(ub_y, fhi_y) + lp.var("x", m, lo=0.0, hi=ub_x) + lp.var("y", m, lo=0.0, hi=ub_y) + lp.var("h", m, lo=hmin, hi=hmax) + lp.var("tx", nt, cost=mc) + lp.var("ty", nt, cost=mc) + lp.var("th", nt, cost=sc) + lp.var("vx", m, cost=soft) + lp.var("vy", m, cost=soft) + lp.var("r", m, cost=rent) + lp.var("ux", m, cost=eps) + lp.var("uy", m, cost=eps) + + _tv_rows(lp, "x", "tx", m, loop) + _tv_rows(lp, "y", "ty", m, loop) + _tv_rows(lp, "h", "th", m, loop) + + rate = p.get("max_zoom_rate", 0.0) + if rate and rate > 1.0: + for i in range(nt): + a, b = lp.off["h"] + i, lp.off["h"] + (i + 1) % m + lp.row([b, a], [1, -rate], 0) + lp.row([a, b], [1, -rate], 0) + + for i in range(m): + x, y, h = lp.off["x"] + i, lp.off["y"] + i, lp.off["h"] + i + vx, vy = lp.off["vx"] + i, lp.off["vy"] + i + r = lp.off["r"] + i + ux, uy = lp.off["ux"] + i, lp.off["uy"] + i + lp.row([x, h], [-1, -ar], -(bx1[i] + 1)) # containment right + lp.row([x, h], [1, ar], img_w) # image right + lp.row([y, h], [-1, -1], -(by1[i] + 1)) + lp.row([y, h], [1, 1], img_h) + lp.row([x, vx], [1, -1], pre["lx"][i]) + lp.row([x, h, vx], [-1, -ar, -1], -pre["rx"][i]) + lp.row([y, vy], [1, -1], pre["ly"][i]) + lp.row([y, h, vy], [-1, -1, -1], -pre["ry"][i]) + lp.row([h, r], [1, -1], need_h[i]) + if floors: + lp.row([x, h], [-1, -ar], -flo_x[i]) + lp.row([y, h], [-1, -1], -flo_y[i]) + cx = (bx0[i] + bx1[i] + 1) / 2 + cy = (by0[i] + by1[i] + 1) / 2 + lp.row([x, h, ux], [1, 0.5 * ar, -1], cx) + lp.row([x, h, ux], [-1, -0.5 * ar, -1], -cx) + lp.row([y, h, uy], [1, 0.5, -1], cy) + lp.row([y, h, uy], [-1, -0.5, -1], -cy) + + res = lp.solve() + o = lp.off + # the objective is the sweep score: floors carry the padding tier as + # hard rows, so candidates compare on how cheaply they serve the + # cell's own priced trade above them + viol = res.x[o["vx"]:o["vx"] + m].sum() + res.x[o["vy"]:o["vy"] + m].sum() + h_t = res.x[o["h"]:o["h"] + m] + # dead width -- width this ratio forces beyond the frame's padded + # horizontal need -- is billed in the score only, at the same rent as + # height slack. Inside the LP that width stays free (billing it there + # collapses manual wide aspects), but across candidates it + # must cost, or the sweep drifts to the widest feasible shape whenever + # the vertical need pins the height. Capped at need_h: excess above + # need_h already carries its width via the (1+ar) rent. + w_need = np.minimum(extw + 2 * padx, img_w) + dead_w = np.maximum(ar * np.minimum(h_t, need_h) - w_need, 0) + dead = ((pre["rent_scale"] * dead_w).sum() + / max(p.get("pad_surplus_tol", 16), 1e-6)) + return { + "ar": ar, "score": res.fun + dead, "viol": viol, + "x": res.x[o["x"]:o["x"] + m], "y": res.x[o["y"]:o["y"] + m], + "h": h_t, + } + + +def _aspect_candidates(pre, img_w, img_h, n_known): + """Hull of the per-frame shape quantiles and the padded union aspect: + the winner lives between what the subject looks like and where it + goes.""" + bx0, bx1, by0, by1 = pre["bx0"], pre["bx1"], pre["by0"], pre["by1"] + padx, pady = pre["padx"], pre["pady"] + frame_ar = (bx1 - bx0 + 1) / (by1 - by0 + 1) + qs = np.quantile(frame_ar, [0.05, 0.25, 0.5, 0.75, 0.95]) + ux0 = (bx0 - padx).min() + ux1 = (bx1 + 1 + padx).max() + uy0 = (by0 - pady).min() + uy1 = (by1 + 1 + pady).max() + union_ar = max(ux1 - ux0, 1.0) / max(uy1 - uy0, 1.0) + lo = min(qs.min(), union_ar) + hi = max(qs.max(), union_ar) + grid = np.geomspace(lo, hi, 7) if hi > lo * 1.02 else np.array([lo]) + cands = np.concatenate([grid, qs, [union_ar]]) + # feasibility band: the box must contain every frame inside the image + extw = bx1 - bx0 + 1 + exth = by1 - by0 + 1 + band_lo = extw.max() / img_h + band_hi = img_w / exth.max() + cands = np.clip(cands, band_lo, band_hi) + cands = np.unique(np.round(cands, 3)) + keep = [cands[0]] + for a in cands[1:]: + if a > keep[-1] * 1.02: + keep.append(a) + return keep + + +def _quantize_steps(v, step, upper): + base = v.min() + k = np.ceil(np.log(v / base) / np.log(step) - 1e-9) + return np.minimum(base * step ** k, upper) + + +def plan(binary, sel, p, divisible_by=16): + """binary: (n, H, W) bool mask stack. Returns (boxes, info).""" + n, img_h, img_w = binary.shape + if not binary.any(): + raise ValueError("all masks are empty, nothing to plan") + scale = p["crop_scale"] + + if sel == "combined": + ys, xs = np.nonzero(binary.any(axis=0)) + w0 = (xs.max() - xs.min() + 1) * scale + h0 = (ys.max() - ys.min() + 1) * scale + ar = p.get("aspect_ratio", 0.0) + if ar > 0: + if w0 < h0 * ar: + w0 = h0 * ar + else: + h0 = w0 / ar + w = min(math.ceil(w0 / divisible_by) * divisible_by, + (img_w // divisible_by) * divisible_by or img_w) + h = min(math.ceil(h0 / divisible_by) * divisible_by, + (img_h // divisible_by) * divisible_by or img_h) + x = min(max(0, round((xs.min() + xs.max() + 1) / 2 - w / 2)), img_w - w) + y = min(max(0, round((ys.min() + ys.max() + 1) / 2 - h / 2)), img_h - h) + box = {"x": int(x), "y": int(y), "width": int(w), "height": int(h)} + return [box] * n, {"aspect": w / h} + + pre = _prep(binary, img_w, img_h, p) + m, s0 = pre["m"], pre["s0"] + bx0, bx1, by0, by1 = pre["bx0"], pre["bx1"], pre["by0"], pre["by1"] + + if sel == "tracked": + Wf, Hf, xt, yt = _solve_tracked(pre, img_w, img_h, p) + w = min(math.ceil(Wf / divisible_by) * divisible_by, + (img_w // divisible_by) * divisible_by or img_w) + h = min(math.ceil(Hf / divisible_by) * divisible_by, + (img_h // divisible_by) * divisible_by or img_h) + lo_x = np.maximum(bx1 + 1 - w, 0) + hi_x = np.minimum(np.maximum(bx0, 0), img_w - w) + lo_y = np.maximum(by1 + 1 - h, 0) + hi_y = np.minimum(np.maximum(by0, 0), img_h - h) + if pre["floor"] > 0: + lo_x = np.maximum(lo_x, np.minimum(pre["flo_x"] - w, hi_x)) + hi_x = np.minimum(hi_x, np.maximum(pre["fhi_x"], lo_x)) + lo_y = np.maximum(lo_y, np.minimum(pre["flo_y"] - h, hi_y)) + hi_y = np.minimum(hi_y, np.maximum(pre["fhi_y"], lo_y)) + xi = _round_corridor(xt, lo_x, hi_x)[s0:s0 + n] + yi = _round_corridor(yt, lo_y, hi_y)[s0:s0 + n] + boxes = [{"x": int(xi[i]), "y": int(yi[i]), + "width": int(w), "height": int(h)} for i in range(n)] + return boxes, {"aspect": w / h, "W": Wf, "H": Hf} + + # zoomed + ar_manual = p.get("aspect_ratio", 0.0) + if ar_manual > 0: + extw = bx1 - bx0 + 1 + exth = by1 - by0 + 1 + ar = min(max(ar_manual, extw.max() / img_h), img_w / exth.max()) + best = _solve_zoomed_at(pre, ar, img_w, img_h, p) + if best is None: + raise RuntimeError("requested aspect_ratio cannot contain the subject") + tried = [best] + else: + tried = [] + for ar in _aspect_candidates(pre, img_w, img_h, pre["known"].sum()): + sol = _solve_zoomed_at(pre, ar, img_w, img_h, p) + if sol is not None: + tried.append(sol) + if not tried: + raise RuntimeError("no feasible aspect candidate") + best = min(tried, key=lambda s: s["score"]) + for ar in (best["ar"] * 0.93, best["ar"] * 1.075): + sol = _solve_zoomed_at(pre, ar, img_w, img_h, p) + if sol is not None and sol["score"] < best["score"]: + best = sol + + ar = best["ar"] + h = best["h"] + if p.get("zoom_step", 1.0) > 1.0: + h = _quantize_steps(h, p["zoom_step"], min(img_h, img_w / ar)) + w = np.minimum(h * ar, img_w) + bw = np.minimum(np.ceil(w - 1e-6), img_w).astype(int) + bh = np.minimum(np.ceil(h - 1e-6), img_h).astype(int) + lo_x = np.maximum(bx1 + 1 - bw, 0) + hi_x = np.minimum(np.maximum(bx0, 0), img_w - bw) + lo_y = np.maximum(by1 + 1 - bh, 0) + hi_y = np.minimum(np.maximum(by0, 0), img_h - bh) + if pre["floor"] > 0: + flo_x, fhi_x, flo_y, fhi_y = _yielded_floors(pre, ar, img_w, img_h) + lo_x = np.maximum(lo_x, np.minimum(flo_x - bw, hi_x)) + hi_x = np.minimum(hi_x, np.maximum(fhi_x, lo_x)) + lo_y = np.maximum(lo_y, np.minimum(flo_y - bh, hi_y)) + hi_y = np.minimum(hi_y, np.maximum(fhi_y, lo_y)) + xi = _round_corridor(best["x"], lo_x, hi_x)[s0:s0 + n] + yi = _round_corridor(best["y"], lo_y, hi_y)[s0:s0 + n] + bw, bh = bw[s0:s0 + n], bh[s0:s0 + n] + boxes = [{"x": int(min(max(xi[i], 0), img_w - bw[i])), + "y": int(min(max(yi[i], 0), img_h - bh[i])), + "width": int(bw[i]), "height": int(bh[i])} for i in range(n)] + return boxes, {"aspect": ar, "swept": ar_manual <= 0, + "candidates": len(tried)} diff --git a/bfs_subject_crop.py b/bfs_subject_crop.py new file mode 100644 index 0000000..516e1f2 --- /dev/null +++ b/bfs_subject_crop.py @@ -0,0 +1,291 @@ +"""Stable subject-crop planning, vendored from drozbay/MaskVidExperiments. + +Source: https://github.com/drozbay/MaskVidExperiments (commit d98cc89, v0.2.0) +Copyright (c) drozbay. Licensed GPL-3.0, the same licence this pack carries, +which is what makes vendoring it legal here. + +Vendored rather than imported so the head-swap sampler has no runtime +dependency on that pack being installed. Behaviour is unchanged; only the +ComfyUI node classes were dropped (they stay in the original pack, where they +belong) and the planner import was repointed. + +Why this code and not a crop of my own: naive per-frame crops around a mask +jitter in position and size, and a video model reads that jitter as camera +motion. These boxes are planned over the whole clip, so they hold still through +mask noise and occlusion and follow only sustained movement. +""" + +"""Subject-tracked batch crop/uncrop for video mask workflows. The crop +boxes come from planner.py, which plans position, size, and shape jointly +over the whole clip.""" + +import logging +import math + +import numpy as np +import torch +from scipy import ndimage + +import comfy.utils + +from . import bfs_crop_planner as planner + +CATEGORY = "MaskVidExperiments" + + +def _clean_mask_stack(masks, value_thresh, min_pixels, min_frames): + """Binarize a mask batch and drop noise blobs. + + Connected components are labelled in 3D (time + space). A blob survives if + its peak single-frame area clears min_pixels and it is either subject-sized + (peak >= 50% of the largest blob, so fast motion that splits the subject + into per-frame components doesn't delete it) or persists min_frames frames. + """ + binary = masks.detach().cpu().numpy() > value_thresh + if not binary.any(): + return binary + + labels, count = ndimage.label(binary, structure=np.ones((3, 3, 3), dtype=bool)) + if count == 0: + return binary + + span = np.zeros(count + 1, dtype=np.int64) + for label, location in enumerate(ndimage.find_objects(labels), start=1): + if location is not None: + span[label] = location[0].stop - location[0].start + + peak = np.zeros(count + 1, dtype=np.int64) + for frame in labels: + peak = np.maximum(peak, np.bincount(frame.ravel(), minlength=count + 1)) + + span[0] = 0 + peak[0] = 0 + subject = peak >= 0.5 * peak.max() if peak.max() > 0 else np.zeros_like(peak, dtype=bool) + keep = (peak >= min_pixels) & (subject | (span >= min_frames)) + keep[0] = False + return keep[labels] + + +def _open_reconstruct(masks, value_thresh, radius, min_frames): + """Morphological opening by reconstruction on a mask batch. + + Erodes each frame spatially so blobs thinner than ~2*radius vanish, then + instead of dilating back, keeps the original shape of every blob with a + surviving core, so the subject's silhouette (thin protrusions included) is + untouched. Blobs are 3D components (time + space): a core surviving in any + frame keeps the whole track. border_value=1 so subjects cut off by the + frame edge are not eroded from that side. + """ + binary = masks.detach().cpu().numpy() > value_thresh + if not binary.any(): + return binary + + spatial = np.zeros((3, 3, 3), dtype=bool) + spatial[1] = [[0, 1, 0], [1, 1, 1], [0, 1, 0]] + eroded = ndimage.binary_erosion(binary, structure=spatial, iterations=radius, border_value=1) + + labels, count = ndimage.label(binary, structure=np.ones((3, 3, 3), dtype=bool)) + if count == 0: + return binary + + keep = np.zeros(count + 1, dtype=bool) + keep[np.unique(labels[eroded])] = True + if min_frames > 1: + for label, location in enumerate(ndimage.find_objects(labels), start=1): + if location is not None and location[0].stop - location[0].start < min_frames: + keep[label] = False + keep[0] = False + return keep[labels] + + +def _resize_image(img_hwc, w, h, method="lanczos"): + if img_hwc.shape[0] == h and img_hwc.shape[1] == w: + return img_hwc + return comfy.utils.common_upscale( + img_hwc.movedim(-1, 0).unsqueeze(0), w, h, method, "disabled" + ).squeeze(0).movedim(0, -1) + + +def _resize_mask(mask_hw, w, h, method="bilinear"): + if mask_hw.shape[0] == h and mask_hw.shape[1] == w: + return mask_hw + return comfy.utils.common_upscale( + mask_hw[None, None], w, h, method, "disabled" + )[0, 0] + + +def _upscale_size(w, h, megapixels, divisible_by): + """Size scaling a w x h crop to about abs(megapixels) pixels on the + divisible_by grid. A positive value is a floor: None when already at or + above it. A negative value is a target, reached by downscaling too.""" + scale = math.sqrt(abs(megapixels) * 1024 * 1024 / (w * h)) + if megapixels > 0 and scale <= 1.0: + return None + return (max(1, round(w * scale / divisible_by)) * divisible_by, + max(1, round(h * scale / divisible_by)) * divisible_by) + + +def _cell_params(mode): + """Raw planner dials for the standard node's (padding, prefer) cell. + The advanced node feeds the same dials directly. guaranteed floors the + full promise against every raw frame (floor window 1): a true + guarantee, so mask noise must be cleaned upstream. firm floors 70% + against the sustained tracks; flexible has no floor. In tracked mode + the crop never rescales, so prefer differentiates through the oversize + rent instead: stillness holds a bigger box to move less. In zoomed + mode the rent is never scaled by prefer (cheapening it tips whole + clips into one constant maximum size, killing zoom-follow); prefer + differentiates through the resize cost.""" + sel = mode["mode"] + pad_level = mode.get("padding", "firm") + prefer = mode.get("prefer", "stillness") + floor = {"guaranteed": 1.0, "firm": 0.7, "flexible": 0.0}[pad_level] + still = prefer == "stillness" + if sel == "tracked": + oversize = 32.0 if still else 8.0 + else: + oversize = float(mode.get("pad_surplus_tol", 16)) + return { + "crop_scale": mode["crop_scale"], + "min_padding_allowed": floor, + "min_padding_allowed_window": 1 if pad_level == "guaranteed" else 16, + "pad_deficit_tol": 16.0, + "pad_surplus_tol": oversize, + "resize_cost": 2.0 if still else 1.0, + "movement_cost": 1.0, + "center_pull": 1e-4, + "end_tightening": 0.0, + "end_tightening_window": 80 if floor > 0 else 0, + "zoom_step": mode.get("zoom_step", 1.0), + "max_zoom_rate": 0.0, + "aspect_ratio": mode["aspect_ratio"], + "seamless_loop": mode.get("seamless_loop", False), + } + + +def _debug_summary(binary, boxes, scale, sel, info, output_size): + """Brief per-run report: what shape the planner chose and how well the + plan delivered the padding promise.""" + n = len(boxes) + worst = [] + img_h, img_w = binary.shape[1:] + for i in range(n): + ys, xs = binary[i].nonzero() + if len(xs) == 0: + worst.append(np.nan) + continue + b = boxes[i] + bx1 = b["x"] + b["width"] - 1 + by1 = b["y"] + b["height"] - 1 + wx = (scale - 1) / 2 * (xs.max() - xs.min() + 1) + wy = (scale - 1) / 2 * (ys.max() - ys.min() + 1) + fr = [] + if b["x"] > 0: fr.append((xs.min() - b["x"]) / max(wx, 1e-6)) + if bx1 < img_w - 1: fr.append((bx1 - xs.max()) / max(wx, 1e-6)) + if b["y"] > 0: fr.append((ys.min() - b["y"]) / max(wy, 1e-6)) + if by1 < img_h - 1: fr.append((by1 - ys.max()) / max(wy, 1e-6)) + worst.append(min(fr) if fr else np.nan) + worst = np.array(worst) + v = worst[~np.isnan(worst)] + cx = np.array([b["x"] + b["width"] / 2 for b in boxes]) + cy = np.array([b["y"] + b["height"] / 2 for b in boxes]) + travel = np.hypot(np.diff(cx), np.diff(cy)).sum() + + ar = info["aspect"] + if info.get("swept"): + shape = f"{ar:.2f} (chosen from {info['candidates']} candidates)" + elif sel == "zoomed": + shape = f"{ar:.2f} (manual)" + else: + shape = f"{ar:.2f}" + ws = [b["width"] for b in boxes] + hs = [b["height"] for b in boxes] + lines = [ + f"mode: {sel}", + f"frames: {n} ({img_w}x{img_h} source)", + f"aspect_ratio: {shape}", + f"smallest_box: {min(ws)}x{min(hs)}", + f"largest_box: {max(ws)}x{max(hs)}", + ] + if sel == "zoomed": + lines.append(f"zoom_ratio: {max(hs) / max(min(hs), 1):.2f} " + "(largest box / smallest box)") + lines.append(f"crop region resizing: {max(ws)} x {max(hs)} -> " + f"{output_size[0]} x {output_size[1]}") + lines.append(f"movement: {travel:.0f}px " + "(total distance the box center travels)") + if len(v) and scale > 1.0: + lines.append(f"padding_promised: {(scale - 1) / 2:.0%} of the " + f"subject's size on each side (crop_scale {scale:.2f})") + lines.append(f"padding_worst: {max(v.min(), 0):.0%} of promised") + lines.append(f"padding_typical: {max(np.median(v), 0):.0%} of promised") + return "\n".join(lines) + + +def _plan_and_crop(original_images, masks, sel, p, divisible_by, + mask_threshold, upscale_megapixels=0.0): + if original_images.shape[0] != masks.shape[0]: + raise ValueError(f"original_images ({original_images.shape[0]}) and masks ({masks.shape[0]}) must have the same frame count") + if original_images.shape[1:3] != masks.shape[1:3]: + raise ValueError(f"original_images ({original_images.shape[2]}x{original_images.shape[1]}) and masks ({masks.shape[2]}x{masks.shape[1]}) must have the same dimensions") + + if 0.0 < p["crop_scale"] < 1.0: + raise ValueError("crop_scale must be 0 (full frame) or at least 1.0") + full_frame = p["crop_scale"] <= 0.0 + + binary = masks.detach().cpu().numpy() > mask_threshold + if full_frame: + img_h, img_w = masks.shape[1], masks.shape[2] + boxes = [{"x": 0, "y": 0, "width": img_w, "height": img_h} + for _ in range(masks.shape[0])] + info = {"aspect": img_w / img_h} + else: + if not binary.any(): + raise ValueError("all masks are empty, nothing to crop") + boxes, info = planner.plan(binary, sel, p, divisible_by) + + bw = max(b["width"] for b in boxes) + bh = max(b["height"] for b in boxes) + size = None + img_method, mask_method = "lanczos", "bilinear" + if full_frame: + gw = math.ceil(bw / divisible_by) * divisible_by + gh = math.ceil(bh / divisible_by) * divisible_by + if (gw, gh) != (bw, bh): + size = (gw, gh) + elif sel == "zoomed": + # Target sized to the largest planned box: that stretch is ~1:1 + # and everything else upscales rather than losing detail. + th = math.ceil(bh / divisible_by) * divisible_by + size = (math.ceil(th * info["aspect"] / divisible_by) * divisible_by, th) + if upscale_megapixels != 0: + up = _upscale_size(*(size or (bw, bh)), upscale_megapixels, divisible_by) + if up is not None: + size = up + img_method, mask_method = "bicubic", "nearest-exact" + + debug = _debug_summary(binary, boxes, p["crop_scale"], + f"{sel} (full frame)" if full_frame else sel, + info, size or (bw, bh)) + + if size is not None: + tw, th = size + cropped_images = torch.stack([ + _resize_image(original_images[i, b["y"]:b["y"] + b["height"], b["x"]:b["x"] + b["width"], :], tw, th, img_method) + for i, b in enumerate(boxes) + ]) + cropped_masks = torch.stack([ + _resize_mask(masks[i, b["y"]:b["y"] + b["height"], b["x"]:b["x"] + b["width"]], tw, th, mask_method) + for i, b in enumerate(boxes) + ]) + else: + cropped_images = torch.stack([ + original_images[i, b["y"]:b["y"] + b["height"], b["x"]:b["x"] + b["width"], :] + for i, b in enumerate(boxes) + ]) + cropped_masks = torch.stack([ + masks[i, b["y"]:b["y"] + b["height"], b["x"]:b["x"] + b["width"]] + for i, b in enumerate(boxes) + ]) + return (cropped_images, cropped_masks, + [[b] for b in boxes], debug) diff --git a/docs/BFSHeadSwapMaskedSampler.md b/docs/BFSHeadSwapMaskedSampler.md new file mode 100644 index 0000000..6d7e08e --- /dev/null +++ b/docs/BFSHeadSwapMaskedSampler.md @@ -0,0 +1,110 @@ +# BFS Head Swap Sampler (crop · mask · loop) + +Runs an LTX head-swap LoRA and adds three things around it, each optional. +Connect only what you need — the node degrades to whatever is wired. + +| what you connect | what you get | +|---|---| +| `guide_video` + `identity_image` | plain head swap, one pass | +| `+ subject_mask` | only the masked region is denoised | +| `+ crop_mode` | the swap runs inside a box around the subject, then is pasted back | +| `+ temporal_tile_size` | long clips sampled in chunks with overlap | + +The LoRA is never asked to understand a mask or a crop. It keeps doing its one +job: the guide rides on `source_id 1`, the identity on `source_id 2`. Everything +else here is plumbing around it. + +## The mask is native inpainting, not a LoRA feature + +The mask becomes `latent["noise_mask"]`, which the guider consumes as +`denoise_mask` — ComfyUI's own inpainting path. Pixels outside the mask keep the +guide's own content, so **the original face stays visible to the model**. This is +deliberately different from a masked-hole recipe, where the region is painted out +before the model sees it. + +Masks are reduced to the latent grid with **max**, not with ComfyUI's trilinear +resize. Trilinear blurs a mask across frames and lets the original content bleed +through the edit. Max keeps a latent cell that any masked pixel touches fully +editable. + +**Watch the `latent_mask` output.** It has one frame per *latent* frame, not per +video frame, and that is the real resolution of the edit. A tight outline around +a face can nearly vanish at that scale — and then nothing changes, no matter how +good the LoRA is. If the mask looks thin there, raise `mask_grow`. + +## Why crop, and which mode + +At 512×288 a person filling a fifth of the frame leaves a face about 25 px tall. +No LoRA recovers identity from 25 px — it is missing pixels, not missing +training. Cropping the head region and sampling it full-frame gives that same +face 200–300 px, and the result is feathered back into untouched frames. + +| `crop_mode` | behaviour | +|---|---| +| `off` | the whole frame is sampled. Use it first, to confirm the swap works at all. | +| `combined` | one static box around the subject's whole travel, held for the clip. No dependency. | +| `tracked` | constant-size box that stays still until the subject would leave it, then moves as little as possible. | +| `zoomed` | the box also follows the subject's size, planned over the whole clip, every crop resampled to one output size. | + +`tracked` and `zoomed` use a planner vendored from +[drozbay/MaskVidExperiments](https://github.com/drozbay/MaskVidExperiments) +(GPL-3.0, the same licence this pack carries) — nothing extra to install. Those +boxes hold still through mask noise and occlusion, and **a jittering crop reads +to a video model as camera motion**, which is what a naive per-frame crop around +a mask produces. + +`crop_scale` is the box as a multiple of the subject. At 1.5 the subject occupies +two thirds and the rest is margin. Keep neck and shoulders inside: a face-tight +crop is a framing the LoRA never saw in training, and it will show. + +## Sizing, which is the one fiddly part + +The connected `latent` sets the sampled size. When you crop, the box is smaller +than the frame, so the latent should match the box — the `debug` output prints +the size to use: + +``` +sample size 416x768 (connect EmptyLTXVLatentVideo at this size to avoid a resize) +``` + +If they disagree the node resamples the crop on the way back, which costs exactly +the sharpness the crop was there to buy. Run once, read `debug`, set +`EmptyLTXVLatentVideo`, run again. + +**The latent must come from the graph.** LTX‑2.5 latents are AV — video and audio +nested together — and the node cannot fabricate that structure. Without one it +builds a plain video latent, warns, and the guide may not reach the model at all. + +## Chunking + +`temporal_tile_size` at 0 samples the clip in one pass. Set it to the length the +LoRA trained at (73 for the LTX head-swap recipe) for longer clips, and the node +slices the guide, the mask and the overlap together per chunk. + +Chunked sampling with an AV latent is refused rather than silently corrupted — +slicing a nested latent in time is not a thing the node does yet. + +## Reading the outputs + +| output | what it answers | +|---|---| +| `images` | the result, with the crop pasted back | +| `mask_over_source` | is the mask where I think it is? Red over the original frames, green box for the crop | +| `cropped_guide` | exactly what the model was fed. If this is wrong, nothing downstream can be right | +| `crop_mask` | the mask after grow/blur, in the crop's pixel space | +| `latent_mask` | the mask as the sampler sees it — where a thin mask disappears | +| `latent` | the sampled latent | +| `debug` | crop mode and box, mask ops, tiling, the size to build the latent at | + +## Suggested order when something looks wrong + +1. `crop_mode: off`, no mask, `temporal_tile_size: 0`. This is the plain swap; if + it fails, the problem is the recipe or the LoRA, not this node. +2. Add the mask. Check `latent_mask` before judging the result. +3. Add the crop. Check `mask_over_source` for the box, and set the latent to the + size `debug` reports. +4. Only then raise the frame count and turn on chunking. + +Sampling settings follow the LoRA's recipe: CFG 3–5 with a real negative, and no +LightX2V (CFG 1.0 means no amplification, and the reference stops being +amplified). diff --git a/headswap_ltx_masked_sampler.py b/headswap_ltx_masked_sampler.py index 115d5b1..b021afe 100644 --- a/headswap_ltx_masked_sampler.py +++ b/headswap_ltx_masked_sampler.py @@ -21,11 +21,11 @@ about 25 px tall. No LoRA recovers identity from that. Cropping the head region and sampling it full-frame gives the same face 200-300 px, then the result is feathered back into the untouched frames. -The crop planner comes from drozbay's MaskVidExperiments when that pack is -installed (https://github.com/drozbay/MaskVidExperiments) — its boxes hold still -through mask noise and occlusion, which naive per-frame crops do not, and a -jittering crop reads to a video model as camera motion. Without the pack the node -falls back to one static box around the subject's whole travel. +The crop planner is vendored from drozbay's MaskVidExperiments (GPL-3.0, same as +this pack) in `bfs_subject_crop.py`, so nothing here depends on that pack being +installed. Its boxes hold still through mask noise and occlusion, which naive +per-frame crops do not, and a jittering crop reads to a video model as camera +motion. """ import logging @@ -41,31 +41,32 @@ CATEGORY = "BFS/video" # ───────────────────────────────────────────────────────────────────────────── -# optional dependency: drozbay/MaskVidExperiments crop planner +# crop planning (vendored from drozbay/MaskVidExperiments, GPL-3.0) # ───────────────────────────────────────────────────────────────────────────── -def _mvex_planner(): - """`_plan_and_crop` from MaskVidExperiments, or None when it is not installed.""" - import importlib - import os - import sys +def _planner(): + """The vendored crop planner (bfs_subject_crop), or None if it fails to import.""" + try: + from .bfs_subject_crop import _plan_and_crop + return _plan_and_crop + except Exception as exc: # pragma: no cover + log.warning("vendored crop planner unavailable (%s); using the static box", exc) + return None - import folder_paths - for root in folder_paths.get_folder_paths("custom_nodes"): - path = os.path.join(root, "MaskVidExperiments", "nodes_subject_crop.py") - if not os.path.isfile(path): - continue - try: - spec = importlib.util.spec_from_file_location("_bfs_mvex_crop", path) - mod = importlib.util.module_from_spec(spec) - sys.modules["_bfs_mvex_crop"] = mod - spec.loader.exec_module(mod) - return mod._plan_and_crop - except Exception as exc: # pragma: no cover - depends on their internals - log.warning("MaskVidExperiments found but not usable (%s); using the static box", exc) - return None - return None +def _feather_ramp(h, w, feather, device): + """Blend weights that fade in from the crop border, as (1,h,w,1).""" + if feather <= 0: + return torch.ones(1, h, w, 1, device=device) + ramp = torch.ones(h, w, device=device) + f = min(int(feather), h // 2, w // 2) + if f > 0: + edge = torch.linspace(0, 1, f, device=device) + ramp[:f, :] *= edge[:, None] + ramp[-f:, :] *= edge.flip(0)[:, None] + ramp[:, :f] *= edge[None, :] + ramp[:, -f:] *= edge.flip(0)[None, :] + return ramp[None, :, :, None] def _static_box(masks, images, crop_scale, divisible_by, thresh=0.1): @@ -109,7 +110,7 @@ def _mask_to_latent(masks, vae, latent_t, latent_h, latent_w): ComfyUI would trilinearly resize the pixel mask instead, which blurs it across frames and lets the original content bleed through the edit -- the - failure MaskVidExperiments' Mask To Latent Space node was written to fix. + failure drozbay's Mask To Latent Space node was written to fix. Max keeps a latent cell that any masked pixel touches fully editable. """ m = masks.unsqueeze(1).float() # (N,1,H,W) @@ -195,8 +196,8 @@ class BFSHeadSwapMaskedSampler: "crop_mode": (["off", "combined", "tracked", "zoomed"], {"default": "off", "tooltip": "Sample inside a box around the subject instead of the whole frame -- the fix for faces " "that are too small to carry identity. combined: one static box for the clip. " - "tracked/zoomed: MaskVidExperiments' planner (installed separately); without it these " - "fall back to combined."}), + "tracked: a constant-size box that stays still until the subject would leave it. " + "zoomed: the box follows the subject's size too, planned over the whole clip."}), "crop_scale": ("FLOAT", {"default": 1.5, "min": 1.0, "max": 4.0, "step": 0.05, "tooltip": "Box size as a multiple of the subject. 1.5 leaves a third as margin. Keep neck and " "shoulders in: a face-tight crop is a framing the LoRA never saw in training."}), @@ -251,7 +252,7 @@ class BFSHeadSwapMaskedSampler: if mode == "off" or masks is None: return guide, masks, None, "crop: off" - planner = _mvex_planner() if mode in ("tracked", "zoomed") else None + planner = _planner() if mode in ("tracked", "zoomed") else None if planner is not None: try: p = {"crop_scale": scale, "aspect_ratio": 0.0, "padding": "firm", @@ -259,9 +260,9 @@ class BFSHeadSwapMaskedSampler: "pad_surplus_tol": 16, "zoom_step": 1.0} out = planner(guide, masks, mode, p, div, 0.1, 0.0) cropped, cropped_masks, bboxes = out[0], out[1], out[2] - return cropped, cropped_masks, ("mvex", bboxes), f"crop: mvex/{mode}" + return cropped, cropped_masks, ("planned", bboxes), f"crop: planned/{mode}" except Exception as exc: - log.warning("MaskVidExperiments planner failed (%s); using the static box", exc) + log.warning("crop planner failed (%s); using the static box", exc) x0, y0, w, h = _static_box(masks, guide, scale, div) cropped = guide[:, y0:y0 + h, x0:x0 + w, :] @@ -272,37 +273,33 @@ class BFSHeadSwapMaskedSampler: if ctx is None: return result kind, box = ctx - if kind == "mvex": - try: - import importlib - mod = importlib.import_module("_bfs_mvex_crop") - return mod.MVEx_SubjectUncropNode.execute(result, original, box, feather).result[0] - except Exception as exc: - log.warning("MaskVidExperiments uncrop failed (%s); pasting the static way", exc) - return result + if kind == "planned": + # one box per frame: paste each crop into its own box + out = original.clone() + n = min(result.shape[0], out.shape[0], len(box)) + for i in range(n): + b = box[i][0] if isinstance(box[i], list) else box[i] + x0, y0, w, h = (int(b[0]), int(b[1]), int(b[2]), int(b[3])) + patch = result[i:i + 1] + if patch.shape[1] != h or patch.shape[2] != w: + patch = comfy.utils.common_upscale( + patch.movedim(-1, 1), w, h, "lanczos", "disabled").movedim(1, -1) + a = _feather_ramp(h, w, feather, patch.device) + out[i:i + 1, y0:y0 + h, x0:x0 + w, :] = ( + a * patch + (1 - a) * out[i:i + 1, y0:y0 + h, x0:x0 + w, :]) + return out x0, y0, w, h = box out = original.clone()[: result.shape[0]] patch = result[: out.shape[0]] # The sampled crop does not have to match the box: the connected latent - # sets the sampled size, and MaskVidExperiments' zoomed mode rescales + # sets the sampled size, and the zoomed crop mode rescales # crops by design. Bring it back to the box before blending. if patch.shape[1] != h or patch.shape[2] != w: log.info("paste-back: resizing the sampled crop %dx%d -> box %dx%d", patch.shape[2], patch.shape[1], w, h) patch = comfy.utils.common_upscale( patch.movedim(-1, 1), w, h, "lanczos", "disabled").movedim(1, -1) - if feather > 0: - ramp = torch.ones(h, w, device=patch.device) - f = min(feather, h // 2, w // 2) - if f > 0: - edge = torch.linspace(0, 1, f, device=patch.device) - ramp[:f, :] *= edge[:, None] - ramp[-f:, :] *= edge.flip(0)[:, None] - ramp[:, :f] *= edge[None, :] - ramp[:, -f:] *= edge.flip(0)[None, :] - a = ramp[None, :, :, None] - else: - a = 1.0 + a = _feather_ramp(h, w, feather, patch.device) out[:, y0:y0 + h, x0:x0 + w, :] = ( a * patch + (1 - a) * out[:, y0:y0 + h, x0:x0 + w, :] ) diff --git a/pyproject.toml b/pyproject.toml index f5e7d75..9993b81 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "bfsnodes" description = "Custom ComfyUI nodes including LTXV Edit Anything, Head-Swap/Anime2Real Bernini conditioning, the AMV Guide Builder, LTX Identity Transfer (IPT2V reference + source-phase RoPE), and MiniMax-H3 single-frame VAE loaders with tile control." -version = "1.31.1" +version = "1.32.0" license = {file = "LICENSE"} dependencies = [ "torch", @@ -13,6 +13,7 @@ dependencies = [ "insightface==0.7.3", "onnxruntime", "transformers>=4.55.0", + "scipy", ] # classifiers = [ # # For OS-independent nodes (works on all operating systems)