Files

2389 lines
93 KiB
Python

from __future__ import annotations
import gc
import json
from typing import Any
import torch
import torch.nn.functional as F
try:
from .scail2_context import apply_scail2_easy_context
except ImportError:
from scail2_context import apply_scail2_easy_context
CATEGORY = "SCAIL-2/Simple"
RESOLUTION_PRESETS = ("512p", "704p", "custom")
LONG_VIDEO_MODES = ("chunk", "context_sampling")
REFERENCE_PACK_TYPE = "SCAIL2_REFERENCE_PACK"
REFERENCE_IMAGE_INPUT_TYPE = f"IMAGE,{REFERENCE_PACK_TYPE}"
MAX_REFERENCE_SUBJECTS = 6
MAX_LEGACY_REFERENCE_IMAGES_PER_SUBJECT = 6
MAX_PREFIX_REFERENCE_IMAGES = 5
MAX_MIXED_REFERENCE_IMAGES = 5
MAX_STAGE_REFERENCE_SOURCE_HEIGHT = 2048
SCAIL_COLOR_PALETTE = (
(0.0, 0.0, 1.0),
(1.0, 0.0, 0.0),
(0.0, 1.0, 0.0),
(1.0, 0.0, 1.0),
(0.0, 1.0, 1.0),
(1.0, 1.0, 0.0),
)
def _shape(value: Any) -> list[int]:
return list(value.shape) if hasattr(value, "shape") else []
def _clamp_int(value: int, low: int, high: int) -> int:
return max(low, min(high, int(value)))
def _is_reference_pack(value: Any) -> bool:
return isinstance(value, dict) and value.get("type") == REFERENCE_PACK_TYPE
def _track_object_count(track_data: Any) -> int:
if not isinstance(track_data, dict):
return 0
packed = track_data.get("packed_masks")
if isinstance(packed, torch.Tensor) and packed.ndim >= 2:
return int(packed.shape[1])
scores = track_data.get("scores")
if scores is not None:
try:
return int(len(scores))
except TypeError:
return 0
return 0
def _round_32(value: int) -> int:
return max(32, (int(value) // 32) * 32)
def _round_nearest_32(value: float) -> int:
return max(32, int((float(value) + 16) // 32) * 32)
def _wan_frame_count_cover(value: int) -> int:
value = max(1, int(value))
if value == 1:
return 1
return 1 + ((value - 1 + 3) // 4) * 4
def _wan_frame_count_floor(value: int) -> int:
value = max(1, int(value))
return 1 + ((value - 1) // 4) * 4
def _trim_frames(value: torch.Tensor | None, frame_count: int) -> torch.Tensor | None:
if value is None:
return None
if not isinstance(value, torch.Tensor):
raise TypeError("Image inputs must be ComfyUI IMAGE tensors.")
if value.ndim != 4:
raise ValueError("Image inputs must have shape [frames, height, width, channels].")
return value[:frame_count].detach()
def _first_image(value: torch.Tensor, name: str) -> torch.Tensor:
if not isinstance(value, torch.Tensor) or value.ndim != 4:
raise ValueError(f"{name} must be a ComfyUI IMAGE tensor.")
if value.shape[0] <= 0:
raise ValueError(f"{name} has no images.")
return value[:1].detach().contiguous()
def _node_result(value: Any) -> tuple:
if hasattr(value, "result"):
result = value.result
if result is None:
return ()
return tuple(result)
if isinstance(value, tuple):
return value
return (value,)
def _get_scail_nodes_module():
try:
from comfy_extras import nodes_scail
return nodes_scail
except ImportError:
from comfy_extras import nodes_wan
return nodes_wan
def _extract_scail_mask_to_28ch(rgb_video: torch.Tensor) -> torch.Tensor:
if rgb_video.ndim != 4 or rgb_video.shape[-1] < 3:
raise ValueError("SCAIL mask video must have shape [frames, height, width, channels].")
native_extract = getattr(_get_scail_nodes_module(), "_extract_mask_to_28ch", None)
if native_extract is not None:
return native_extract(rgb_video)
frames, height, width, _ = rgb_video.shape
threshold = 225.0 / 255.0
mask = rgb_video[..., :3].movedim(-1, 1).float()
red = (mask[:, 0:1] > threshold).float()
green = (mask[:, 1:2] > threshold).float()
blue = (mask[:, 2:3] > threshold).float()
not_red = 1.0 - red
not_green = 1.0 - green
not_blue = 1.0 - blue
channels = torch.cat(
[
red * green * blue,
red * not_green * not_blue,
not_red * green * not_blue,
not_red * not_green * blue,
red * green * not_blue,
red * not_green * blue,
not_red * green * blue,
],
dim=1,
)
latent_height = (int(height) + 7) // 8
latent_width = (int(width) + 7) // 8
channels = F.interpolate(channels, size=(latent_height, latent_width), mode="area")
latent_frames = ((int(frames) - 1) // 4) + 1
packed = torch.cat([channels[:1].repeat(4, 1, 1, 1), channels[1:]], dim=0)
return packed.reshape(latent_frames, 28, latent_height, latent_width).unsqueeze(0)
def _infer_generation_size(pose_video: torch.Tensor) -> tuple[int, int]:
if pose_video.ndim != 4:
raise ValueError("pose_video must be a ComfyUI IMAGE tensor.")
height = _round_32(int(pose_video.shape[1]))
width = _round_32(int(pose_video.shape[2]))
return width, height
def _empty_cache(force: bool = False):
if force:
gc.collect()
try:
import comfy.model_management
comfy.model_management.cleanup_models_gc()
try:
comfy.model_management.soft_empty_cache(force=force)
except TypeError:
comfy.model_management.soft_empty_cache()
except Exception:
if torch.cuda.is_available():
torch.cuda.empty_cache()
def _target_size_for_video(images: torch.Tensor, resolution: str, custom_width: int = 832, custom_height: int = 480) -> tuple[int, int]:
if images.ndim != 4:
raise ValueError("video must be a ComfyUI IMAGE tensor.")
height = int(images.shape[1])
width = int(images.shape[2])
if resolution == "custom":
return _round_nearest_32(max(32, int(custom_width))), _round_nearest_32(max(32, int(custom_height)))
target_short = 704 if resolution == "704p" else 512
source_short = max(1, min(width, height))
scale = target_short / source_short
target_width = _round_nearest_32(width * scale)
target_height = _round_nearest_32(height * scale)
return target_width, target_height
def _resize_to_target(images: torch.Tensor, target_width: int, target_height: int) -> torch.Tensor:
if images.ndim != 4:
raise ValueError("video must be a ComfyUI IMAGE tensor.")
source_height = int(images.shape[1])
source_width = int(images.shape[2])
if source_width == target_width and source_height == target_height:
return images.detach().contiguous()
dtype = images.dtype
chunks = []
for start in range(0, int(images.shape[0]), 32):
chunk = images[start : start + 32].detach().float().movedim(-1, 1)
chunk = F.interpolate(
chunk,
size=(target_height, target_width),
mode="bilinear",
align_corners=False,
)
chunks.append(chunk.movedim(1, -1).to(dtype=dtype).contiguous())
return torch.cat(chunks, dim=0).clamp(0, 1)
def _resize_mask_like(mask: torch.Tensor, image: torch.Tensor) -> torch.Tensor:
if mask.shape[1] == image.shape[1] and mask.shape[2] == image.shape[2]:
return mask.detach()
resized = F.interpolate(
mask.detach().float().movedim(-1, 1),
size=(int(image.shape[1]), int(image.shape[2])),
mode="nearest",
)
return resized.movedim(1, -1).to(dtype=image.dtype).contiguous()
def _apply_reference_mask(reference_image: torch.Tensor, reference_image_mask: torch.Tensor) -> torch.Tensor:
mask = _resize_mask_like(reference_image_mask[:1], reference_image[:1])
alpha = (mask[..., :3].max(dim=-1, keepdim=True).values > 0.1).to(dtype=reference_image.dtype)
return (reference_image[:1].detach() * alpha).contiguous()
def _float_list(value: torch.Tensor) -> list[float]:
return [round(float(item), 4) for item in value.detach().cpu().flatten().tolist()]
def _match_chunk_color_to_overlap(
frames: torch.Tensor,
current_overlap: torch.Tensor | None,
reference_overlap: torch.Tensor | None,
strength: float = 0.65,
) -> tuple[torch.Tensor, dict[str, Any]]:
info: dict[str, Any] = {"applied": False}
if frames.ndim != 4 or frames.shape[-1] < 3 or frames.shape[0] <= 0:
info["reason"] = "invalid_frames"
return frames, info
if current_overlap is None or reference_overlap is None:
info["reason"] = "missing_overlap"
return frames, info
overlap_count = min(int(current_overlap.shape[0]), int(reference_overlap.shape[0]))
if overlap_count <= 0:
info["reason"] = "empty_overlap"
return frames, info
if current_overlap.ndim != 4 or reference_overlap.ndim != 4:
info["reason"] = "invalid_overlap"
return frames, info
current = current_overlap[-overlap_count:, :, :, :3].detach()
reference = reference_overlap[-overlap_count:, :, :, :3].detach()
if current.shape[1:3] != reference.shape[1:3]:
reference = F.interpolate(
reference.float().movedim(-1, 1),
size=(int(current.shape[1]), int(current.shape[2])),
mode="bilinear",
align_corners=False,
).movedim(1, -1)
device = frames.device
current = current.to(device=device, dtype=torch.float32)
reference = reference.to(device=device, dtype=torch.float32)
current_mean = current.mean(dim=(0, 1, 2))
reference_mean = reference.mean(dim=(0, 1, 2))
current_std = current.std(dim=(0, 1, 2), unbiased=False).clamp_min(1e-4)
reference_std = reference.std(dim=(0, 1, 2), unbiased=False).clamp_min(1e-4)
scale = (reference_std / current_std).clamp(0.7, 1.35)
shift = (reference_mean - current_mean * scale).clamp(-0.25, 0.25)
strength = max(0.0, min(1.0, float(strength)))
corrected_rgb = frames[:, :, :, :3].to(torch.float32) * scale.view(1, 1, 1, 3) + shift.view(1, 1, 1, 3)
blended_rgb = torch.lerp(frames[:, :, :, :3].to(torch.float32), corrected_rgb, strength).clamp(0, 1)
corrected = frames.clone()
corrected[:, :, :, :3] = blended_rgb.to(dtype=frames.dtype)
info.update(
{
"applied": True,
"overlap_frames": int(overlap_count),
"strength": float(strength),
"scale": _float_list(scale),
"shift": _float_list(shift),
}
)
return corrected.contiguous(), info
def _strip_latent_for_decode(latent: dict) -> dict:
if not isinstance(latent, dict) or "samples" not in latent:
raise RuntimeError("SamplerCustom returned an invalid latent output.")
samples = latent["samples"]
if hasattr(samples, "detach"):
samples = samples.detach()
if hasattr(samples, "contiguous"):
samples = samples.contiguous()
return {"samples": samples}
def _sample_for_decode(
*,
model,
positive,
negative,
sampler,
sigmas,
latent,
seed: int,
cfg: float,
) -> dict:
from comfy_extras.nodes_custom_sampler import SamplerCustom
sampled = _node_result(
SamplerCustom.execute(
model,
True,
int(seed),
float(cfg),
positive,
negative,
sampler,
sigmas,
latent,
)
)
if not sampled:
raise RuntimeError("SamplerCustom returned no latent output.")
latent_to_decode = sampled[1] if len(sampled) > 1 else sampled[0]
decode_latent = _strip_latent_for_decode(latent_to_decode)
del sampled, latent_to_decode
return decode_latent
def _decode_latent_to_frames(vae, latent_to_decode: dict) -> torch.Tensor:
import nodes
decoded = nodes.VAEDecode().decode(vae, latent_to_decode)[0]
frames = decoded.detach().cpu().contiguous().clamp(0, 1)
del decoded, latent_to_decode
_empty_cache()
return frames
def _run_native_scail_chunk(
*,
model,
positive,
negative,
vae,
sampler,
sigmas,
reference_image: torch.Tensor,
pose_video: torch.Tensor,
clip_vision_output,
pose_video_mask: torch.Tensor | None,
reference_image_mask: torch.Tensor | None,
previous_frames: torch.Tensor | None,
width: int,
height: int,
length: int,
video_frame_offset: int,
previous_frame_count: int,
replacement_mode: bool,
seed: int,
cfg: float,
pose_strength: float,
) -> tuple[torch.Tensor, int, dict]:
WanSCAILToVideo = _get_scail_nodes_module().WanSCAILToVideo
scail_out = _node_result(
WanSCAILToVideo.execute(
positive,
negative,
vae,
width,
height,
length,
1,
pose_strength,
0.0,
1.0,
video_frame_offset,
previous_frame_count,
replacement_mode=replacement_mode,
reference_image=reference_image,
clip_vision_output=clip_vision_output,
pose_video=pose_video,
pose_video_mask=pose_video_mask,
reference_image_mask=reference_image_mask,
previous_frames=previous_frames,
)
)
if len(scail_out) != 4:
raise RuntimeError("WanSCAILToVideo returned an unexpected output shape.")
chunk_positive, chunk_negative, latent, next_offset = scail_out
latent_to_decode = _sample_for_decode(
model=model,
positive=chunk_positive,
negative=chunk_negative,
sampler=sampler,
sigmas=sigmas,
latent=latent,
seed=seed,
cfg=cfg,
)
del scail_out, chunk_positive, chunk_negative, latent
reference_image = pose_video = clip_vision_output = pose_video_mask = reference_image_mask = previous_frames = None
_empty_cache(force=True)
frames = _decode_latent_to_frames(vae, latent_to_decode)
summary = {
"width": int(width),
"height": int(height),
"length": int(length),
"seed": int(seed),
"cfg": float(cfg),
"input_video_frame_offset": int(video_frame_offset),
"output_video_frame_offset": int(next_offset),
"decoded_shape": _shape(frames),
}
return frames, int(next_offset), summary
def _create_scail_masks(driving_track_data, reference_track_data, replacement_mode: bool) -> tuple[torch.Tensor, torch.Tensor]:
SCAIL2ColoredMask = getattr(_get_scail_nodes_module(), "SCAIL2ColoredMask", None)
if SCAIL2ColoredMask is None:
raise RuntimeError("SCAIL2ColoredMask is unavailable in this ComfyUI build.")
masks = _node_result(
SCAIL2ColoredMask.execute(
driving_track_data,
"",
"left_to_right",
bool(replacement_mode),
ref_track_data=reference_track_data,
)
)
if len(masks) != 2:
raise RuntimeError("SCAIL2ColoredMask returned an unexpected output shape.")
return masks[0], masks[1]
def _create_driving_scail_mask(driving_track_data, replacement_mode: bool) -> torch.Tensor:
SCAIL2ColoredMask = getattr(_get_scail_nodes_module(), "SCAIL2ColoredMask", None)
if SCAIL2ColoredMask is None:
raise RuntimeError("SCAIL2ColoredMask is unavailable in this ComfyUI build.")
masks = _node_result(
SCAIL2ColoredMask.execute(
driving_track_data,
"",
"left_to_right",
bool(replacement_mode),
)
)
if not masks:
raise RuntimeError("SCAIL2ColoredMask returned no driving mask.")
return masks[0]
def _run_sam3_track(images: torch.Tensor, model, conditioning, detection_threshold: float, max_objects: int, detect_interval: int):
from comfy_extras.nodes_sam3 import SAM3_VideoTrack
if conditioning is None:
raise ValueError("SCAIL-2 Reference SAM Builder needs a SAM3 conditioning input, usually CLIPTextEncode('person').")
result = _node_result(
SAM3_VideoTrack.execute(
images,
model,
initial_mask=None,
conditioning=conditioning,
detection_threshold=float(detection_threshold),
max_objects=int(max_objects),
detect_interval=max(1, int(detect_interval)),
)
)
if len(result) != 1:
raise RuntimeError("SAM3_VideoTrack returned an unexpected output shape.")
return result[0]
def _subject_color(subject_index: int) -> tuple[float, float, float]:
return SCAIL_COLOR_PALETTE[int(subject_index) % len(SCAIL_COLOR_PALETTE)]
def _solid_color_mask_like(image: torch.Tensor, color: tuple[float, float, float], alpha: torch.Tensor | None = None) -> torch.Tensor:
mask = torch.empty((1, int(image.shape[1]), int(image.shape[2]), 3), device=image.device, dtype=image.dtype)
mask[..., 0] = color[0]
mask[..., 1] = color[1]
mask[..., 2] = color[2]
if alpha is not None:
mask = mask * alpha.to(device=image.device, dtype=image.dtype)
return mask.contiguous()
def _render_reference_sam_mask(
reference_image: torch.Tensor,
track_data,
subject_index: int,
) -> torch.Tensor:
raw_mask = _create_driving_scail_mask(track_data, False)
raw_mask = _resize_mask_like(raw_mask[:1], reference_image[:1])
alpha = (raw_mask[..., :3].max(dim=-1, keepdim=True).values > 0.1).to(dtype=reference_image.dtype)
return _solid_color_mask_like(reference_image[:1], _subject_color(subject_index), alpha)
def _render_mixed_reference_sam_mask(reference_image: torch.Tensor, track_data) -> torch.Tensor:
from comfy.ldm.sam3.tracker import unpack_masks
output = torch.zeros((1, int(reference_image.shape[1]), int(reference_image.shape[2]), 3), device=reference_image.device, dtype=reference_image.dtype)
if not isinstance(track_data, dict):
return output.contiguous()
packed = track_data.get("packed_masks")
if not isinstance(packed, torch.Tensor) or packed.ndim < 2 or int(packed.shape[1]) <= 0:
return output.contiguous()
masks = unpack_masks(packed.to(reference_image.device)).float()
if masks.ndim != 4 or int(masks.shape[0]) <= 0 or int(masks.shape[1]) <= 0:
return output.contiguous()
frame_masks = masks[:1]
n_obj = int(frame_masks.shape[1])
height = int(reference_image.shape[1])
width = int(reference_image.shape[2])
frame_masks = F.interpolate(
frame_masks.reshape(n_obj, 1, int(frame_masks.shape[-2]), int(frame_masks.shape[-1])),
size=(height, width),
mode="nearest",
).reshape(n_obj, height, width) > 0.5
if not bool(frame_masks.any().item()):
return output.contiguous()
areas = frame_masks.float().sum(dim=(1, 2)).clamp(min=1.0)
grid_x = torch.arange(width, device=frame_masks.device, dtype=torch.float32).view(1, width)
centers = (frame_masks.float() * grid_x).sum(dim=(1, 2)) / areas
order = torch.argsort(centers)
frame_masks = frame_masks[order]
any_mask = frame_masks.any(dim=0)
obj_idx_map = frame_masks.to(torch.uint8).argmax(dim=0)
colors = torch.tensor(
[SCAIL_COLOR_PALETTE[index % len(SCAIL_COLOR_PALETTE)] for index in range(int(frame_masks.shape[0]))],
device=reference_image.device,
dtype=reference_image.dtype,
)
color_overlay = colors[obj_idx_map].unsqueeze(0)
return torch.where(any_mask.unsqueeze(0).unsqueeze(-1), color_overlay, output).contiguous()
def _mask_with_background(mask: torch.Tensor, *, white_background: bool) -> torch.Tensor:
if not white_background:
return mask.contiguous().clamp(0, 1)
alpha = (mask[..., :3].max(dim=-1, keepdim=True).values > 0.1).to(device=mask.device, dtype=mask.dtype)
return (mask + (1.0 - alpha)).contiguous().clamp(0, 1)
def _encode_clip_vision(clip_vision, image: torch.Tensor):
import nodes
return nodes.CLIPVisionEncode().encode(clip_vision, image, "none")[0]
def _sample_and_decode(
*,
model,
positive,
negative,
vae,
sampler,
sigmas,
latent,
seed: int,
cfg: float,
) -> torch.Tensor:
latent_to_decode = _sample_for_decode(
model=model,
positive=positive,
negative=negative,
sampler=sampler,
sigmas=sigmas,
latent=latent,
seed=seed,
cfg=cfg,
)
return _decode_latent_to_frames(vae, latent_to_decode)
def _reference_pack_primary_image(reference_pack: dict) -> torch.Tensor:
for subject in reference_pack.get("subjects", []):
images = subject.get("images", [])
if images:
return images[0]
scenes = reference_pack.get("scene_images", [])
if scenes:
return scenes[0]
raise ValueError("Reference Pack has no images.")
def _reference_pack_clip_image(reference_pack: dict, *, replacement_mode: bool) -> torch.Tensor:
return _reference_pack_primary_image(reference_pack)
def _colored_reference_mask(item: dict[str, Any], image: torch.Tensor, *, white_background: bool) -> torch.Tensor:
if item["kind"] == "background":
return _solid_color_mask_like(image[:1], (1.0, 1.0, 1.0))
color = _subject_color(int(item["subject_index"]))
if item.get("mask") is not None:
source_mask = _resize_mask_like(item["mask"][:1], image[:1])
alpha = (source_mask[..., :3].max(dim=-1, keepdim=True).values > 0.1).to(dtype=image.dtype)
mask = _solid_color_mask_like(image[:1], color, alpha)
if white_background:
mask = mask + (1.0 - alpha.to(device=mask.device, dtype=mask.dtype))
return mask.clamp(0, 1).contiguous()
return _solid_color_mask_like(image[:1], color)
def _resize_image_exact(image: torch.Tensor, width: int, height: int, *, mode: str = "bicubic") -> torch.Tensor:
if image.ndim != 4:
raise ValueError("Reference images must have shape [frames, height, width, channels].")
if int(image.shape[1]) == int(height) and int(image.shape[2]) == int(width):
return image[:1, :, :, :3].detach().contiguous()
tensor = image[:1, :, :, :3].detach().float().movedim(-1, 1)
kwargs: dict[str, Any] = {}
if mode in {"bilinear", "bicubic"}:
kwargs["align_corners"] = False
resized = F.interpolate(tensor, size=(int(height), int(width)), mode=mode, **kwargs)
return resized.movedim(1, -1).to(dtype=image.dtype).clamp(0, 1).contiguous()
def _resize_image_cover(image: torch.Tensor, width: int, height: int, *, mode: str = "bicubic") -> torch.Tensor:
if image.ndim != 4:
raise ValueError("Reference images must have shape [frames, height, width, channels].")
target_width = max(1, int(width))
target_height = max(1, int(height))
source_height = int(image.shape[1])
source_width = int(image.shape[2])
if source_height <= 0 or source_width <= 0:
raise ValueError("Reference image dimensions must be greater than zero.")
crop = image[:1, :, :, :3].detach()
if source_width * target_height > source_height * target_width:
crop_width = max(1, min(source_width, round(source_height * target_width / target_height)))
left = max(0, (source_width - crop_width) // 2)
crop = crop[:, :, left:left + crop_width, :]
elif source_width * target_height < source_height * target_width:
crop_height = max(1, min(source_height, round(source_width * target_height / target_width)))
top = max(0, (source_height - crop_height) // 2)
crop = crop[:, top:top + crop_height, :, :]
return _resize_image_exact(crop, target_width, target_height, mode=mode)
def _resize_image_contain(
image: torch.Tensor,
width: int,
height: int,
*,
fill: tuple[float, float, float] = (0.0, 0.0, 0.0),
mode: str = "bicubic",
) -> torch.Tensor:
if image.ndim != 4:
raise ValueError("Reference images must have shape [frames, height, width, channels].")
target_width = max(1, int(width))
target_height = max(1, int(height))
source_height = int(image.shape[1])
source_width = int(image.shape[2])
if source_height <= 0 or source_width <= 0:
raise ValueError("Reference image dimensions must be greater than zero.")
scale = min(target_width / source_width, target_height / source_height)
resized_width = max(1, min(target_width, int(round(source_width * scale))))
resized_height = max(1, min(target_height, int(round(source_height * scale))))
resized = _resize_image_exact(image, resized_width, resized_height, mode=mode)
canvas = torch.empty((1, target_height, target_width, 3), device=resized.device, dtype=resized.dtype)
canvas[..., 0] = fill[0]
canvas[..., 1] = fill[1]
canvas[..., 2] = fill[2]
paste_x = max(0, (target_width - resized_width) // 2)
paste_y = max(0, (target_height - resized_height) // 2)
canvas[:, paste_y:paste_y + resized_height, paste_x:paste_x + resized_width, :] = resized[:, :, :, :3]
return canvas.contiguous().clamp(0, 1)
def _resize_mask_contain_like(mask: torch.Tensor, source_image: torch.Tensor, target_image: torch.Tensor) -> torch.Tensor:
target_width = int(target_image.shape[2])
target_height = int(target_image.shape[1])
source_height = int(source_image.shape[1])
source_width = int(source_image.shape[2])
scale = min(target_width / max(1, source_width), target_height / max(1, source_height))
resized_width = max(1, min(target_width, int(round(source_width * scale))))
resized_height = max(1, min(target_height, int(round(source_height * scale))))
resized = _resize_image_exact(mask[:1], resized_width, resized_height, mode="nearest")
canvas = torch.zeros((1, target_height, target_width, 3), device=resized.device, dtype=resized.dtype)
paste_x = max(0, (target_width - resized_width) // 2)
paste_y = max(0, (target_height - resized_height) // 2)
canvas[:, paste_y:paste_y + resized_height, paste_x:paste_x + resized_width, :] = resized[:, :, :, :3]
return canvas.contiguous().clamp(0, 1)
def _resize_mask_cover_like(mask: torch.Tensor, source_image: torch.Tensor, width: int, height: int) -> torch.Tensor:
target_width = max(1, int(width))
target_height = max(1, int(height))
source_height = int(source_image.shape[1])
source_width = int(source_image.shape[2])
crop = mask[:1, :, :, :3].detach()
if source_width * target_height > source_height * target_width:
crop_width = max(1, min(source_width, round(source_height * target_width / target_height)))
left = max(0, (source_width - crop_width) // 2)
crop = crop[:, :, left:left + crop_width, :]
elif source_width * target_height < source_height * target_width:
crop_height = max(1, min(source_height, round(source_width * target_height / target_width)))
top = max(0, (source_height - crop_height) // 2)
crop = crop[:, top:top + crop_height, :, :]
return _resize_image_exact(crop, target_width, target_height, mode="nearest")
def _resize_stage_subject_to_height(
image: torch.Tensor,
mask: torch.Tensor | None,
target_height: int,
) -> tuple[torch.Tensor, torch.Tensor | None]:
source_height = max(1, int(image.shape[1]))
source_width = max(1, int(image.shape[2]))
target_height = max(1, int(target_height))
target_width = max(1, int(round(source_width * (target_height / source_height))))
if source_height == target_height:
return image, _resize_mask_like(mask[:1], image[:1]) if mask is not None else None
resized_image = _resize_image_exact(image[:1], target_width, target_height, mode="bilinear")
resized_mask = None
if mask is not None:
mask_like_image = _resize_mask_like(mask[:1], image[:1])
resized_mask = _resize_image_exact(mask_like_image, target_width, target_height, mode="nearest")
return resized_image.contiguous(), resized_mask.contiguous() if resized_mask is not None else None
def _alpha_from_mask(mask: torch.Tensor | None, image: torch.Tensor) -> torch.Tensor:
if mask is None:
return torch.ones((1, int(image.shape[1]), int(image.shape[2]), 1), device=image.device, dtype=image.dtype)
resized = _resize_mask_like(mask[:1], image[:1])
return (resized[..., :3].max(dim=-1, keepdim=True).values > 0.1).to(device=image.device, dtype=image.dtype)
def _alpha_bbox(alpha: torch.Tensor, padding: float = 0.06) -> tuple[int, int, int, int]:
plane = alpha[0, :, :, 0] > 0.05
coords = torch.nonzero(plane, as_tuple=False)
height = int(alpha.shape[1])
width = int(alpha.shape[2])
if coords.numel() == 0:
return 0, 0, width, height
y0 = int(coords[:, 0].min().item())
y1 = int(coords[:, 0].max().item()) + 1
x0 = int(coords[:, 1].min().item())
x1 = int(coords[:, 1].max().item()) + 1
pad = int(max(width, height) * float(padding))
return max(0, x0 - pad), max(0, y0 - pad), min(width, x1 + pad), min(height, y1 + pad)
def _reference_canvas_aspect(width: int, height: int) -> str:
ratio = float(width) / max(1.0, float(height))
if ratio < 0.84:
return "portrait"
if ratio > 1.18:
return "landscape"
return "square"
def _visual_stage_rows(count: int, row_count: int, max_per_row: int) -> list[list[int]] | None:
count = max(1, int(count))
row_count = max(1, int(row_count))
max_per_row = max(1, int(max_per_row))
rows = [[] for _ in range(row_count)]
for index in range(count):
rows[index % row_count].append(index)
if any(len(row) == 0 or len(row) > max_per_row for row in rows):
return None
return rows
def _stage_row_candidates(count: int, aspect: str) -> list[list[list[int]]]:
count = max(1, int(count))
if count <= 1:
return [[[0]]]
if aspect == "portrait":
if count == 3:
return [[[0, 2], [1]]]
max_rows = 1 if count <= 2 else (3 if count >= 5 else 2)
max_per_row = 3
elif aspect == "square":
max_rows = 1 if count <= 3 else (3 if count >= 5 else 2)
max_per_row = 4
else:
max_rows = 1 if count <= 4 else 2
max_per_row = 6
candidates: list[list[list[int]]] = []
for row_count in range(1, min(max_rows, count) + 1):
rows = _visual_stage_rows(count, row_count, max_per_row)
if rows is not None:
candidates.append(rows)
return candidates or [[list(range(count))]]
def _stage_row_profile(row_count: int, row_index: int, aspect: str) -> dict[str, float]:
if row_count <= 1:
max_h = 0.96 if aspect != "square" else 0.92
return {"base": 0.985, "max_h": max_h, "row_w": 0.94, "overlap": 0.10}
if row_count == 2 and aspect == "portrait":
profiles = [
{"base": 0.74, "max_h": 0.78, "row_w": 1.04, "overlap": 0.24},
{"base": 1.02, "max_h": 0.92, "row_w": 0.96, "overlap": 0.16},
]
elif row_count == 2 and aspect == "square":
profiles = [
{"base": 0.78, "max_h": 0.66, "row_w": 0.92, "overlap": 0.14},
{"base": 0.995, "max_h": 0.74, "row_w": 0.88, "overlap": 0.12},
]
elif row_count == 2:
profiles = [
{"base": 0.74, "max_h": 0.58, "row_w": 0.94, "overlap": 0.14},
{"base": 0.99, "max_h": 0.66, "row_w": 0.90, "overlap": 0.12},
]
elif aspect == "portrait":
profiles = [
{"base": 0.56, "max_h": 0.48, "row_w": 0.92, "overlap": 0.16},
{"base": 0.78, "max_h": 0.56, "row_w": 0.94, "overlap": 0.15},
{"base": 1.02, "max_h": 0.64, "row_w": 0.90, "overlap": 0.13},
]
elif aspect == "square":
profiles = [
{"base": 0.54, "max_h": 0.44, "row_w": 0.90, "overlap": 0.16},
{"base": 0.76, "max_h": 0.52, "row_w": 0.92, "overlap": 0.15},
{"base": 0.99, "max_h": 0.60, "row_w": 0.88, "overlap": 0.13},
]
else:
profiles = [
{"base": 0.50, "max_h": 0.40, "row_w": 0.94, "overlap": 0.16},
{"base": 0.73, "max_h": 0.48, "row_w": 0.94, "overlap": 0.15},
{"base": 0.99, "max_h": 0.56, "row_w": 0.90, "overlap": 0.13},
]
return profiles[min(int(row_index), len(profiles) - 1)]
def _subject_bbox_metrics(image: torch.Tensor, mask: torch.Tensor | None) -> dict[str, float]:
alpha = _alpha_from_mask(mask, image)
x0, y0, x1, y1 = _alpha_bbox(alpha)
crop_w = max(1, int(x1 - x0))
crop_h = max(1, int(y1 - y0))
return {
"crop_w": float(crop_w),
"crop_h": float(crop_h),
"image_w": float(max(1, int(image.shape[2]))),
"image_h": float(max(1, int(image.shape[1]))),
}
def _effective_row_width(widths: list[float], overlap: float) -> float:
if not widths:
return 1.0
effective = float(sum(widths))
for index in range(1, len(widths)):
effective -= min(widths[index - 1], widths[index]) * float(overlap)
return max(1.0, effective)
def _stage_layout_from_rows(
entries: list[dict[str, Any]],
rows: list[list[int]],
canvas_w: int,
canvas_h: int,
aspect: str,
) -> dict[str, Any]:
fit_scales: list[float] = []
row_profiles: list[dict[str, float]] = []
for row_index, row in enumerate(rows):
profile = _stage_row_profile(len(rows), row_index, aspect)
row_profiles.append(profile)
row_widths = [entries[index]["metrics"]["crop_w"] for index in row]
fit_scales.append((float(canvas_w) * float(profile["row_w"])) / _effective_row_width(row_widths, profile["overlap"]))
for index in row:
metrics = entries[index]["metrics"]
fit_scales.append((float(canvas_h) * float(profile["max_h"])) / max(1.0, metrics["crop_h"]))
group_scale = max(0.001, min(fit_scales) if fit_scales else 1.0)
specs: list[dict[str, float] | None] = [None for _ in entries]
for row_index, row in enumerate(rows):
profile = row_profiles[row_index]
overlap = float(profile["overlap"])
scaled_widths = [entries[index]["metrics"]["crop_w"] * group_scale for index in row]
total_width = _effective_row_width(scaled_widths, overlap)
left = (float(canvas_w) - total_width) * 0.5
for local_index, index in enumerate(row):
width = scaled_widths[local_index]
center_x = (left + width * 0.5) / max(1.0, float(canvas_w))
specs[index] = {
"x": max(0.0, min(1.0, center_x)),
"base": float(profile["base"]),
"max_w": 1.0,
"max_h": 1.0,
"row_index": float(row_index),
"row_count": float(len(rows)),
"row_w": float(profile["row_w"]),
"row_max_h": float(profile["max_h"]),
"overlap": overlap,
}
if local_index + 1 < len(row):
next_width = scaled_widths[local_index + 1]
left += width - min(width, next_width) * overlap
row_lengths = [len(row) for row in rows]
balance_penalty = (max(row_lengths) - min(row_lengths)) * 0.002 if row_lengths else 0.0
row_penalty = max(0, len(rows) - 1) * 0.012
layout_score = group_scale * max(0.90, 1.0 - row_penalty) - balance_penalty
return {
"specs": [spec if spec is not None else {"x": 0.5, "base": 0.985, "max_w": 1.0, "max_h": 1.0} for spec in specs],
"scales": [group_scale for _ in entries],
"rows": rows,
"group_scale": group_scale,
"score": layout_score,
}
def _layout_stage_entries(entries: list[dict[str, Any]], canvas_w: int, canvas_h: int) -> tuple[list[dict[str, float]], list[float | None]]:
count = len(entries)
if count <= 0:
return [], []
if count == 1:
return [{"x": 0.5, "base": 0.985, "max_w": 0.86, "max_h": 0.96, "row_index": 0.0}], [None]
aspect = _reference_canvas_aspect(canvas_w, canvas_h)
candidates = _stage_row_candidates(count, aspect)
best = None
for rows in candidates:
layout = _stage_layout_from_rows(entries, rows, canvas_w, canvas_h, aspect)
if best is None or (layout["score"], layout["group_scale"], -len(layout["rows"])) > (
best["score"],
best["group_scale"],
-len(best["rows"]),
):
best = layout
if best is None:
return [{"x": 0.5, "base": 0.985, "max_w": 1.0, "max_h": 1.0} for _ in entries], [None for _ in entries]
return best["specs"], best["scales"]
def _paste_subject_cutout(
canvas: torch.Tensor,
mask_canvas: torch.Tensor,
image: torch.Tensor,
mask: torch.Tensor | None,
color: tuple[float, float, float],
spec: dict[str, float],
scale_override: float | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
alpha = _alpha_from_mask(mask, image)
x0, y0, x1, y1 = _alpha_bbox(alpha)
crop = image[:, y0:y1, x0:x1, :3]
crop_alpha = alpha[:, y0:y1, x0:x1, :]
if crop.shape[1] <= 0 or crop.shape[2] <= 0:
return canvas, mask_canvas
canvas_w = int(canvas.shape[2])
canvas_h = int(canvas.shape[1])
target_w = max(1, int(round(canvas_w * float(spec["max_w"]))))
target_h = max(1, int(round(canvas_h * float(spec["max_h"]))))
if scale_override is None:
scale = min(target_w / max(1, int(crop.shape[2])), target_h / max(1, int(crop.shape[1])))
else:
scale = min(
float(scale_override),
target_w / max(1, int(crop.shape[2])),
target_h / max(1, int(crop.shape[1])),
)
new_w = max(1, min(canvas_w, int(round(crop.shape[2] * scale))))
new_h = max(1, min(canvas_h, int(round(crop.shape[1] * scale))))
resized_crop = F.interpolate(crop.float().movedim(-1, 1), size=(new_h, new_w), mode="bilinear", align_corners=False).movedim(1, -1)
resized_alpha = F.interpolate(crop_alpha.float().movedim(-1, 1), size=(new_h, new_w), mode="nearest").movedim(1, -1)
resized_crop = resized_crop.to(device=canvas.device, dtype=canvas.dtype).clamp(0, 1)
resized_alpha = resized_alpha.to(device=canvas.device, dtype=canvas.dtype).clamp(0, 1)
center_x = int(round(canvas_w * float(spec["x"])))
baseline_y = int(round(canvas_h * float(spec["base"])))
paste_x = center_x - new_w // 2
paste_y = baseline_y - new_h
paste_x = max(0, min(canvas_w - new_w, paste_x))
paste_y = max(0, min(canvas_h - new_h, paste_y))
region = canvas[:, paste_y:paste_y + new_h, paste_x:paste_x + new_w, :3]
canvas[:, paste_y:paste_y + new_h, paste_x:paste_x + new_w, :3] = (
resized_crop * resized_alpha + region * (1.0 - resized_alpha)
).contiguous()
color_tensor = torch.tensor(color, device=mask_canvas.device, dtype=mask_canvas.dtype).view(1, 1, 1, 3)
mask_region = mask_canvas[:, paste_y:paste_y + new_h, paste_x:paste_x + new_w, :3]
mask_canvas[:, paste_y:paste_y + new_h, paste_x:paste_x + new_w, :3] = torch.where(
resized_alpha > 0.05,
color_tensor.expand_as(mask_region),
mask_region,
)
return canvas.contiguous(), mask_canvas.contiguous()
def _compose_main_reference(
reference_pack: dict,
*,
width: int,
height: int,
replacement_mode: bool,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None, list[dict[str, Any]]]:
subjects = list(reference_pack.get("subjects", []))
if not subjects:
raise ValueError("Reference Pack has no usable subject images.")
scene_images = reference_pack.get("scene_images", [])
scene_resized = _resize_image_cover(scene_images[0], width, height) if (scene_images and not replacement_mode) else None
single_subject = len(subjects) == 1
subject_summaries: list[dict[str, Any]] = []
first_subject = subjects[0]
first_image = _first_image(first_subject["images"][0], "subject_1_image")
first_mask = (first_subject.get("masks") or [None])[0] if first_subject.get("masks") else None
if single_subject and scene_resized is None and not replacement_mode:
main_image = _resize_image_cover(first_image, width, height)
main_mask_source = (
_resize_mask_cover_like(first_mask, first_image, width, height)
if first_mask is not None
else None
)
main_mask = _colored_reference_mask(
{
"kind": "subject",
"subject_index": int(first_subject.get("index", 0)),
"mask": main_mask_source,
},
main_image,
white_background=True,
)
subject_summaries.append(
{
"subject_index": int(first_subject.get("index", 0)),
"main_shape": _shape(first_image),
"mask_source": first_subject.get("mask_source", "full_image"),
"canvas_role": "single_main_preserve_background_center_crop",
}
)
return main_image, main_mask, scene_resized, subject_summaries
if scene_resized is not None:
canvas = scene_resized.clone()
mask_canvas = _solid_color_mask_like(canvas, (1.0, 1.0, 1.0))
base_role = "scene_background"
elif replacement_mode:
canvas = torch.zeros((1, int(height), int(width), 3), device=first_image.device, dtype=first_image.dtype)
mask_canvas = _solid_color_mask_like(canvas, (0.0, 0.0, 0.0))
base_role = "black_background"
else:
canvas = torch.ones((1, int(height), int(width), 3), device=first_image.device, dtype=first_image.dtype)
mask_canvas = _solid_color_mask_like(canvas, (1.0, 1.0, 1.0))
base_role = "white_background"
stage_entries: list[dict[str, Any]] = []
for subject_position, subject in enumerate(subjects):
subject_index = int(subject.get("index", subject_position))
images = subject.get("images", [])
if not images:
continue
image = _first_image(images[0], f"subject_{subject_index + 1}_image")
masks = subject.get("masks") or []
mask = masks[0] if masks else None
stage_entries.append(
{
"subject": subject,
"subject_index": subject_index,
"image": image,
"mask": mask,
"metrics": _subject_bbox_metrics(image, mask),
}
)
if len(stage_entries) > 1:
target_stage_height = min(
MAX_STAGE_REFERENCE_SOURCE_HEIGHT,
max(max(1, int(entry["image"].shape[1])) for entry in stage_entries),
)
for entry in stage_entries:
image, mask = _resize_stage_subject_to_height(entry["image"], entry["mask"], target_stage_height)
entry["image"] = image
entry["mask"] = mask
entry["metrics"] = _subject_bbox_metrics(image, mask)
stage_specs, stage_scales = _layout_stage_entries(stage_entries, width, height)
for entry, stage_spec, stage_scale in zip(stage_entries, stage_specs, stage_scales):
subject = entry["subject"]
subject_index = int(entry["subject_index"])
image = entry["image"]
mask = entry["mask"]
canvas, mask_canvas = _paste_subject_cutout(
canvas,
mask_canvas,
image,
mask,
_subject_color(subject_index),
stage_spec,
stage_scale,
)
subject_summaries.append(
{
"subject_index": subject_index,
"main_shape": _shape(image),
"mask_source": subject.get("mask_source", "full_image") if mask is not None else "full_image",
"canvas_role": "stage_cutout",
"stage_spec": stage_spec,
"stage_scale": None if stage_scale is None else round(float(stage_scale), 6),
}
)
return canvas.contiguous().clamp(0, 1), mask_canvas.contiguous().clamp(0, 1), scene_resized, [
{"base_role": base_role, "subjects": subject_summaries}
]
def _mixed_reference_items(reference_pack: dict, *, limit: int) -> tuple[list[dict[str, Any]], int]:
items: list[dict[str, Any]] = []
masks = reference_pack.get("reference_masks") or []
for image_index, image in enumerate(reference_pack.get("reference_images", [])):
items.append(
{
"kind": "mixed_reference",
"image_index": int(image_index),
"image": image,
"mask": masks[image_index] if image_index < len(masks) else None,
"mask_source": reference_pack.get("reference_mask_source", "full_image") if image_index < len(masks) else "full_image",
"role": "prefix",
}
)
dropped = max(0, len(items) - max(0, int(limit)))
return items[: max(0, int(limit))], dropped
def _prepare_mixed_reference(
item: dict[str, Any],
*,
width: int,
height: int,
replacement_mode: bool,
scene_image: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, dict[str, Any]]:
source_image = item["image"]
if replacement_mode:
image = _resize_image_contain(source_image, width, height)
fitted_mask = _resize_mask_contain_like(item.get("mask"), source_image, image) if item.get("mask") is not None else None
else:
image = _resize_image_cover(source_image, width, height)
fitted_mask = _resize_mask_cover_like(item.get("mask"), source_image, width, height) if item.get("mask") is not None else None
alpha = _alpha_from_mask(fitted_mask, image) if fitted_mask is not None else None
alpha_cropped = False
scene_composited = False
if alpha is not None and replacement_mode:
image = (image * alpha.to(device=image.device, dtype=image.dtype)).contiguous()
alpha_cropped = True
elif alpha is not None and scene_image is not None:
scene = scene_image.to(device=image.device, dtype=image.dtype)
if scene.shape[1] != image.shape[1] or scene.shape[2] != image.shape[2]:
scene = _resize_image_exact(scene, int(image.shape[2]), int(image.shape[1]))
alpha = alpha.to(device=image.device, dtype=image.dtype)
image = (image * alpha + scene * (1.0 - alpha)).contiguous()
scene_composited = True
if fitted_mask is not None:
reference_mask = _mask_with_background(
fitted_mask,
white_background=not replacement_mode,
)
else:
reference_mask = _solid_color_mask_like(image, (1.0, 1.0, 1.0) if not replacement_mode else (0.0, 0.0, 0.0))
summary = {
"kind": "mixed_reference",
"conditioning_role": "prefix",
"image_index": int(item["image_index"]),
"source_shape": _shape(item["image"]),
"mask_source": item.get("mask_source", "full_image"),
"alpha_cropped": bool(alpha_cropped),
"scene_composited": bool(scene_composited),
}
return image.contiguous().clamp(0, 1), reference_mask.contiguous().clamp(0, 1), summary
def _encode_reference_sequence(
vae,
images: list[torch.Tensor],
masks: list[torch.Tensor],
*,
mask_condition_enabled: bool,
) -> tuple[torch.Tensor, torch.Tensor | None, list[list[int]]]:
latents = []
mask_latents = []
latent_shapes = []
for image, mask in zip(images, masks):
latent = vae.encode(image[:1, :, :, :3])
if latent.ndim != 5:
raise RuntimeError(f"SCAIL-2 reference VAE encode returned unexpected shape: {_shape(latent)}")
latents.append(latent)
latent_shapes.append(_shape(latent))
if mask_condition_enabled:
mask_latents.append(_extract_scail_mask_to_28ch(mask[:1, :, :, :3]))
if not latents:
raise ValueError("Reference Pack has no reference images to encode.")
reference_latent = torch.cat(latents, dim=2).detach().cpu().contiguous()
reference_mask = torch.cat(mask_latents, dim=1).detach().cpu().contiguous() if mask_condition_enabled and mask_latents else None
return reference_latent, reference_mask, latent_shapes
def _prepare_scail_reference_pack(
*,
vae,
reference_pack: dict,
width: int,
height: int,
replacement_mode: bool,
) -> dict[str, Any]:
background_images = len(reference_pack.get("scene_images", []))
include_background = not replacement_mode
background_enabled = bool(include_background and background_images > 0)
subject_count = len(reference_pack.get("subjects", []))
subject_image_count = sum(1 for subject in reference_pack.get("subjects", []) if subject.get("images"))
mixed_reference_count = len(reference_pack.get("reference_images", []))
multi_reference = bool(mixed_reference_count > 0 or background_enabled or subject_count > 1)
has_subject_masks = any(subject.get("masks") for subject in reference_pack.get("subjects", []))
has_reference_masks = bool(reference_pack.get("reference_masks"))
mask_condition_enabled = bool(replacement_mode or subject_count > 1)
if mask_condition_enabled:
missing_masks = []
for subject_position, subject in enumerate(reference_pack.get("subjects", [])):
images = subject.get("images", [])
masks = subject.get("masks") or []
for image_index in range(len(images)):
if image_index >= len(masks) or masks[image_index] is None:
missing_masks.append((subject_position, image_index))
reference_masks = reference_pack.get("reference_masks") or []
for image_index, _image in enumerate(reference_pack.get("reference_images", [])):
if image_index >= len(reference_masks) or reference_masks[image_index] is None:
missing_masks.append(("reference", image_index))
if missing_masks:
raise ValueError(
"This Reference Pack needs subject/reference masks. Connect Reference Pack through "
"SCAIL-2 Reference SAM Builder before SCAIL-2 Simple Video."
)
main_image, main_mask, scene_resized, main_summary = _compose_main_reference(
reference_pack,
width=width,
height=height,
replacement_mode=replacement_mode,
)
prefix_budget = MAX_PREFIX_REFERENCE_IMAGES
mixed_items, dropped_mixed = _mixed_reference_items(reference_pack, limit=prefix_budget)
reference_images: list[torch.Tensor] = [main_image]
reference_masks: list[torch.Tensor] = [main_mask]
item_summaries: list[dict[str, Any]] = [
{
"kind": "main_reference",
"conditioning_role": "primary",
"subjects": int(subject_count),
"source": main_summary,
"background_composited": bool(scene_resized is not None),
"mask_background": "black" if replacement_mode else "white",
}
]
for item in mixed_items:
image, mask, summary_item = _prepare_mixed_reference(
item,
width=width,
height=height,
replacement_mode=replacement_mode,
scene_image=scene_resized,
)
reference_images.append(image)
reference_masks.append(mask)
item_summaries.append(summary_item)
# Scene images are folded into the primary reference collage. Feeding them
# as a separate background-only prefix can pull attention on the first frame.
background_prefix_used = False
ref_latent, ref_mask_1f, latent_shapes = _encode_reference_sequence(
vae,
reference_images,
reference_masks,
mask_condition_enabled=mask_condition_enabled,
)
summary = {
"reference_items": len(reference_images),
"subject_items": int(subject_image_count),
"mixed_reference_items": int(mixed_reference_count),
"additional_items": max(0, len(reference_images) - 1),
"primary_items": 1,
"background_images": int(background_images),
"background_enabled": bool(background_enabled),
"background_prefix_used": bool(background_prefix_used),
"background_ignored": bool(replacement_mode and background_images > 0),
"background_fit": "center_crop_cover" if background_enabled else None,
"mixed_reference_used": int(len(mixed_items)),
"mixed_reference_dropped": int(dropped_mixed),
"max_prefix_reference_images": int(MAX_PREFIX_REFERENCE_IMAGES),
"generation_width": int(width),
"generation_height": int(height),
"reference_pixel_frames": int(len(reference_images)),
"reference_latent_shape": _shape(ref_latent),
"reference_latent_parts": latent_shapes,
"reference_encoding": "main_collage_plus_single_frame_prefix_latents",
"reference_mask_enabled": bool(mask_condition_enabled),
"reference_mask_reason": (
None
if not mask_condition_enabled
else (
"replacement"
if replacement_mode
else ("multi_subject" if subject_count > 1 else "reference_sam_masks")
)
),
"items": item_summaries,
}
return {
"reference_latent": ref_latent,
"reference_mask_prefix": ref_mask_1f,
"clip_image": main_image,
"summary": summary,
}
def _set_scail_reference_pack_conditioning(
*,
positive,
negative,
vae,
latent: torch.Tensor,
reference_pack: dict,
width: int,
height: int,
replacement_mode: bool,
prepared_reference_pack: dict[str, Any] | None = None,
) -> tuple[Any, Any, dict]:
import node_helpers
prepared = prepared_reference_pack or _prepare_scail_reference_pack(
vae=vae,
reference_pack=reference_pack,
width=width,
height=height,
replacement_mode=replacement_mode,
)
ref_latent = prepared["reference_latent"]
positive = node_helpers.conditioning_set_values(positive, {"reference_latents": [ref_latent]}, append=True)
negative = node_helpers.conditioning_set_values(negative, {"reference_latents": [ref_latent]}, append=True)
ref_mask_1f = prepared.get("reference_mask_prefix")
ref_mask_shape = None
if ref_mask_1f is not None:
zeros = torch.zeros(
(1, latent.shape[2], 28, ref_mask_1f.shape[-2], ref_mask_1f.shape[-1]),
device=ref_mask_1f.device,
dtype=ref_mask_1f.dtype,
)
ref_mask_28ch = torch.cat([ref_mask_1f, zeros], dim=1)
positive = node_helpers.conditioning_set_values(positive, {"ref_mask_28ch": ref_mask_28ch})
negative = node_helpers.conditioning_set_values(negative, {"ref_mask_28ch": ref_mask_28ch})
ref_mask_shape = _shape(ref_mask_28ch)
summary = dict(prepared["summary"])
summary["reference_mask_shape"] = ref_mask_shape
return positive, negative, summary
def _run_multi_ref_scail_chunk(
*,
model,
positive,
negative,
vae,
sampler,
sigmas,
reference_pack: dict,
pose_video: torch.Tensor,
clip_vision_output,
pose_video_mask: torch.Tensor | None,
previous_frames: torch.Tensor | None,
width: int,
height: int,
length: int,
video_frame_offset: int,
previous_frame_count: int,
replacement_mode: bool,
seed: int,
cfg: float,
pose_strength: float,
prepared_reference_pack: dict[str, Any] | None = None,
) -> tuple[torch.Tensor, int, dict]:
import comfy.model_management
import comfy.utils
import node_helpers
latent = torch.zeros(
[1, 16, ((length - 1) // 4) + 1, height // 8, width // 8],
device=comfy.model_management.intermediate_device(),
)
noise_mask = None
ref_mask_flag = not replacement_mode
positive = node_helpers.conditioning_set_values(positive, {"ref_mask_flag": ref_mask_flag})
negative = node_helpers.conditioning_set_values(negative, {"ref_mask_flag": ref_mask_flag})
prev_trimmed = None
if previous_frames is not None and previous_frames.shape[0] > 0:
prev_trimmed = previous_frames[-previous_frame_count:]
video_frame_offset = max(0, int(video_frame_offset) - int(prev_trimmed.shape[0]))
positive, negative, ref_summary = _set_scail_reference_pack_conditioning(
positive=positive,
negative=negative,
vae=vae,
latent=latent,
reference_pack=reference_pack,
width=width,
height=height,
replacement_mode=replacement_mode,
prepared_reference_pack=prepared_reference_pack,
)
if clip_vision_output is not None:
positive = node_helpers.conditioning_set_values(positive, {"clip_vision_output": clip_vision_output})
negative = node_helpers.conditioning_set_values(negative, {"clip_vision_output": clip_vision_output})
if pose_video is not None:
pose_video = None if pose_video.shape[0] <= video_frame_offset else pose_video[video_frame_offset:]
if pose_video_mask is not None:
pose_video_mask = None if pose_video_mask.shape[0] <= video_frame_offset else pose_video_mask[video_frame_offset:]
ts = [v.shape[0] for v in (pose_video, pose_video_mask) if v is not None]
if ts:
kept = ((min(min(ts), length) - 1) // 4) * 4 + 1
if pose_video is not None:
pose_video = pose_video[:kept]
if pose_video_mask is not None:
pose_video_mask = pose_video_mask[:kept]
if pose_video is not None:
pose_video = comfy.utils.common_upscale(
pose_video[:length].movedim(-1, 1),
width // 2,
height // 2,
"area",
"center",
).movedim(1, -1)
pose_video_latent = vae.encode(pose_video[:, :, :, :3]) * pose_strength
positive = node_helpers.conditioning_set_values_with_timestep_range(
positive, {"pose_video_latent": pose_video_latent}, 0.0, 1.0
)
negative = node_helpers.conditioning_set_values_with_timestep_range(
negative, {"pose_video_latent": pose_video_latent}, 0.0, 1.0
)
if pose_video_mask is not None:
mask_video_hw = comfy.utils.common_upscale(
pose_video_mask[:length].movedim(-1, 1),
width // 2,
height // 2,
"area",
"center",
).movedim(1, -1)
driving_mask_28ch = _extract_scail_mask_to_28ch(mask_video_hw)
positive = node_helpers.conditioning_set_values(positive, {"driving_mask_28ch": driving_mask_28ch})
negative = node_helpers.conditioning_set_values(negative, {"driving_mask_28ch": driving_mask_28ch})
if prev_trimmed is not None:
previous = comfy.utils.common_upscale(prev_trimmed.movedim(-1, 1), width, height, "bicubic", "center").movedim(1, -1)
prev_latent = vae.encode(previous[:, :, :, :3])
prev_latent_frames = min(prev_latent.shape[2], latent.shape[2])
latent[:, :, :prev_latent_frames] = prev_latent[:, :, :prev_latent_frames].to(latent.dtype)
noise_mask = torch.ones(
(1, 1, latent.shape[2], latent.shape[-2], latent.shape[-1]),
device=latent.device,
dtype=latent.dtype,
)
noise_mask[:, :, :prev_latent_frames] = 0.0
out_latent = {"samples": latent}
if noise_mask is not None:
out_latent["noise_mask"] = noise_mask
latent_to_decode = _sample_for_decode(
model=model,
positive=positive,
negative=negative,
sampler=sampler,
sigmas=sigmas,
latent=out_latent,
seed=seed,
cfg=cfg,
)
positive = negative = out_latent = latent = noise_mask = None
reference_pack = clip_vision_output = prepared_reference_pack = None
pose_video = pose_video_mask = pose_video_latent = mask_video_hw = driving_mask_28ch = None
previous = prev_latent = prev_trimmed = None
_empty_cache(force=True)
frames = _decode_latent_to_frames(vae, latent_to_decode)
summary = {
"width": int(width),
"height": int(height),
"length": int(length),
"seed": int(seed),
"cfg": float(cfg),
"input_video_frame_offset": int(video_frame_offset),
"output_video_frame_offset": int(video_frame_offset + length),
"decoded_shape": _shape(frames),
"reference_pack": ref_summary,
}
return frames, int(video_frame_offset + length), summary
def _set_scail_single_reference_conditioning(
*,
positive,
negative,
vae,
latent: torch.Tensor,
reference_image: torch.Tensor,
reference_image_mask: torch.Tensor | None,
width: int,
height: int,
replacement_mode: bool,
) -> tuple[Any, Any, dict]:
import comfy.utils
import node_helpers
ref_pixels = comfy.utils.common_upscale(
reference_image[:1].movedim(-1, 1),
width,
height,
"bilinear",
"center",
).movedim(1, -1)
ref_mask = None
if replacement_mode and reference_image_mask is not None:
ref_mask = comfy.utils.common_upscale(
reference_image_mask[:1].movedim(-1, 1),
width,
height,
"nearest-exact",
"center",
).movedim(1, -1)
alpha = (ref_mask[..., :3].max(dim=-1, keepdim=True).values > 0.1).to(dtype=ref_pixels.dtype)
ref_pixels = ref_pixels * alpha
ref_latent = vae.encode(ref_pixels[:, :, :, :3])
positive = node_helpers.conditioning_set_values(positive, {"reference_latents": [ref_latent]}, append=True)
negative = node_helpers.conditioning_set_values(negative, {"reference_latents": [torch.zeros_like(ref_latent)]}, append=True)
ref_mask_shape = None
if replacement_mode and ref_mask is not None:
ref_mask_1f = _extract_scail_mask_to_28ch(ref_mask)
zeros = torch.zeros(
(1, latent.shape[2], 28, ref_mask_1f.shape[-2], ref_mask_1f.shape[-1]),
device=ref_mask_1f.device,
dtype=ref_mask_1f.dtype,
)
ref_mask_28ch = torch.cat([ref_mask_1f, zeros], dim=1)
positive = node_helpers.conditioning_set_values(positive, {"ref_mask_28ch": ref_mask_28ch})
negative = node_helpers.conditioning_set_values(negative, {"ref_mask_28ch": ref_mask_28ch})
ref_mask_shape = _shape(ref_mask_28ch)
summary = {
"reference_latent_shape": _shape(ref_latent),
"reference_mask_shape": ref_mask_shape,
}
return positive, negative, summary
def _set_scail_pose_conditioning(
*,
positive,
negative,
vae,
pose_video: torch.Tensor,
pose_video_mask: torch.Tensor | None,
width: int,
height: int,
length: int,
pose_strength: float,
) -> tuple[Any, Any, dict]:
import comfy.utils
import node_helpers
if pose_video.shape[0] < length:
raise ValueError("pose_video is shorter than the requested generation length.")
if pose_video_mask is not None and pose_video_mask.shape[0] < length:
raise ValueError("pose_video_mask is shorter than the requested generation length.")
pose_video = pose_video[:length]
pose_video_hw = comfy.utils.common_upscale(
pose_video.movedim(-1, 1),
width // 2,
height // 2,
"area",
"center",
).movedim(1, -1)
pose_video_latent = vae.encode(pose_video_hw[:, :, :, :3]) * pose_strength
positive = node_helpers.conditioning_set_values_with_timestep_range(
positive,
{"pose_video_latent": pose_video_latent},
0.0,
1.0,
)
negative = node_helpers.conditioning_set_values_with_timestep_range(
negative,
{"pose_video_latent": pose_video_latent},
0.0,
1.0,
)
driving_mask_shape = None
if pose_video_mask is not None:
mask_video_hw = comfy.utils.common_upscale(
pose_video_mask[:length].movedim(-1, 1),
width // 2,
height // 2,
"area",
"center",
).movedim(1, -1)
driving_mask_28ch = _extract_scail_mask_to_28ch(mask_video_hw)
positive = node_helpers.conditioning_set_values(positive, {"driving_mask_28ch": driving_mask_28ch})
negative = node_helpers.conditioning_set_values(negative, {"driving_mask_28ch": driving_mask_28ch})
driving_mask_shape = _shape(driving_mask_28ch)
summary = {
"pose_video_latent_shape": _shape(pose_video_latent),
"driving_mask_shape": driving_mask_shape,
}
return positive, negative, summary
def _run_context_scail(
*,
model,
positive,
negative,
vae,
sampler,
sigmas,
reference_image: torch.Tensor | None,
reference_pack: dict | None,
pose_video: torch.Tensor,
clip_vision_output,
pose_video_mask: torch.Tensor | None,
reference_image_mask: torch.Tensor | None,
width: int,
height: int,
length: int,
context_frames: int,
context_overlap_frames: int,
replacement_mode: bool,
seed: int,
cfg: float,
pose_strength: float,
prepared_reference_pack: dict[str, Any] | None = None,
) -> tuple[torch.Tensor, dict]:
import comfy.model_management
import node_helpers
latent = torch.zeros(
[1, 16, ((length - 1) // 4) + 1, height // 8, width // 8],
device=comfy.model_management.intermediate_device(),
)
ref_mask_flag = not replacement_mode
positive = node_helpers.conditioning_set_values(positive, {"ref_mask_flag": ref_mask_flag})
negative = node_helpers.conditioning_set_values(negative, {"ref_mask_flag": ref_mask_flag})
if reference_pack is not None:
positive, negative, reference_summary = _set_scail_reference_pack_conditioning(
positive=positive,
negative=negative,
vae=vae,
latent=latent,
reference_pack=reference_pack,
width=width,
height=height,
replacement_mode=replacement_mode,
prepared_reference_pack=prepared_reference_pack,
)
elif reference_image is not None:
positive, negative, reference_summary = _set_scail_single_reference_conditioning(
positive=positive,
negative=negative,
vae=vae,
latent=latent,
reference_image=reference_image,
reference_image_mask=reference_image_mask,
width=width,
height=height,
replacement_mode=replacement_mode,
)
else:
raise ValueError("reference_image or reference_pack is required.")
if clip_vision_output is not None:
positive = node_helpers.conditioning_set_values(positive, {"clip_vision_output": clip_vision_output})
negative = node_helpers.conditioning_set_values(negative, {"clip_vision_output": clip_vision_output})
positive, negative, pose_summary = _set_scail_pose_conditioning(
positive=positive,
negative=negative,
vae=vae,
pose_video=pose_video,
pose_video_mask=pose_video_mask,
width=width,
height=height,
length=length,
pose_strength=pose_strength,
)
context_model, context_summary = apply_scail2_easy_context(
model,
context_frames=context_frames,
context_overlap_frames=context_overlap_frames,
)
latent_to_decode = _sample_for_decode(
model=context_model,
positive=positive,
negative=negative,
sampler=sampler,
sigmas=sigmas,
latent={"samples": latent},
seed=seed,
cfg=cfg,
)
positive = negative = latent = pose_video = pose_video_mask = None
reference_image = reference_pack = reference_image_mask = clip_vision_output = prepared_reference_pack = None
context_model = None
_empty_cache(force=True)
frames = _decode_latent_to_frames(vae, latent_to_decode)
summary = {
"width": int(width),
"height": int(height),
"length": int(length),
"seed": int(seed),
"cfg": float(cfg),
"decoded_shape": _shape(frames),
"context": context_summary,
"reference": reference_summary,
"pose": pose_summary,
}
return frames, summary
class SCAIL2ReferencePack:
@classmethod
def INPUT_TYPES(cls):
required = {
"subject_count": ("INT", {"default": 1, "min": 1, "max": MAX_REFERENCE_SUBJECTS, "step": 1}),
"reference_count": ("INT", {"default": 0, "min": 0, "max": MAX_MIXED_REFERENCE_IMAGES, "step": 1}),
}
optional = {}
for subject_index in range(1, MAX_REFERENCE_SUBJECTS + 1):
optional[f"subject_{subject_index}_image"] = ("IMAGE",)
for reference_index in range(1, MAX_MIXED_REFERENCE_IMAGES + 1):
optional[f"reference_{reference_index}"] = ("IMAGE",)
optional["scene_image"] = ("IMAGE",)
for subject_index in range(1, MAX_REFERENCE_SUBJECTS + 1):
for image_index in range(1, MAX_LEGACY_REFERENCE_IMAGES_PER_SUBJECT + 1):
optional[f"subject_{subject_index}_image_{image_index}"] = ("IMAGE",)
return {"required": required, "optional": optional}
RETURN_TYPES = (REFERENCE_PACK_TYPE,)
RETURN_NAMES = ("reference_pack",)
FUNCTION = "pack"
CATEGORY = CATEGORY
def pack(self, subject_count: int, reference_count: int = 0, scene_image: torch.Tensor | None = None, **kwargs):
active_subjects = _clamp_int(subject_count, 1, MAX_REFERENCE_SUBJECTS)
active_references = _clamp_int(reference_count, 0, MAX_MIXED_REFERENCE_IMAGES)
subjects = []
connected_subject_indices = []
for subject_index in range(1, active_subjects + 1):
name = f"subject_{subject_index}_image"
image = kwargs.get(name)
legacy_name = f"subject_{subject_index}_image_1"
if image is None:
image = kwargs.get(legacy_name)
name = legacy_name
if image is None:
continue
first = _first_image(image, name)
subjects.append({"index": len(subjects), "source_subject_index": subject_index - 1, "images": [first]})
connected_subject_indices.append(subject_index)
if not subjects:
raise ValueError("Reference Pack needs at least one connected subject image.")
reference_images = []
for reference_index in range(1, active_references + 1):
name = f"reference_{reference_index}"
image = kwargs.get(name)
if image is not None:
reference_images.append(_first_image(image, name))
if not reference_images:
for subject_index in connected_subject_indices:
for image_index in range(2, MAX_LEGACY_REFERENCE_IMAGES_PER_SUBJECT + 1):
legacy_name = f"subject_{subject_index}_image_{image_index}"
image = kwargs.get(legacy_name)
if image is not None:
reference_images.append(_first_image(image, legacy_name))
if len(reference_images) >= MAX_MIXED_REFERENCE_IMAGES:
break
if len(reference_images) >= MAX_MIXED_REFERENCE_IMAGES:
break
scene_images = []
if scene_image is not None:
scene_images.append(_first_image(scene_image, "scene_image"))
reference_pack = {
"type": REFERENCE_PACK_TYPE,
"version": 3,
"subjects": subjects,
"reference_images": reference_images,
"scene_images": scene_images,
"subject_count": len(subjects),
"configured_subject_count": active_subjects,
"reference_count": len(reference_images),
"max_subjects": MAX_REFERENCE_SUBJECTS,
"max_reference_images": MAX_MIXED_REFERENCE_IMAGES,
}
return (reference_pack,)
class SCAIL2ReferenceSAMBuilder:
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"reference_pack": (REFERENCE_PACK_TYPE,),
"sam_model": ("MODEL",),
"conditioning": ("CONDITIONING",),
"detection_threshold": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}),
"max_objects": ("INT", {"default": 2, "min": 1, "max": 8, "step": 1}),
"detect_interval": ("INT", {"default": 1, "min": 1, "max": 999, "step": 1}),
},
}
RETURN_TYPES = (REFERENCE_PACK_TYPE, "STRING")
RETURN_NAMES = ("reference_pack", "summary")
FUNCTION = "build"
CATEGORY = CATEGORY
def build(
self,
reference_pack: dict,
sam_model,
conditioning,
detection_threshold: float,
max_objects: int,
detect_interval: int,
):
if not _is_reference_pack(reference_pack):
raise ValueError("reference_pack must come from SCAIL-2 Reference Pack.")
subjects = []
summary_subjects = []
for subject_position, subject in enumerate(reference_pack.get("subjects", [])):
subject_index = int(subject.get("index", subject_position))
images = list(subject.get("images", []))[:1]
masks = []
image_summaries = []
for image_index, image in enumerate(images):
first = _first_image(image, f"subject_{subject_index + 1}_image")
track = _run_sam3_track(
first,
sam_model,
conditioning,
detection_threshold=float(detection_threshold),
max_objects=int(max_objects),
detect_interval=int(detect_interval),
)
mask = _render_reference_sam_mask(first, track, subject_index)
masks.append(mask.detach().contiguous())
image_summaries.append(
{
"image_index": int(image_index),
"image_shape": _shape(first),
"mask_shape": _shape(mask),
}
)
enriched_subject = dict(subject)
enriched_subject["images"] = images
enriched_subject["masks"] = masks
enriched_subject["mask_source"] = "sam3"
subjects.append(enriched_subject)
summary_subjects.append(
{
"subject": int(subject_index + 1),
"reference_images": len(images),
"masks": len(masks),
"images": image_summaries,
}
)
reference_images = []
reference_masks = []
summary_references = []
for reference_index, image in enumerate(reference_pack.get("reference_images", [])):
first = _first_image(image, f"reference_{reference_index + 1}")
track = _run_sam3_track(
first,
sam_model,
conditioning,
detection_threshold=float(detection_threshold),
max_objects=int(max_objects),
detect_interval=int(detect_interval),
)
mask = _render_mixed_reference_sam_mask(first, track)
reference_images.append(first)
reference_masks.append(mask.detach().contiguous())
summary_references.append(
{
"reference": int(reference_index + 1),
"image_shape": _shape(first),
"mask_shape": _shape(mask),
"objects": _track_object_count(track),
}
)
enriched_pack = dict(reference_pack)
enriched_pack["version"] = 4
enriched_pack["subjects"] = subjects
enriched_pack["reference_images"] = reference_images
enriched_pack["reference_masks"] = reference_masks
enriched_pack["reference_mask_source"] = "sam3"
summary = {
"type": REFERENCE_PACK_TYPE,
"reference_mask_source": "sam3",
"subject_count": len(subjects),
"subjects": summary_subjects,
"mixed_references": summary_references,
"scene_images": len(reference_pack.get("scene_images", [])),
"settings": {
"detection_threshold": float(detection_threshold),
"max_objects": int(max_objects),
"detect_interval": int(detect_interval),
},
}
return (enriched_pack, json.dumps(summary, indent=2))
class SCAIL2SimpleVideo:
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"model": ("MODEL",),
"positive": ("CONDITIONING",),
"negative": ("CONDITIONING",),
"vae": ("VAE",),
"sampler": ("SAMPLER",),
"sigmas": ("SIGMAS",),
"reference_image": (REFERENCE_IMAGE_INPUT_TYPE,),
"pose_video": ("IMAGE",),
"clip_vision": ("CLIP_VISION",),
"seed": ("INT", {"default": 1, "min": 0, "max": 0xffffffffffffffff}),
"cfg": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 20.0, "step": 0.1}),
"mode": (["replacement", "animation"], {"default": "replacement"}),
"advanced": ("BOOLEAN", {"default": False}),
"long_video_mode": (list(LONG_VIDEO_MODES), {"default": "chunk"}),
"max_frames": ("INT", {"default": 0, "min": 0, "max": 100000, "step": 1}),
"chunk_frames": ("INT", {"default": 81, "min": 17, "max": 321, "step": 4}),
"overlap_frames": ("INT", {"default": 5, "min": 0, "max": 33, "step": 1}),
"color_correction": ("BOOLEAN", {"default": False}),
"context_frames": ("INT", {"default": 81, "min": 17, "max": 321, "step": 4}),
"context_overlap_frames": ("INT", {"default": 20, "min": 0, "max": 320, "step": 1}),
},
"optional": {
"driving_track_data": ("SAM3_TRACK_DATA",),
"reference_track_data": ("SAM3_TRACK_DATA",),
},
}
RETURN_TYPES = ("IMAGE", "STRING")
RETURN_NAMES = ("frames", "summary")
FUNCTION = "generate"
CATEGORY = CATEGORY
def generate(
self,
model,
positive,
negative,
vae,
sampler,
sigmas,
reference_image,
pose_video: torch.Tensor,
clip_vision,
seed: int,
cfg: float,
mode: str,
long_video_mode: str = "chunk",
advanced: bool = False,
max_frames: int = 0,
chunk_frames: int = 81,
overlap_frames: int = 5,
color_correction: bool = False,
context_frames: int = 81,
context_overlap_frames: int = 20,
driving_track_data=None,
reference_track_data=None,
):
reference_is_pack = _is_reference_pack(reference_image)
if pose_video.ndim != 4:
raise ValueError("pose_video must be a ComfyUI IMAGE tensor.")
if not reference_is_pack and (not isinstance(reference_image, torch.Tensor) or reference_image.ndim != 4):
raise ValueError("reference_image must be a ComfyUI IMAGE tensor or a SCAIL-2 Reference Pack.")
total_frames = int(pose_video.shape[0])
if max_frames > 0:
total_frames = min(total_frames, int(max_frames))
if total_frames <= 0:
raise ValueError("pose_video has no frames.")
mask_replacement_mode = mode == "replacement"
pose_video_mask = None
reference_image_mask = None
if reference_is_pack:
subject_count = len(reference_image.get("subjects", []))
reference_pack_identity_mask_required = bool(mask_replacement_mode or subject_count > 1)
use_optional_animation_driving_mask = bool(
not mask_replacement_mode
and subject_count == 1
and driving_track_data is not None
)
if reference_pack_identity_mask_required or use_optional_animation_driving_mask:
if driving_track_data is None:
raise ValueError(
"replacement or multi-subject Reference Pack requires driving_track_data from SAM3_VideoTrack."
)
driving_object_count = _track_object_count(driving_track_data)
if subject_count > 1 and driving_object_count < subject_count:
raise ValueError(
"Reference Pack has "
f"{subject_count} subjects, but driving_track_data only contains "
f"{driving_object_count} tracked object(s). Increase SAM3_VideoTrack.max_objects "
"for the driving video and make sure each target person is detected."
)
pose_video_mask = _create_driving_scail_mask(driving_track_data, mask_replacement_mode)
elif mask_replacement_mode:
if driving_track_data is None or reference_track_data is None:
raise ValueError("replacement mode requires driving_track_data and reference_track_data from SAM3_VideoTrack.")
pose_video_mask, reference_image_mask = _create_scail_masks(
driving_track_data,
reference_track_data,
True,
)
pose_video = _trim_frames(pose_video, total_frames)
pose_video_mask = _trim_frames(pose_video_mask, total_frames)
width, height = _infer_generation_size(pose_video)
reference_pack_cache: dict[bool, dict[str, Any]] = {}
if reference_is_pack:
reference_pack = reference_image
initial_prepared_reference_pack = _prepare_scail_reference_pack(
vae=vae,
reference_pack=reference_pack,
width=width,
height=height,
replacement_mode=mask_replacement_mode,
)
reference_pack_cache[mask_replacement_mode] = initial_prepared_reference_pack
clip_vision_output = _encode_clip_vision(
clip_vision,
initial_prepared_reference_pack["clip_image"],
)
else:
reference_pack = None
initial_prepared_reference_pack = None
reference_image = reference_image[:1].detach()
if reference_image_mask is not None:
reference_image_mask = reference_image_mask[:1].detach()
reference_image = _apply_reference_mask(reference_image, reference_image_mask)
clip_vision_output = _encode_clip_vision(clip_vision, reference_image)
if long_video_mode not in LONG_VIDEO_MODES:
long_video_mode = "chunk"
pose_strength = 1.0
if long_video_mode == "context_sampling":
generation_length = _wan_frame_count_floor(total_frames)
if generation_length <= 0:
raise ValueError("pose_video has no usable 4n+1 frame range.")
context_frames = _wan_frame_count_cover(max(17, int(context_frames)))
requested_context_overlap = max(0, int(context_overlap_frames))
context_overlap_frames = (
0
if requested_context_overlap == 0
else min(_wan_frame_count_floor(requested_context_overlap), max(0, context_frames - 4))
)
context_pose_video = pose_video[:generation_length].contiguous()
context_pose_video_mask = _trim_frames(pose_video_mask, generation_length)
prepared_reference_pack = initial_prepared_reference_pack
frames, context_summary = _run_context_scail(
model=model,
positive=positive,
negative=negative,
vae=vae,
sampler=sampler,
sigmas=sigmas,
reference_image=None if reference_pack is not None else reference_image,
reference_pack=reference_pack,
pose_video=context_pose_video,
clip_vision_output=clip_vision_output,
pose_video_mask=context_pose_video_mask,
reference_image_mask=reference_image_mask,
width=width,
height=height,
length=generation_length,
context_frames=context_frames,
context_overlap_frames=context_overlap_frames,
replacement_mode=mask_replacement_mode,
seed=int(seed),
cfg=float(cfg),
pose_strength=float(pose_strength),
prepared_reference_pack=prepared_reference_pack,
)
summary = {
"mode": mode,
"long_video_mode": long_video_mode,
"mask_replacement_mode": bool(mask_replacement_mode),
"frames": _shape(frames),
"width": width,
"height": height,
"source_frames": int(total_frames),
"generated_frames": int(frames.shape[0]),
"dropped_tail_frames": int(total_frames - frames.shape[0]),
"color_correction_enabled": False,
"cfg": float(cfg),
"seed_start": int(seed),
"reference_input": "reference_pack" if reference_pack is not None else "image",
"core": "SCAIL-2 context sampling -> SamplerCustom -> VAEDecode",
"context_details": context_summary,
}
return (frames.contiguous().clamp(0, 1), json.dumps(summary, indent=2))
chunk_frames = _wan_frame_count_cover(max(17, int(chunk_frames)))
first_length = chunk_frames
requested_overlap = max(0, int(overlap_frames))
previous_frame_count = 0 if requested_overlap == 0 else max(1, min(_wan_frame_count_floor(requested_overlap), 33, max(1, chunk_frames - 4)))
if chunk_frames <= previous_frame_count:
raise ValueError("chunk_frames must be larger than overlap_frames.")
color_correction_enabled = bool(color_correction and previous_frame_count > 0)
stitched: list[torch.Tensor] = []
chunk_summaries: list[dict[str, Any]] = []
previous_frames = None
video_frame_offset = 0
produced = 0
chunk_index = 0
max_chunks = max(1, (total_frames // max(1, chunk_frames - previous_frame_count)) + 4)
while produced < total_frames and chunk_index < max_chunks:
anchor_start = max(0, video_frame_offset - (previous_frame_count if previous_frames is not None else 0))
remaining_from_anchor = max(1, total_frames - anchor_start)
target_length = first_length if chunk_index == 0 else chunk_frames
length = min(target_length, _wan_frame_count_floor(remaining_from_anchor))
if previous_frames is not None and previous_frame_count > 0 and length <= previous_frame_count:
break
if reference_pack is not None:
chunk_replacement_mode = bool(mask_replacement_mode)
prepared_reference_pack = reference_pack_cache.get(chunk_replacement_mode)
if prepared_reference_pack is None:
prepared_reference_pack = _prepare_scail_reference_pack(
vae=vae,
reference_pack=reference_pack,
width=width,
height=height,
replacement_mode=chunk_replacement_mode,
)
reference_pack_cache[chunk_replacement_mode] = prepared_reference_pack
decoded, next_offset, chunk_summary = _run_multi_ref_scail_chunk(
model=model,
positive=positive,
negative=negative,
vae=vae,
sampler=sampler,
sigmas=sigmas,
reference_pack=reference_pack,
pose_video=pose_video,
clip_vision_output=clip_vision_output,
pose_video_mask=pose_video_mask,
previous_frames=previous_frames,
width=width,
height=height,
length=length,
video_frame_offset=video_frame_offset,
previous_frame_count=previous_frame_count,
replacement_mode=chunk_replacement_mode,
seed=int(seed) + chunk_index,
cfg=float(cfg),
pose_strength=float(pose_strength),
prepared_reference_pack=prepared_reference_pack,
)
else:
decoded, next_offset, chunk_summary = _run_native_scail_chunk(
model=model,
positive=positive,
negative=negative,
vae=vae,
sampler=sampler,
sigmas=sigmas,
reference_image=reference_image,
pose_video=pose_video,
clip_vision_output=clip_vision_output,
pose_video_mask=pose_video_mask,
reference_image_mask=reference_image_mask,
previous_frames=previous_frames,
width=width,
height=height,
length=length,
video_frame_offset=video_frame_offset,
previous_frame_count=previous_frame_count,
replacement_mode=mask_replacement_mode,
seed=int(seed) + chunk_index,
cfg=float(cfg),
pose_strength=float(pose_strength),
)
discard_head = 0 if chunk_index == 0 else min(previous_frame_count, int(decoded.shape[0]))
current_overlap = decoded[:discard_head].contiguous() if discard_head > 0 else None
reference_overlap = previous_frames[-discard_head:].contiguous() if previous_frames is not None and discard_head > 0 else None
kept = decoded[discard_head:].contiguous()
remaining_output = total_frames - produced
if kept.shape[0] > remaining_output:
kept = kept[:remaining_output].contiguous()
if kept.shape[0] <= 0:
raise RuntimeError("A SCAIL-2 chunk produced no keepable frames.")
color_summary: dict[str, Any] = {"applied": False}
if color_correction_enabled and discard_head > 0:
kept, color_summary = _match_chunk_color_to_overlap(kept, current_overlap, reference_overlap)
kept_cpu = kept.detach().cpu().contiguous()
stitched.append(kept_cpu)
produced += int(kept.shape[0])
chunk_summary.update(
{
"chunk_index": chunk_index,
"replacement_mode": bool(mask_replacement_mode),
"discard_head": int(discard_head),
"kept_frames": int(kept.shape[0]),
"produced_total": int(produced),
}
)
if color_correction_enabled:
chunk_summary["color_correction"] = color_summary
chunk_summaries.append(chunk_summary)
if previous_frame_count > 0:
tail_parts = [kept_cpu] if previous_frames is None else [previous_frames, kept_cpu]
previous_frames = torch.cat(tail_parts, dim=0)[-previous_frame_count:].contiguous()
else:
previous_frames = None
video_frame_offset = int(next_offset)
chunk_index += 1
del decoded, kept, kept_cpu, current_overlap, reference_overlap
_empty_cache(force=True)
frames = torch.cat(stitched, dim=0).contiguous().clamp(0, 1)
summary = {
"mode": mode,
"long_video_mode": long_video_mode,
"mask_replacement_mode": bool(mask_replacement_mode),
"chunk_replacement_mode": bool(mask_replacement_mode),
"frames": _shape(frames),
"width": width,
"height": height,
"source_frames": int(total_frames),
"generated_frames": int(frames.shape[0]),
"dropped_tail_frames": int(total_frames - frames.shape[0]),
"chunks": len(chunk_summaries),
"first_chunk_frames": int(first_length),
"chunk_frames": int(chunk_frames),
"overlap_frames": int(previous_frame_count),
"color_correction_enabled": bool(color_correction_enabled),
"cfg": float(cfg),
"seed_start": int(seed),
"reference_input": "reference_pack" if reference_pack is not None else "image",
"core": "SCAIL-2 multi-reference conditioning -> SamplerCustom -> VAEDecode"
if reference_pack is not None
else "ComfyUI native WanSCAILToVideo -> SamplerCustom -> VAEDecode",
"chunk_details": chunk_summaries,
}
return (frames, json.dumps(summary, indent=2))
class SCAIL2FitVideo:
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"video": ("IMAGE",),
"resolution": (list(RESOLUTION_PRESETS), {"default": "512p"}),
"custom_width": ("INT", {"default": 832, "min": 32, "max": 4096, "step": 32}),
"custom_height": ("INT", {"default": 480, "min": 32, "max": 4096, "step": 32}),
},
}
RETURN_TYPES = ("IMAGE", "INT", "INT", "STRING")
RETURN_NAMES = ("video", "width", "height", "summary")
FUNCTION = "fit"
CATEGORY = CATEGORY
def fit(self, video: torch.Tensor, resolution: str, custom_width: int = 832, custom_height: int = 480):
target_width, target_height = _target_size_for_video(video, resolution, custom_width, custom_height)
fitted = _resize_to_target(video, target_width, target_height)
summary = {
"resolution": resolution,
"custom_width": int(custom_width),
"custom_height": int(custom_height),
"source": _shape(video),
"target": _shape(fitted),
"width": target_width,
"height": target_height,
"fit": "preserve_aspect_resize_to_nearest_32",
}
return (fitted, target_width, target_height, json.dumps(summary, indent=2))
NODE_CLASS_MAPPINGS = {
"SCAIL2FitVideo": SCAIL2FitVideo,
"SCAIL2ReferencePack": SCAIL2ReferencePack,
"SCAIL2ReferenceSAMBuilder": SCAIL2ReferenceSAMBuilder,
"SCAIL2SimpleVideo": SCAIL2SimpleVideo,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"SCAIL2FitVideo": "SCAIL-2 Fit Video",
"SCAIL2ReferencePack": "SCAIL-2 Reference Pack",
"SCAIL2ReferenceSAMBuilder": "SCAIL-2 Reference SAM Builder",
"SCAIL2SimpleVideo": "SCAIL-2 Simple Video",
}