Files
Baverne-comfyUI-TiledWan/nodes.py
T
2025-08-05 15:05:31 +02:00

3846 lines
195 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import torch
import comfy.utils
import comfy.model_management
import node_helpers
import random
import os
import sys
import importlib
import math
import nodes
import numpy as np
import torchvision.transforms.functional as F
from PIL import Image
from scipy.ndimage import gaussian_filter, grey_dilation, binary_closing, binary_fill_holes
class ImageToMask:
"""
Convert an image to a mask by extracting a specific channel.
Supports red, green, blue, and alpha channels with optional clamping and normalization.
"""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"image": ("IMAGE", {"tooltip": "Input image to convert to mask. Can be any standard image format."}),
"channel": (["red", "green", "blue", "alpha"], {"tooltip": "Which color channel to extract as the mask. Alpha channel only available if image has transparency."}),
"clamp_output": ("BOOLEAN", {
"default": True,
"label_on": "enabled",
"label_off": "disabled",
"tooltip": "Clamp output values to 0-1 range. Recommended to keep enabled for proper mask values."
}),
"normalize_output": ("BOOLEAN", {
"default": True,
"label_on": "enabled",
"label_off": "disabled",
"tooltip": "Normalize the output to use full 0-1 range."
}),
},
}
CATEGORY = "TiledWan"
DESCRIPTION = """
Extracts a specific color channel from an image to create a mask.
Useful for converting colored regions, alpha channels, or channel-specific data into masks
for use with inpainting, compositing, or other masking operations. The output mask values
represent the intensity of the selected channel, with proper normalization and clamping.
"""
RETURN_TYPES = ("MASK",)
RETURN_NAMES = ("mask",)
OUTPUT_TOOLTIPS = ("Grayscale mask derived from the selected image channel. White areas indicate high channel values, black areas indicate low values.",)
FUNCTION = "image_to_mask"
def image_to_mask(self, image, channel, clamp_output, normalize_output):
"""
Convert image to mask by extracting specified channel.
Args:
image: Input image tensor [batch, height, width, channels]
channel: Channel to extract ("red", "green", "blue", "alpha")
clamp_output: Whether to clamp values between 0 and 1
normalize_output: Whether to normalize values to 0-1 range
Returns:
tuple: Mask tensor [batch, height, width]
"""
channels = ["red", "green", "blue", "alpha"]
channel_index = channels.index(channel)
# Handle case where alpha channel is requested but doesn't exist
if channel_index == 3 and image.shape[-1] < 4:
# Create an alpha channel filled with ones (fully opaque)
mask = torch.ones(image.shape[0], image.shape[1], image.shape[2], dtype=image.dtype, device=image.device)
else:
# Extract the specified channel
mask = image[:, :, :, channel_index]
# Apply clamping if requested
if clamp_output:
mask = torch.clamp(mask, 0.0, 1.0)
# Apply normalization if requested
if normalize_output:
mask_min = mask.min()
mask_max = mask.max()
if mask_max > mask_min: # Avoid division by zero
mask = (mask - mask_min) / (mask_max - mask_min)
return (mask,)
class ImageStatistics:
"""
Calculate and display comprehensive statistics for an image.
Shows min, max, mean, variance, median, and other useful statistics.
"""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"image": ("IMAGE", {"tooltip": "Input image to analyze. Statistics will be calculated across all pixels and channels."}),
"show_per_channel": ("BOOLEAN", {
"default": True,
"label_on": "enabled",
"label_off": "disabled",
"tooltip": "Display separate statistics for each color channel (R, G, B) in addition to overall statistics."
}),
},
}
RETURN_TYPES = ("IMAGE", "FLOAT", "FLOAT", "FLOAT", "FLOAT", "FLOAT")
RETURN_NAMES = ("image", "min_value", "max_value", "mean_value", "variance", "median_value")
OUTPUT_TOOLTIPS = ("Pass-through of the input image",
"Minimum pixel value found in the image",
"Maximum pixel value found in the image",
"Average pixel value across the entire image",
"Variance of pixel values (measure of spread/contrast)",
"Median pixel value (middle value when sorted)")
FUNCTION = "calculate_statistics"
OUTPUT_NODE = True # This allows the node to display output in the console
CATEGORY = "TiledWan"
DESCRIPTION = """
Analyzes image pixel values and provides comprehensive statistical information.
Calculates key metrics including minimum, maximum, mean, variance, and median values.
Useful for understanding image characteristics, checking normalization, detecting
issues with image processing, and debugging workflows. Can show per-channel
statistics for color analysis.
"""
def calculate_statistics(self, image: torch.Tensor, show_per_channel: bool):
"""
Calculate comprehensive statistics for the input image.
Args:
image: Input image tensor [batch, height, width, channels]
show_per_channel: Whether to show per-channel statistics
Returns:
tuple: (original_image, min, max, mean, variance, median)
"""
print("\n=== TiledWan Image Statistics ===")
# Basic image info
batch_size, height, width, channels = image.shape
total_pixels = height * width * channels * batch_size
print(f"Image shape: {image.shape} (B×H×W×C)")
print(f"Total pixels: {total_pixels:,}")
print(f"Memory usage: {image.numel() * image.element_size() / (1024*1024):.2f} MB")
print(f"Data type: {image.dtype}")
print(f"Device: {image.device}")
# Calculate global statistics
min_val = image.min().item()
max_val = image.max().item()
mean_val = image.mean().item()
variance_val = image.var().item()
std_val = image.std().item()
# Calculate median (flatten the tensor first)
flattened = image.flatten()
# Sort and find median manually to avoid compatibility issues
sorted_values = torch.sort(flattened).values
n = len(sorted_values)
if n % 2 == 0:
median_val = (sorted_values[n//2 - 1] + sorted_values[n//2]).item() / 2.0
else:
median_val = sorted_values[n//2].item()
# Additional useful statistics
non_zero_count = torch.count_nonzero(image).item()
zero_count = total_pixels - non_zero_count
print("\n--- Global Statistics ---")
print(f"Min value: {min_val:.6f}")
print(f"Max value: {max_val:.6f}")
print(f"Mean value: {mean_val:.6f}")
print(f"Median value: {median_val:.6f}")
print(f"Variance: {variance_val:.6f}")
print(f"Standard deviation: {std_val:.6f}")
print(f"Value range: {max_val - min_val:.6f}")
print(f"Non-zero pixels: {non_zero_count:,} ({non_zero_count/total_pixels:.2%})")
print(f"Zero pixels: {zero_count:,} ({zero_count/total_pixels:.2%})")
# Per-channel statistics if requested
if show_per_channel and channels > 1:
print("\n--- Per-Channel Statistics ---")
channel_names = ["Red", "Green", "Blue", "Alpha"][:channels]
for c in range(channels):
channel_data = image[:, :, :, c]
ch_min = channel_data.min().item()
ch_max = channel_data.max().item()
ch_mean = channel_data.mean().item()
# Calculate median manually for compatibility
ch_flattened = channel_data.flatten()
ch_sorted = torch.sort(ch_flattened).values
ch_n = len(ch_sorted)
if ch_n % 2 == 0:
ch_median = (ch_sorted[ch_n//2 - 1] + ch_sorted[ch_n//2]).item() / 2.0
else:
ch_median = ch_sorted[ch_n//2].item()
ch_std = channel_data.std().item()
print(f"{channel_names[c]} Channel:")
print(f" Min: {ch_min:.6f}, Max: {ch_max:.6f}")
print(f" Mean: {ch_mean:.6f}, Median: {ch_median:.6f}")
print(f" Std: {ch_std:.6f}, Range: {ch_max - ch_min:.6f}")
# Value distribution analysis
print("\n--- Value Distribution ---")
# Create bins for histogram analysis
bins = torch.linspace(min_val, max_val, 11) # 10 bins
hist = torch.histc(flattened, bins=10, min=min_val, max=max_val)
print("Value distribution (10 bins):")
for i in range(len(hist)):
bin_start = bins[i].item()
bin_end = bins[i+1].item() if i < len(bins)-1 else max_val
count = hist[i].item()
percentage = count / total_pixels * 100
print(f" [{bin_start:.3f} - {bin_end:.3f}]: {count:,.0f} pixels ({percentage:.1f}%)")
# Percentiles
print("\n--- Percentiles ---")
percentiles = [1, 5, 10, 25, 50, 75, 90, 95, 99]
# Use the already sorted values from median calculation
for p in percentiles:
idx = int((p / 100.0) * (len(sorted_values) - 1))
value = sorted_values[idx].item()
print(f" {p:2d}th percentile: {value:.6f}")
# Potential issues detection
print("\n--- Analysis ---")
if min_val < 0:
print("⚠️ Warning: Image contains negative values!")
if max_val > 1:
print("⚠️ Warning: Image contains values > 1.0!")
if min_val == max_val:
print("⚠️ Warning: Image has constant values (no variation)!")
if std_val < 0.01:
print("⚠️ Notice: Very low standard deviation - image might be nearly uniform")
if non_zero_count / total_pixels < 0.1:
print("⚠️ Notice: Most pixels are zero - sparse image")
print("================================\n")
# Return the image and key statistics
return (image, min_val, max_val, mean_val, variance_val, median_val)
class MaskStatistics:
"""
Calculate and display comprehensive statistics for a mask.
Shows min, max, mean, variance, median, and mask-specific statistics.
"""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"mask": ("MASK", {"tooltip": "Input mask to analyze. Should be a grayscale mask with values typically between 0-1."}),
"analyze_connected_components": ("BOOLEAN", {
"default": True,
"label_on": "enabled",
"label_off": "disabled",
"tooltip": "Analyze connected regions in the mask to count separate masked areas and find the largest connected component."
}),
},
}
RETURN_TYPES = ("MASK", "FLOAT", "FLOAT", "FLOAT", "FLOAT", "FLOAT", "INT")
RETURN_NAMES = ("mask", "min_value", "max_value", "mean_value", "variance", "median_value", "white_pixel_count")
OUTPUT_TOOLTIPS = ("Pass-through of the input mask",
"Minimum mask value found",
"Maximum mask value found",
"Average mask value (coverage ratio)",
"Variance of mask values",
"Median mask value",
"Number of white/masked pixels (where value > 0.5)")
FUNCTION = "calculate_mask_statistics"
OUTPUT_NODE = True # This allows the node to display output in the console
CATEGORY = "TiledWan"
DESCRIPTION = """
Analyzes mask properties and provides detailed statistical information.
Calculates standard statistics plus mask-specific metrics like coverage ratio,
connected component analysis, and white pixel count. Essential for understanding
mask quality, coverage area, and structure. Helps debug masking operations and
validate mask inputs for inpainting or compositing workflows.
"""
def calculate_mask_statistics(self, mask: torch.Tensor, analyze_connected_components: bool):
"""
Calculate comprehensive statistics for the input mask.
Args:
mask: Input mask tensor [batch, height, width]
analyze_connected_components: Whether to analyze connected components
Returns:
tuple: (original_mask, min, max, mean, variance, median, white_pixel_count)
"""
print("\n=== TiledWan Mask Statistics ===")
# Basic mask info
if len(mask.shape) == 3:
batch_size, height, width = mask.shape
channels = 1
else:
batch_size, height, width, channels = mask.shape
total_pixels = height * width * batch_size
print(f"Mask shape: {mask.shape} (B×H×W)")
print(f"Total pixels: {total_pixels:,}")
print(f"Memory usage: {mask.numel() * mask.element_size() / (1024*1024):.2f} MB")
print(f"Data type: {mask.dtype}")
print(f"Device: {mask.device}")
# Calculate global statistics
min_val = mask.min().item()
max_val = mask.max().item()
mean_val = mask.mean().item()
variance_val = mask.var().item()
std_val = mask.std().item()
# Calculate median (flatten the tensor first)
flattened = mask.flatten()
# Sort and find median manually to avoid compatibility issues
sorted_values = torch.sort(flattened).values
n = len(sorted_values)
if n % 2 == 0:
median_val = (sorted_values[n//2 - 1] + sorted_values[n//2]).item() / 2.0
else:
median_val = sorted_values[n//2].item()
# Mask-specific statistics
white_pixels = torch.sum(mask > 0.5).item() # Pixels considered "white" (active)
black_pixels = total_pixels - white_pixels
coverage = white_pixels / total_pixels
# Count unique values (useful for masks)
unique_values = torch.unique(mask)
unique_count = len(unique_values)
print("\n--- Global Statistics ---")
print(f"Min value: {min_val:.6f}")
print(f"Max value: {max_val:.6f}")
print(f"Mean value: {mean_val:.6f}")
print(f"Median value: {median_val:.6f}")
print(f"Variance: {variance_val:.6f}")
print(f"Standard deviation: {std_val:.6f}")
print(f"Value range: {max_val - min_val:.6f}")
print("\n--- Mask-Specific Statistics ---")
print(f"White pixels (>0.5): {white_pixels:,} ({coverage:.2%})")
print(f"Black pixels (≤0.5): {black_pixels:,} ({(1-coverage):.2%})")
print(f"Unique values: {unique_count}")
if unique_count <= 10:
print(f"Unique values list: {[f'{val:.3f}' for val in unique_values.tolist()]}")
# Mask density analysis
print(f"Mask density: {coverage:.4f}")
if coverage > 0.8:
print(" → High density mask (mostly white)")
elif coverage > 0.5:
print(" → Medium-high density mask")
elif coverage > 0.2:
print(" → Medium-low density mask")
elif coverage > 0.05:
print(" → Low density mask")
else:
print(" → Very sparse mask")
# Value distribution analysis
print("\n--- Value Distribution ---")
# For masks, show distribution more relevant to binary/grayscale nature
if unique_count <= 20: # If few unique values, show exact counts
print("Exact value counts:")
for val in unique_values:
count = torch.sum(mask == val).item()
percentage = count / total_pixels * 100
print(f" Value {val:.3f}: {count:,} pixels ({percentage:.1f}%)")
else:
# Create bins for histogram analysis
bins = torch.linspace(min_val, max_val, 11) # 10 bins
hist = torch.histc(flattened, bins=10, min=min_val, max=max_val)
print("Value distribution (10 bins):")
for i in range(len(hist)):
bin_start = bins[i].item()
bin_end = bins[i+1].item() if i < len(bins)-1 else max_val
count = hist[i].item()
percentage = count / total_pixels * 100
print(f" [{bin_start:.3f} - {bin_end:.3f}]: {count:,.0f} pixels ({percentage:.1f}%)")
# Percentiles
print("\n--- Percentiles ---")
percentiles = [1, 5, 10, 25, 50, 75, 90, 95, 99]
# Use the already sorted values from median calculation
for p in percentiles:
idx = int((p / 100.0) * (len(sorted_values) - 1))
value = sorted_values[idx].item()
print(f" {p:2d}th percentile: {value:.6f}")
# Connected components analysis (if requested)
if analyze_connected_components and coverage > 0 and coverage < 1:
print("\n--- Connected Components Analysis ---")
try:
# Simple connected components analysis
# For each batch item
total_components = 0
largest_component = 0
smallest_component = float('inf')
for b in range(batch_size):
mask_slice = mask[b] if len(mask.shape) == 3 else mask[b, :, :, 0]
binary_mask = (mask_slice > 0.5).float()
if binary_mask.sum() > 0: # Only analyze if there are white pixels
# Simple component counting (not perfect but gives an idea)
# Count clusters by looking at transitions
transitions = 0
prev_row_white = torch.zeros(width, dtype=torch.bool, device=mask.device)
for row in range(height):
current_row = binary_mask[row] > 0.5
# Count horizontal transitions in this row
if row == 0:
transitions += torch.sum(current_row[1:] != current_row[:-1]).item()
# Count vertical transitions
transitions += torch.sum(current_row != prev_row_white).item()
prev_row_white = current_row
# Rough estimate of components (very approximate)
estimated_components = max(1, transitions // 4)
total_components += estimated_components
component_size = binary_mask.sum().item()
largest_component = max(largest_component, component_size)
smallest_component = min(smallest_component, component_size)
if total_components > 0:
avg_components = total_components / batch_size
print(f"Estimated components per image: {avg_components:.1f}")
print(f"Largest component: {largest_component:,} pixels")
if smallest_component != float('inf'):
print(f"Smallest component: {smallest_component:,} pixels")
else:
print("No connected components found")
except Exception as e:
print(f"Connected components analysis failed: {str(e)}")
# Potential issues detection
print("\n--- Analysis ---")
if min_val < 0:
print("⚠️ Warning: Mask contains negative values!")
if max_val > 1:
print("⚠️ Warning: Mask contains values > 1.0!")
if min_val == max_val:
print("⚠️ Warning: Mask has constant values (uniform mask)!")
if unique_count == 2 and min_val == 0 and max_val == 1:
print("✓ Perfect binary mask (0 and 1 only)")
elif unique_count <= 5:
print(f"✓ Nearly binary mask ({unique_count} unique values)")
if coverage < 0.01:
print("⚠️ Notice: Very sparse mask - almost empty")
if coverage > 0.99:
print("⚠️ Notice: Very dense mask - almost full")
if std_val < 0.01:
print("⚠️ Notice: Very low standard deviation - mask might be nearly uniform")
print("================================\n")
# Return the mask and key statistics
return (mask, min_val, max_val, mean_val, variance_val, median_val, white_pixels)
class Tile:
"""
Represents a single tile with its content and position in the tiling grid
"""
def __init__(self, content, temporal_index, line_index, column_index,
temporal_range, spatial_range_h, spatial_range_w):
self.content = content # The actual tile tensor
self.temporal_index = temporal_index # Index in temporal dimension
self.line_index = line_index # Index in height/line dimension
self.column_index = column_index # Index in width/column dimension
# Store the ranges for overlap calculations
self.temporal_range = temporal_range # (start_frame, end_frame)
self.spatial_range_h = spatial_range_h # (start_h, end_h)
self.spatial_range_w = spatial_range_w # (start_w, end_w)
def __str__(self):
return f"Tile[T{self.temporal_index}:L{self.line_index}:C{self.column_index}] " \
f"Shape: {self.content.shape} " \
f"Temporal: {self.temporal_range} " \
f"Spatial: H{self.spatial_range_h} W{self.spatial_range_w}"
class TileAndStitchBack:
"""
Tile and Stitch Back node with proper dimension-wise stitching.
New approach:
1. Temporal tiling: Split video into chunks with overlap
2. Spatial tiling: Split each chunk into tiles with overlap
3. Processing: Apply transformations to each tile
4. Dimension-wise stitching:
a) Column-wise: Stitch tiles in same column (same temporal chunk)
b) Line-wise: Stitch columns together (complete temporal chunks)
c) Time-wise: Stitch temporal chunks together
This approach eliminates the black-to-color ramp issue by handling
overlaps systematically in each dimension.
"""
def __init__(self):
pass
@classmethod
def INPUT_TYPES(cls):
"""
Tile and Stitch Back inputs
"""
return {
"required": {
"video": ("IMAGE",),
"target_frames": ("INT", {"default": 81, "min": 16, "max": 200, "step": 1}),
"target_width": ("INT", {"default": 832, "min": 64, "max": 2048, "step": 8}),
"target_height": ("INT", {"default": 480, "min": 64, "max": 2048, "step": 8}),
"frame_overlap": ("INT", {"default": 10, "min": 0, "max": 40, "step": 1}),
"spatial_overlap": ("INT", {"default": 20, "min": 0, "max": 100, "step": 4}),
"color_shift_strength": ("FLOAT", {"default": 0.1, "min": 0.0, "max": 1.0, "step": 0.01}),
"debug_mode": ("BOOLEAN", {"default": True}),
},
}
RETURN_TYPES = ("IMAGE", "STRING")
RETURN_NAMES = ("stitched_video", "tile_info")
FUNCTION = "tile_and_stitch"
OUTPUT_NODE = True
CATEGORY = "TiledWan"
def tile_and_stitch(self, video, target_frames, target_width, target_height,
frame_overlap, spatial_overlap, color_shift_strength, debug_mode):
"""
Tile video temporally and spatially, apply transformations, then stitch back using dimension-wise approach
"""
print("\n" + "="*80)
print(" TILE AND STITCH BACK (NEW ALGORITHM)")
print("="*80)
print("🧩 Starting dimension-wise video tiling and stitching...")
try:
import random
import traceback
# Input video info
batch_size, height, width, channels = video.shape
print(f"📹 Input video shape: {video.shape} (B×H×W×C)")
print(f"🎯 Target tile size: {target_frames} frames × {target_width}×{target_height}")
print(f"🔗 Overlaps: {frame_overlap} frames, {spatial_overlap} pixels")
# Calculate tile ranges
temporal_tiles = self._calculate_temporal_tiles(batch_size, target_frames, frame_overlap)
spatial_tiles_h = self._calculate_spatial_tiles(height, target_height, spatial_overlap)
spatial_tiles_w = self._calculate_spatial_tiles(width, target_width, spatial_overlap)
print(f"⏱️ Temporal chunks: {len(temporal_tiles)}")
print(f"🗺️ Spatial grid: {len(spatial_tiles_h)}×{len(spatial_tiles_w)} tiles per frame")
print(f"📦 Total tiles: {len(temporal_tiles) * len(spatial_tiles_h) * len(spatial_tiles_w)}")
# STEP 1: Extract and process all tiles
print(f"\n🔸 STEP 1: Extracting and processing tiles...")
all_tiles = self._extract_and_process_tiles(
video, temporal_tiles, spatial_tiles_h, spatial_tiles_w,
color_shift_strength, debug_mode
)
# STEP 2: Column-wise stitching (stitch tiles in same column, same temporal chunk)
print(f"\n🔸 STEP 2: Column-wise stitching...")
column_strips = self._stitch_columns(all_tiles, spatial_overlap, debug_mode)
# STEP 3: Line-wise stitching (stitch column strips to complete temporal chunks)
print(f"\n🔸 STEP 3: Line-wise stitching...")
temporal_chunks = self._stitch_lines(column_strips, spatial_overlap, debug_mode)
# STEP 4: Time-wise stitching (stitch temporal chunks together)
print(f"\n🔸 STEP 4: Time-wise stitching...")
stitched_video = self._stitch_temporal_chunks_new(temporal_chunks, temporal_tiles, frame_overlap)
# STEP 5: Crop back to original dimensions
print(f"\n🔸 STEP 5: Cropping to original dimensions...")
print(f" 📐 Stitched video shape: {stitched_video.shape}")
print(f" 🎯 Target shape: [{batch_size}, {height}, {width}, {channels}]")
# Crop spatial dimensions back to original size
final_video = stitched_video[:batch_size, :height, :width, :channels]
print(f" ✂️ Cropped to: {final_video.shape}")
# Generate summary
tile_info_summary = self._generate_tile_info_summary_new(
all_tiles, temporal_tiles, spatial_tiles_h, spatial_tiles_w
)
print(f"✅ Dimension-wise tiling and stitching completed!")
print(f"📤 Final video shape: {final_video.shape} (cropped to match input)")
print(f"🧩 Total tiles processed: {len(all_tiles)}")
print(f"📏 Original input: {batch_size}×{height}×{width}×{channels}")
print(f"📏 After stitching: {stitched_video.shape}")
print(f"📏 After cropping: {final_video.shape}")
print("="*80 + "\n")
return (final_video, tile_info_summary)
except Exception as e:
print(f"❌ Error in tile and stitch: {str(e)}")
print(f"📋 Full traceback:")
print(traceback.format_exc())
print("="*80 + "\n")
# Return original video in case of error
dummy_info = f"Error during tiling: {str(e)}"
return (video, dummy_info)
def _extract_and_process_tiles(self, video, temporal_tiles, spatial_tiles_h, spatial_tiles_w,
color_shift_strength, debug_mode):
"""
Extract all tiles and apply transformations, storing them in Tile objects
"""
all_tiles = []
for temporal_idx, (t_start, t_end) in enumerate(temporal_tiles):
if debug_mode:
print(f" 🎬 Temporal chunk {temporal_idx + 1}/{len(temporal_tiles)} (frames {t_start}-{t_end-1})")
# Extract temporal chunk
chunk = video[t_start:t_end]
# Process each spatial tile within this temporal chunk
for h_idx, (h_start, h_end) in enumerate(spatial_tiles_h):
for w_idx, (w_start, w_end) in enumerate(spatial_tiles_w):
if debug_mode and temporal_idx == 0: # Only show spatial info for first temporal chunk
print(f" 🧩 Spatial tile L{h_idx}:C{w_idx} H[{h_start}:{h_end}] × W[{w_start}:{w_end}]")
# Extract spatial tile
tile_content = chunk[:, h_start:h_end, w_start:w_end, :]
# Apply color transformation (for debugging)
tile_transformed = self._apply_color_shift(tile_content, color_shift_strength,
temporal_idx, h_idx, w_idx)
# Create Tile object
tile = Tile(
content=tile_transformed,
temporal_index=temporal_idx,
line_index=h_idx,
column_index=w_idx,
temporal_range=(t_start, t_end),
spatial_range_h=(h_start, h_end),
spatial_range_w=(w_start, w_end)
)
all_tiles.append(tile)
print(f" ✅ Extracted {len(all_tiles)} tiles")
return all_tiles
def _stitch_columns(self, all_tiles, spatial_overlap, debug_mode):
"""
Stitch tiles vertically (same column, same temporal chunk) to create column strips
"""
# Group tiles by temporal_index and column_index
column_groups = {}
for tile in all_tiles:
key = (tile.temporal_index, tile.column_index)
if key not in column_groups:
column_groups[key] = []
column_groups[key].append(tile)
# Sort each group by line_index
for key in column_groups:
column_groups[key].sort(key=lambda t: t.line_index)
column_strips = []
for (temporal_idx, col_idx), column_tiles in column_groups.items():
if debug_mode and temporal_idx == 0: # Only show info for first temporal chunk
print(f" 📏 Column {col_idx}: {len(column_tiles)} tiles")
# Stitch tiles in this column vertically
if len(column_tiles) == 1:
# Single tile - no stitching needed
column_strip = column_tiles[0].content
else:
# Multiple tiles - stitch vertically with overlap
column_strip = self._stitch_tiles_vertically(column_tiles, spatial_overlap)
# Store column strip with its position info
strip_info = {
'content': column_strip,
'temporal_index': temporal_idx,
'column_index': col_idx,
'spatial_range_w': column_tiles[0].spatial_range_w, # All tiles in column have same W range
'spatial_range_h': (column_tiles[0].spatial_range_h[0], column_tiles[-1].spatial_range_h[1])
}
column_strips.append(strip_info)
print(f" ✅ Created {len(column_strips)} column strips")
return column_strips
def _stitch_lines(self, column_strips, spatial_overlap, debug_mode):
"""
Stitch column strips horizontally (same temporal chunk) to create complete temporal chunks
"""
# Group column strips by temporal_index
temporal_groups = {}
for strip in column_strips:
temporal_idx = strip['temporal_index']
if temporal_idx not in temporal_groups:
temporal_groups[temporal_idx] = []
temporal_groups[temporal_idx].append(strip)
# Sort each group by column_index
for temporal_idx in temporal_groups:
temporal_groups[temporal_idx].sort(key=lambda s: s['column_index'])
temporal_chunks = []
for temporal_idx, strips in temporal_groups.items():
if debug_mode and temporal_idx == 0: # Only show info for first temporal chunk
print(f" 📐 Temporal chunk {temporal_idx}: {len(strips)} column strips")
# Stitch column strips horizontally
if len(strips) == 1:
# Single strip - no stitching needed
chunk_content = strips[0]['content']
else:
# Multiple strips - stitch horizontally with overlap
chunk_content = self._stitch_strips_horizontally(strips, spatial_overlap)
# Store temporal chunk with its info
chunk_info = {
'content': chunk_content,
'temporal_index': temporal_idx,
'spatial_range_h': strips[0]['spatial_range_h'], # All strips have same H range
'spatial_range_w': (strips[0]['spatial_range_w'][0], strips[-1]['spatial_range_w'][1])
}
temporal_chunks.append(chunk_info)
# Sort by temporal index
temporal_chunks.sort(key=lambda c: c['temporal_index'])
print(f" ✅ Created {len(temporal_chunks)} temporal chunks")
return temporal_chunks
def _stitch_tiles_vertically(self, column_tiles, spatial_overlap):
"""
Stitch tiles vertically (in height dimension) with proper fade blending
"""
if len(column_tiles) == 1:
return column_tiles[0].content
# Get dimensions
first_tile = column_tiles[0].content
total_height = sum(tile.spatial_range_h[1] - tile.spatial_range_h[0] for tile in column_tiles)
# Subtract overlaps
total_height -= spatial_overlap * (len(column_tiles) - 1)
width = first_tile.shape[2]
channels = first_tile.shape[3]
frames = first_tile.shape[0]
# Create result tensor
result = torch.zeros((frames, total_height, width, channels),
dtype=first_tile.dtype, device=first_tile.device)
current_h = 0
for i, tile in enumerate(column_tiles):
tile_h = tile.content.shape[1]
if i == 0:
# First tile - place directly
result[:, current_h:current_h+tile_h, :, :] = tile.content
current_h += tile_h
else:
# Subsequent tiles - check for last tile with large overlap
is_last_tile = (i == len(column_tiles) - 1)
# Calculate actual overlap by comparing tile positions
expected_start = tile.spatial_range_h[0]
actual_overlap = current_h - expected_start
actual_overlap = max(0, actual_overlap) # Ensure non-negative
if is_last_tile and actual_overlap > spatial_overlap:
# Last tile with large overlap - fade across entire overlapping region
print(f" 🔥 Last tile in column: Large overlap detected ({actual_overlap} > {spatial_overlap})")
overlap_start = expected_start
overlap_size = actual_overlap
non_overlap_start = current_h
non_overlap_size = tile_h - actual_overlap
# Place non-overlapping part
if non_overlap_size > 0:
result[:, non_overlap_start:non_overlap_start+non_overlap_size, :, :] = \
tile.content[:, actual_overlap:, :, :]
# Handle large overlapping region with fade blending
if overlap_size > 0:
existing_region = result[:, overlap_start:overlap_start+overlap_size, :, :]
new_region = tile.content[:, :overlap_size, :, :]
# Create fade mask for entire overlapping region
fade_mask = self._create_vertical_fade_mask(overlap_size, existing_region.dtype, existing_region.device)
# Apply fade blending
blended = existing_region * (1.0 - fade_mask) + new_region * fade_mask
result[:, overlap_start:overlap_start+overlap_size, :, :] = blended
current_h += non_overlap_size
else:
# Normal tile with standard overlap handling
overlap_start = current_h - spatial_overlap
# Place non-overlapping part first
non_overlap_h = tile_h - spatial_overlap
result[:, current_h:current_h+non_overlap_h, :, :] = tile.content[:, spatial_overlap:, :, :]
# Handle overlapping region with fade blending
if spatial_overlap > 0:
existing_region = result[:, overlap_start:current_h, :, :]
new_region = tile.content[:, :spatial_overlap, :, :]
# Create vertical fade mask
fade_mask = self._create_vertical_fade_mask(spatial_overlap, existing_region.dtype, existing_region.device)
# Apply fade blending
blended = existing_region * (1.0 - fade_mask) + new_region * fade_mask
result[:, overlap_start:current_h, :, :] = blended
current_h += non_overlap_h
return result
def _stitch_strips_horizontally(self, strips, spatial_overlap):
"""
Stitch column strips horizontally (in width dimension) with proper fade blending
"""
if len(strips) == 1:
return strips[0]['content']
# Get dimensions
first_strip = strips[0]['content']
total_width = sum(s['spatial_range_w'][1] - s['spatial_range_w'][0] for s in strips)
# Subtract overlaps
total_width -= spatial_overlap * (len(strips) - 1)
height = first_strip.shape[1]
channels = first_strip.shape[3]
frames = first_strip.shape[0]
# Create result tensor
result = torch.zeros((frames, height, total_width, channels),
dtype=first_strip.dtype, device=first_strip.device)
current_w = 0
for i, strip in enumerate(strips):
strip_content = strip['content']
strip_w = strip_content.shape[2]
if i == 0:
# First strip - place directly
result[:, :, current_w:current_w+strip_w, :] = strip_content
current_w += strip_w
else:
# Subsequent strips - check for last strip with large overlap
is_last_strip = (i == len(strips) - 1)
# Calculate actual overlap by comparing strip positions
expected_start = strip['spatial_range_w'][0]
actual_overlap = current_w - expected_start
actual_overlap = max(0, actual_overlap) # Ensure non-negative
if is_last_strip and actual_overlap > spatial_overlap:
# Last strip with large overlap - fade across entire overlapping region
print(f" 🔥 Last strip in line: Large overlap detected ({actual_overlap} > {spatial_overlap})")
overlap_start = expected_start
overlap_size = actual_overlap
non_overlap_start = current_w
non_overlap_size = strip_w - actual_overlap
# Place non-overlapping part
if non_overlap_size > 0:
result[:, :, non_overlap_start:non_overlap_start+non_overlap_size, :] = \
strip_content[:, :, actual_overlap:, :]
# Handle large overlapping region with fade blending
if overlap_size > 0:
existing_region = result[:, :, overlap_start:overlap_start+overlap_size, :]
new_region = strip_content[:, :, :overlap_size, :]
# Create fade mask for entire overlapping region
fade_mask = self._create_horizontal_fade_mask(overlap_size, existing_region.dtype, existing_region.device)
# Apply fade blending
blended_region = existing_region * (1 - fade_mask) + new_region * fade_mask
result[:, :, overlap_start:overlap_start+overlap_size, :] = blended_region
current_w += non_overlap_size
else:
# Normal strip with standard overlap handling
overlap_start = current_w - spatial_overlap
# Place non-overlapping part first
non_overlap_w = strip_w - spatial_overlap
result[:, :, current_w:current_w+non_overlap_w, :] = strip_content[:, :, spatial_overlap:, :]
# Handle overlapping region with fade blending
if spatial_overlap > 0:
existing_region = result[:, :, overlap_start:current_w, :]
new_region = strip_content[:, :, :spatial_overlap, :]
# Create horizontal fade mask
fade_mask = self._create_horizontal_fade_mask(spatial_overlap, existing_region.dtype, existing_region.device)
# Apply fade blending
blended_region = existing_region * (1 - fade_mask) + new_region * fade_mask
result[:, :, overlap_start:current_w, :] = blended_region
current_w += non_overlap_w
return result
return result
def _create_vertical_fade_mask(self, fade_size, dtype, device):
"""Create a vertical fade mask for blending tiles vertically."""
fade_mask = torch.linspace(0, 1, fade_size, dtype=dtype, device=device)
# Shape: (1, fade_size, 1, 1) for broadcasting
return fade_mask.view(1, fade_size, 1, 1)
def _create_horizontal_fade_mask(self, fade_size, dtype, device):
"""Create a horizontal fade mask for blending tiles horizontally."""
fade_mask = torch.linspace(0, 1, fade_size, dtype=dtype, device=device)
# Shape: (1, 1, fade_size, 1) for broadcasting
return fade_mask.view(1, 1, fade_size, 1)
def _stitch_temporal_chunks_new(self, temporal_chunks, temporal_tiles, frame_overlap):
"""Stitch temporal chunks to create the final output with temporal blending."""
if not temporal_chunks:
return None
if len(temporal_chunks) == 1:
return temporal_chunks[0]['content']
# Get dimensions from first chunk
first_chunk_content = temporal_chunks[0]['content']
# Calculate total frames accounting for overlaps
total_frames = sum(chunk['content'].shape[0] for chunk in temporal_chunks)
if frame_overlap > 0:
total_frames -= (len(temporal_chunks) - 1) * frame_overlap
# Get spatial dimensions from the stitched chunks (not original input)
height = first_chunk_content.shape[1]
width = first_chunk_content.shape[2]
channels = first_chunk_content.shape[3]
print(f" 📏 Temporal stitching dimensions:")
print(f" • Total frames: {total_frames}")
print(f" • Spatial size: {height}×{width}")
print(f" • Channels: {channels}")
print(f" • Chunk sizes: {[chunk['content'].shape for chunk in temporal_chunks]}")
# Initialize result tensor with correct dimensions
result = torch.zeros(
(total_frames, height, width, channels),
dtype=first_chunk_content.dtype,
device=first_chunk_content.device
)
current_t = 0
for i, chunk_info in enumerate(temporal_chunks):
chunk_content = chunk_info['content']
chunk_frames = chunk_content.shape[0]
if i == 0:
# First chunk - place entirely
result[current_t:current_t+chunk_frames] = chunk_content
current_t += chunk_frames
else:
# Subsequent chunks - check for last chunk with large overlap
is_last_chunk = (i == len(temporal_chunks) - 1)
# Get the temporal range for this chunk from temporal_tiles
temporal_tile_info = temporal_tiles[i]
expected_start = temporal_tile_info[0]
actual_overlap = current_t - expected_start
actual_overlap = max(0, actual_overlap) # Ensure non-negative
if is_last_chunk and actual_overlap > frame_overlap:
# Last chunk with large overlap - fade across entire overlapping region
print(f" 🔥 Last temporal chunk: Large overlap detected ({actual_overlap} > {frame_overlap})")
overlap_start = expected_start
overlap_size = actual_overlap
non_overlap_start = current_t
non_overlap_size = chunk_frames - actual_overlap
# Place non-overlapping part
if non_overlap_size > 0:
result[non_overlap_start:non_overlap_start+non_overlap_size] = \
chunk_content[actual_overlap:]
# Handle large overlapping region with fade blending
if overlap_size > 0:
existing_region = result[overlap_start:overlap_start+overlap_size]
new_region = chunk_content[:overlap_size]
# Create temporal fade mask for entire overlapping region
fade_mask = self._create_temporal_fade_mask(overlap_size, existing_region.dtype, existing_region.device)
# Apply fade blending
blended_region = existing_region * (1 - fade_mask) + new_region * fade_mask
result[overlap_start:overlap_start+overlap_size] = blended_region
current_t += non_overlap_size
else:
# Normal chunk with standard overlap handling
non_overlap_frames = chunk_frames - frame_overlap
overlap_start = current_t - frame_overlap
# Place non-overlapping part
result[current_t:current_t+non_overlap_frames] = chunk_content[frame_overlap:]
# Handle overlapping region with temporal fade blending
if frame_overlap > 0:
existing_region = result[overlap_start:current_t]
new_region = chunk_content[:frame_overlap]
# Create temporal fade mask
fade_mask = self._create_temporal_fade_mask(frame_overlap, existing_region.dtype, existing_region.device)
# Apply fade blending
blended_region = existing_region * (1 - fade_mask) + new_region * fade_mask
result[overlap_start:current_t] = blended_region
current_t += non_overlap_frames
return result
def _create_temporal_fade_mask(self, fade_size, dtype, device):
"""Create a temporal fade mask for blending across time dimension."""
fade_mask = torch.linspace(0, 1, fade_size, dtype=dtype, device=device)
# Shape: (fade_size, 1, 1, 1) for broadcasting
return fade_mask.view(fade_size, 1, 1, 1)
def _generate_tile_info_summary_new(self, tiles, temporal_tiles, spatial_tiles_h, spatial_tiles_w):
"""Generate a summary of tile processing for the new dimension-wise algorithm."""
summary_lines = []
summary_lines.append("=== Tile Processing Summary (Dimension-wise Algorithm) ===")
summary_lines.append(f"Input video - Temporal chunks: {len(temporal_tiles)}")
summary_lines.append(f"Input video - Spatial grid: {len(spatial_tiles_h)}×{len(spatial_tiles_w)}")
summary_lines.append(f"Total tiles processed: {len(tiles)}")
# Group tiles by temporal chunk
temporal_chunks = {}
for tile in tiles:
t_idx = tile.temporal_index
if t_idx not in temporal_chunks:
temporal_chunks[t_idx] = []
temporal_chunks[t_idx].append(tile)
summary_lines.append(f"Temporal chunks: {len(temporal_chunks)}")
# Analyze grid structure
for t_idx, chunk_tiles in temporal_chunks.items():
line_groups = {}
for tile in chunk_tiles:
l_idx = tile.line_index
if l_idx not in line_groups:
line_groups[l_idx] = []
line_groups[l_idx].append(tile)
max_cols = max(len(line_tiles) for line_tiles in line_groups.values())
summary_lines.append(f" Chunk {t_idx}: {len(line_groups)} lines × {max_cols} columns")
summary_lines.append("Processing order: Column-wise → Line-wise → Temporal → Crop")
summary_lines.append("Last tiles in each dimension use full-region fade blending")
summary_lines.append("Final video cropped back to original input dimensions")
summary_lines.append("="*60)
return "\n".join(summary_lines)
def _calculate_temporal_tiles(self, total_frames, target_frames, overlap):
"""Calculate temporal tile ranges with overlap handling"""
tiles = []
if total_frames <= target_frames:
# If video is shorter than target, use the whole video
tiles.append((0, total_frames))
else:
stride = target_frames - overlap
current = 0
while current < total_frames:
end = min(current + target_frames, total_frames)
# If this would be the last tile and it doesn't cover remaining frames
remaining = total_frames - end
if remaining > 0:
# Add this tile normally
tiles.append((current, end))
# Check if we need another tile to cover the remaining frames
if remaining < stride:
# Add a final tile that is exactly target_frames but starts from the end
final_start = total_frames - target_frames
tiles.append((final_start, total_frames))
break
else:
# This tile reaches exactly to the end
tiles.append((current, end))
break
current += stride
return tiles
def _calculate_spatial_tiles(self, total_size, target_size, overlap):
"""Calculate spatial tile ranges with overlap handling"""
tiles = []
if total_size <= target_size:
# If dimension is smaller than target, use the whole dimension
tiles.append((0, total_size))
else:
stride = target_size - overlap
current = 0
while current < total_size:
end = min(current + target_size, total_size)
# If this would be the last tile and it doesn't cover remaining pixels
remaining = total_size - end
if remaining > 0:
# Add this tile normally
tiles.append((current, end))
# Check if we need another tile to cover the remaining pixels
if remaining < stride:
# Add a final tile that is exactly target_size but starts from the end
final_start = total_size - target_size
tiles.append((final_start, total_size))
break
else:
# This tile reaches exactly to the end
tiles.append((current, end))
break
current += stride
return tiles
def _apply_color_shift(self, tile, strength, temporal_idx, h_idx, w_idx):
"""Apply color transformation to a tile for debugging purposes"""
if strength <= 0:
return tile.clone()
# Generate deterministic but varied color shifts based on tile indices
random.seed(temporal_idx * 1000 + h_idx * 100 + w_idx)
# Random RGB shifts
r_shift = (random.random() - 0.5) * 2 * strength
g_shift = (random.random() - 0.5) * 2 * strength
b_shift = (random.random() - 0.5) * 2 * strength
transformed = tile.clone()
if tile.shape[-1] >= 3: # RGB channels
transformed[:, :, :, 0] = torch.clamp(transformed[:, :, :, 0] + r_shift, 0, 1)
transformed[:, :, :, 1] = torch.clamp(transformed[:, :, :, 1] + g_shift, 0, 1)
transformed[:, :, :, 2] = torch.clamp(transformed[:, :, :, 2] + b_shift, 0, 1)
# Debug output to confirm color shift is being applied
if strength > 0:
print(f" 🎨 Color shift applied: R{r_shift:+.3f}, G{g_shift:+.3f}, B{b_shift:+.3f}")
return transformed
def _place_tile_with_overlap(self, target, tile, h_start, h_end, w_start, w_end, spatial_overlap):
"""Place tile into target tensor, handling overlaps with spatial fade blending"""
tile_h, tile_w = tile.shape[1:3]
# Get the target region
target_region = target[:, h_start:h_start+tile_h, w_start:w_start+tile_w, :]
# Check if there's existing content (non-zero) in the target area
if target_region.sum() > 0:
# Always use production-quality fade blending regardless of color shift strength
# Color shift is purely for visual debugging and shouldn't affect blending quality
fade_mask = self._create_spatial_fade_mask(tile_h, tile_w, target_region, tile, spatial_overlap)
# Apply fade blending: existing * (1 - fade_mask) + tile * fade_mask
blended = target_region * (1.0 - fade_mask) + tile * fade_mask
target[:, h_start:h_start+tile_h, w_start:w_start+tile_w, :] = blended
else:
# First tile in this area or non-overlapping area - always place the tile
# (this includes color-shifted tiles when debugging is enabled)
target[:, h_start:h_start+tile_h, w_start:w_start+tile_w, :] = tile
def _create_spatial_fade_mask(self, tile_h, tile_w, existing, new_tile, spatial_overlap):
"""Create a spatial fade mask for smooth blending between tiles"""
# Create base mask (1.0 = use new tile, 0.0 = use existing)
mask = torch.ones(1, tile_h, tile_w, 1, dtype=existing.dtype, device=existing.device)
# Define fade distance using spatial_overlap parameter
fade_h = min(spatial_overlap, tile_h // 2) # Fade over spatial_overlap pixels or half tile height
fade_w = min(spatial_overlap, tile_w // 2) # Fade over spatial_overlap pixels or half tile width
# Check which edges have existing content to determine fade direction
has_top = existing[:, :fade_h, :, :].sum() > 0
has_bottom = existing[:, -fade_h:, :, :].sum() > 0
has_left = existing[:, :, :fade_w, :].sum() > 0
has_right = existing[:, :, -fade_w:, :].sum() > 0
# Create fade gradients for each edge that has existing content
if has_top:
# Fade from 0 at top to 1 after fade_h pixels
for i in range(fade_h):
alpha = i / fade_h
mask[:, i, :, :] = alpha
if has_bottom:
# Fade from 1 before last fade_h pixels to 0 at bottom
for i in range(fade_h):
alpha = 1.0 - (i / fade_h)
mask[:, tile_h - 1 - i, :, :] = alpha
if has_left:
# Fade from 0 at left to 1 after fade_w pixels
for i in range(fade_w):
alpha = i / fade_w
mask[:, :, i, :] = torch.minimum(mask[:, :, i, :], torch.tensor(alpha, dtype=mask.dtype, device=mask.device))
if has_right:
# Fade from 1 before last fade_w pixels to 0 at right
for i in range(fade_w):
alpha = 1.0 - (i / fade_w)
mask[:, :, tile_w - 1 - i, :] = torch.minimum(mask[:, :, tile_w - 1 - i, :], torch.tensor(alpha, dtype=mask.dtype, device=mask.device))
# Broadcast mask to match tile dimensions
return mask.expand_as(new_tile)
def _stitch_temporal_chunks(self, chunks, temporal_tiles, total_frames, height, width, channels, overlap):
"""Stitch temporal chunks back together with temporal fade blending"""
result = torch.zeros((total_frames, height, width, channels), dtype=chunks[0].dtype, device=chunks[0].device)
for i, ((t_start, t_end), chunk) in enumerate(zip(temporal_tiles, chunks)):
chunk_frames = chunk.shape[0]
if i == 0:
# First chunk, place directly
result[t_start:t_start+chunk_frames] = chunk
else:
# Handle overlap with previous chunk using temporal fade
prev_end = temporal_tiles[i-1][1]
overlap_start = max(t_start, prev_end - overlap)
overlap_end = min(t_end, prev_end)
if overlap_start < overlap_end:
# There's an overlap, apply temporal fade blending
overlap_frames = overlap_end - overlap_start
chunk_overlap_start = overlap_start - t_start
# Get overlapping regions
existing_frames = result[overlap_start:overlap_end]
new_frames = chunk[chunk_overlap_start:chunk_overlap_start+overlap_frames]
# Create temporal fade mask
fade_mask = self._create_temporal_fade_mask(overlap_frames, existing_frames.dtype, existing_frames.device)
# Apply temporal fade: existing * (1 - fade) + new * fade
blended_frames = existing_frames * (1.0 - fade_mask) + new_frames * fade_mask
result[overlap_start:overlap_end] = blended_frames
# Place non-overlapping frames
if overlap_end < t_start + chunk_frames:
non_overlap_start = overlap_end
chunk_offset = non_overlap_start - t_start
result[non_overlap_start:t_start+chunk_frames] = chunk[chunk_offset:]
else:
# No overlap, place directly
result[t_start:t_start+chunk_frames] = chunk
return result
def _create_temporal_fade_mask(self, overlap_frames, dtype, device):
"""Create a temporal fade mask for smooth frame transitions"""
# Create fade mask: 0.0 at start (keep existing) to 1.0 at end (use new)
fade_values = torch.linspace(0.0, 1.0, overlap_frames, dtype=dtype, device=device)
# Reshape to broadcast properly: [frames, 1, 1, 1]
fade_mask = fade_values.view(overlap_frames, 1, 1, 1)
return fade_mask
def _generate_tile_info_summary(self, tile_info_list, temporal_tiles, spatial_tiles_h, spatial_tiles_w):
"""Generate a summary of the tiling process"""
summary = f"=== TILE AND STITCH SUMMARY ===\n"
summary += f"Total tiles processed: {len(tile_info_list)}\n"
summary += f"Temporal chunks: {len(temporal_tiles)}\n"
summary += f"Spatial tiles per frame: {len(spatial_tiles_h)}×{len(spatial_tiles_w)}\n\n"
summary += "Temporal chunks:\n"
for i, (start, end) in enumerate(temporal_tiles):
summary += f" Chunk {i+1}: frames {start}-{end-1} ({end-start} frames)\n"
summary += f"\nSpatial tiles (Height):\n"
for i, (start, end) in enumerate(spatial_tiles_h):
summary += f" Row {i+1}: pixels {start}-{end-1} ({end-start} pixels)\n"
summary += f"\nSpatial tiles (Width):\n"
for i, (start, end) in enumerate(spatial_tiles_w):
summary += f" Col {i+1}: pixels {start}-{end-1} ({end-start} pixels)\n"
summary += f"\nProcessing completed successfully!"
return summary
class TiledWanVideoVACEpipe:
"""
Dimension-wise Tiled WanVideo VACE Pipeline - The ultimate node for processing large videos.
This node combines the sophisticated dimension-wise tiling system from TileAndStitchBack with
the complete WanVideo VACE pipeline. It handles large videos using the advanced algorithm:
1. Temporal tiling: Split video into chunks with overlap (default: 81 frames, 10-frame overlap)
2. Spatial tiling: Split each chunk into tiles with overlap (default: 832×480, 20-pixel overlap)
3. WanVideo VACE processing: Process each tile through the complete pipeline
4. Dimension-wise stitching:
a) Column-wise: Stitch tiles vertically within each temporal chunk
b) Line-wise: Stitch columns horizontally to complete temporal chunks
c) Temporal: Stitch temporal chunks together across time
5. STRICT OVERLAP CONTROL: Overlap blending is limited to specified parameters
- Spatial overlap never exceeds spatial_overlap pixels
- Temporal overlap never exceeds frame_overlap frames
- Extra pixels/frames are ignored (they help generation but don't affect output)
6. Cropping: Final video cropped to exact input dimensions
7. Memory management: Proper model offloading between tiles to prevent leaks
This enables processing of arbitrarily large videos that would otherwise exceed memory limits,
while maintaining WanVideo's optimal performance and eliminating overlap artifacts through
sophisticated dimension-wise fade blending with strict overlap control.
"""
def __init__(self):
pass
@classmethod
def INPUT_TYPES(cls):
"""
Complete inputs combining tiling parameters with WanVideo VACE pipeline
"""
return {
"required": {
# Input video and mask
"video": ("IMAGE", {"tooltip": "Input video frames to process. Can be very large videos that will be automatically tiled for memory efficiency."}),
"mask": ("MASK", {"tooltip": "Mask defining areas to be processed. Should match the video dimensions and frame count."}),
# Core WanVideo pipeline inputs
"model": ("WANVIDEOMODEL", {"tooltip": "WanVideo model for video generation/processing."}),
"vae": ("WANVAE", {"tooltip": "Video VAE for encoding/decoding frames."}),
# Tiling parameters
"target_frames": ("INT", {"default": 81, "min": 16, "max": 200, "step": 1, "tooltip": "Target number of frames per temporal chunk. The model works best with 81 frames. If different, might generate artefact or frame drops"}),
"target_width": ("INT", {"default": 832, "min": 64, "max": 2048, "step": 8, "tooltip": "Target width for spatial tiles. Works best around 832."}),
"target_height": ("INT", {"default": 480, "min": 64, "max": 2048, "step": 8, "tooltip": "Target height for spatial tiles. Works best around 480."}),
"frame_overlap": ("INT", {"default": 10, "min": 0, "max": 40, "step": 1, "tooltip": "Number of overlapping frames between temporal chunks."}),
"spatial_overlap": ("INT", {"default": 20, "min": 0, "max": 100, "step": 4, "tooltip": "Pixel overlap between spatial tiles."}),
# WanVideoSampler core parameters
"steps": ("INT", {"default": 30, "min": 1, "tooltip": "Number of diffusion sampling steps."}),
"cfg": ("FLOAT", {"default": 6.0, "min": 0.0, "max": 30.0, "step": 0.01, "tooltip": "Classifier-free guidance strength. Higher values follow the conditioning more closely."}),
"shift": ("FLOAT", {"default": 5.0, "min": 0.0, "max": 1000.0, "step": 0.01, "tooltip": "Shift parameter for the sampling schedule."}),
"seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff, "tooltip": "Random seed for reproducible results. Use the same seed to get identical outputs."}),
"scheduler": (["unipc", "unipc/beta", "dpm++", "dpm++/beta","dpm++_sde", "dpm++_sde/beta", "euler", "euler/beta", "euler/accvideo", "deis", "lcm", "lcm/beta", "flowmatch_causvid", "flowmatch_distill", "multitalk"],
{"default": 'unipc', "tooltip": "Sampling scheduler algorithm."}),
# WanVideoVACEEncode parameters
"vace_strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.001, "tooltip": "VACE encoding strength. Controls how much the video is modified during processing. 1.0 mean total reconstruction, 0.0 means no change."}),
"vace_start_percent": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}),
"vace_end_percent": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}),
# WanVideoDecode parameters
"decode_enable_vae_tiling": ("BOOLEAN", {"default": False}),
"decode_tile_x": ("INT", {"default": 272, "min": 40, "max": 2048, "step": 8}),
"decode_tile_y": ("INT", {"default": 272, "min": 40, "max": 2048, "step": 8}),
"decode_tile_stride_x": ("INT", {"default": 144, "min": 32, "max": 2040, "step": 8}),
"decode_tile_stride_y": ("INT", {"default": 128, "min": 32, "max": 2040, "step": 8}),
# Processing parameters
"debug_mode": ("BOOLEAN", {"default": True}),
"debug_color_shift": ("BOOLEAN", {"default": True}),
"force_offload_between_tiles": ("BOOLEAN", {"default": True}),
},
"optional": {
# WanVideoSampler optional inputs
"text_embeds": ("WANVIDEOTEXTEMBEDS",),
"samples": ("LATENT",),
"riflex_freq_index": ("INT", {"default": 0, "min": 0, "max": 1000, "step": 1}),
"denoise_strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}),
"force_offload": ("BOOLEAN", {"default": True}),
"batched_cfg": ("BOOLEAN", {"default": False}),
"rope_function": (["default", "comfy"], {"default": "comfy"}),
# WanVideoSampler advanced optional inputs
"feta_args": ("FETAARGS",),
"context_options": ("CONTEXTOPTIONS",),
"loop_args": ("LOOPARGS",),
"sigmas": ("SIGMAS",),
"unianimate_poses": ("UNIANIMATE_POSES",),
"fantasytalking_embeds": ("FANTASYTALKING_EMBEDS",),
"uni3c_embeds": ("UNI3C_EMBEDS",),
"multitalk_embeds": ("MULTITALK_EMBEDS",),
"freeinit_args": ("FREEINIT_ARGS",),
# External pre-built arguments (for modularity)
"cache_args": ("CACHEARGS",),
"slg_args": ("SLGARGS",),
"experimental_args": ("EXPERIMENTALARGS",),
# WanVideoVACEEncode optional inputs
"vace_ref_images": ("IMAGE",),
"vace_tiled_vae": ("BOOLEAN", {"default": False}),
# WanVideoDecode optional inputs
"decode_normalization": (["default", "minmax"], {"default": "default"}),
},
}
RETURN_TYPES = ("IMAGE", "STRING")
RETURN_NAMES = ("processed_video", "processing_info")
OUTPUT_TOOLTIPS = ("Processed video with WanVideo VACE pipeline applied through tiled processing. Maintains original dimensions and quality.",
"Detailed information about the processing including tile counts, timing, and processing summary.")
FUNCTION = "process_tiled_wanvideo"
OUTPUT_NODE = True
CATEGORY = "TiledWan"
DESCRIPTION = """
Video processing node that tiles WanVideo VACE pipeline.
Enables processing of arbitrarily large videos that would otherwise exceed model limits.
Uses dimension-wise tiling with temporal and spatial overlap to maintain
quality and eliminate artifacts. The complete WanVideo VACE pipeline is applied to each
tile with proper stitching and blending for seamless results.
Give last chunk frame as reference for the upcoming generation in order to maintain consistency across temporal chunks
Key Features:
- Handles any video size through tiling
- Temporal consistency across chunks thanks to cross-chunk reference
- Spatial seamless stitching with fade blending
- STRICT OVERLAP CONTROL: Overlap never exceeds specified parameters
- Memory-efficient processing with model offloading
- Complete WanVideo VACE pipeline integration
- Debug modes for monitoring and troubleshooting
Overlap Behavior:
- Overlap blending is limited to frame_overlap and spatial_overlap parameters
- When actual overlap exceeds specified limits, blending starts from the correct position
- Extra pixels/frames are skipped (they help generation but don't affect output)
- This ensures consistent results regardless of tile positioning
"""
def process_tiled_wanvideo(self, video, mask, model, vae, target_frames, target_width, target_height,
frame_overlap, spatial_overlap, steps, cfg, shift, seed, scheduler,
vace_strength, vace_start_percent, vace_end_percent,
decode_enable_vae_tiling, decode_tile_x, decode_tile_y,
decode_tile_stride_x, decode_tile_stride_y, debug_mode, debug_color_shift,
force_offload_between_tiles, **kwargs):
"""
Process large video through tiled WanVideo VACE pipeline with dimension-wise stitching
"""
print("\n" + "="*80)
print(" TILED WANVIDEO VACE PIPELINE (DIMENSION-WISE)")
print("="*80)
print("🚀 Starting dimension-wise tiled WanVideo VACE processing...")
try:
# Import WanVideo nodes
custom_nodes_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "ComfyUI-WanVideoWrapper")
if custom_nodes_path not in sys.path:
sys.path.append(custom_nodes_path)
parent_path = os.path.dirname(custom_nodes_path)
if parent_path not in sys.path:
sys.path.insert(0, parent_path)
package_name = os.path.basename(custom_nodes_path)
wanvideo_package = importlib.import_module(f"{package_name}.nodes")
WanVideoSampler = wanvideo_package.WanVideoSampler
WanVideoVACEEncode = wanvideo_package.WanVideoVACEEncode
WanVideoDecode = wanvideo_package.WanVideoDecode
print("✅ WanVideo nodes imported successfully!")
# Input validation and preprocessing
batch_size, height, width, channels = video.shape
mask_batch, mask_height, mask_width = mask.shape
print(f"📹 Input video shape: {video.shape} (B×H×W×C)")
print(f"🎭 Input mask shape: {mask.shape} (B×H×W)")
print(f"🎯 Target tile size: {target_frames} frames × {target_width}×{target_height}")
print(f"🔗 Overlaps: {frame_overlap} frames, {spatial_overlap} pixels")
# Validate mask dimensions
if mask_batch != batch_size or mask_height != height or mask_width != width:
print(f"⚠️ Warning: Mask dimensions {mask.shape} don't match video {video.shape}")
print(" Resizing mask to match video...")
mask = torch.nn.functional.interpolate(
mask.unsqueeze(1), size=(height, width), mode='nearest'
).squeeze(1)
print(f"✅ Mask resized to: {mask.shape}")
# STEP 1: Calculate tile dimensions using improved algorithm
print(f"\n📏 STEP 1: Calculating optimal tile layout...")
temporal_tiles = self._calculate_temporal_tiles(batch_size, target_frames, frame_overlap)
spatial_tiles_h = self._calculate_spatial_tiles(height, target_height, spatial_overlap)
spatial_tiles_w = self._calculate_spatial_tiles(width, target_width, spatial_overlap)
total_tiles = len(temporal_tiles) * len(spatial_tiles_h) * len(spatial_tiles_w)
print(f"⏱️ Temporal chunks: {len(temporal_tiles)}")
print(f"🗺️ Spatial tiles per frame: {len(spatial_tiles_h)}×{len(spatial_tiles_w)}")
print(f"📦 Total tiles to process: {total_tiles}")
# STEP 2: Extract all tiles and process through WanVideo VACE pipeline
print(f"\n🧩 STEP 2: Extracting and processing {total_tiles} tiles through WanVideo VACE...")
all_tiles = self._extract_and_process_wanvideo_tiles(
video, mask, temporal_tiles, spatial_tiles_h, spatial_tiles_w,
model, vae, WanVideoVACEEncode, WanVideoSampler, WanVideoDecode,
steps, cfg, shift, seed, scheduler, vace_strength, vace_start_percent, vace_end_percent,
decode_enable_vae_tiling, decode_tile_x, decode_tile_y, decode_tile_stride_x, decode_tile_stride_y,
debug_color_shift, force_offload_between_tiles, debug_mode, frame_overlap, spatial_overlap, kwargs
)
# STEP 3: Dimension-wise stitching - Column-wise (vertical stitching)
print(f"\n🔄 STEP 3: Column-wise stitching (vertical)...")
column_strips = self._stitch_columns(all_tiles, spatial_overlap, debug_mode)
# STEP 4: Dimension-wise stitching - Line-wise (horizontal stitching)
print(f"\n🔄 STEP 4: Line-wise stitching (horizontal)...")
temporal_chunks = self._stitch_lines(column_strips, spatial_overlap, debug_mode)
# STEP 5: Dimension-wise stitching - Temporal stitching
print(f"\n🔄 STEP 5: Temporal stitching...")
stitched_video = self._stitch_temporal_chunks_new(temporal_chunks, temporal_tiles, frame_overlap)
# STEP 6: Crop to original dimensions (ensure exact input size)
print(f"\n✂️ STEP 6: Cropping to original dimensions...")
print(f"📐 Stitched video shape: {stitched_video.shape}")
print(f"🎯 Target shape: {video.shape}")
final_video = stitched_video[:batch_size, :height, :width, :channels]
print(f"✂️ Cropped to: {final_video.shape}")
# Generate comprehensive processing info
processing_summary = self._generate_wanvideo_processing_summary(
all_tiles, temporal_tiles, spatial_tiles_h, spatial_tiles_w, total_tiles
)
print(f"✅ Dimension-wise tiled WanVideo VACE processing completed!")
print(f"📤 Final video shape: {final_video.shape}")
print(f"🧩 Total tiles processed: {len(all_tiles)}")
successful_tiles = sum(1 for tile in all_tiles if hasattr(tile, 'processing_status') and tile.processing_status == 'success')
print(f"✅ Successful tiles: {successful_tiles}/{total_tiles}")
print("="*80 + "\n")
return (final_video, processing_summary)
except Exception as e:
print(f"❌ Error in tiled WanVideo VACE pipeline: {str(e)}")
print(f"📋 Full traceback:")
import traceback
print(traceback.format_exc())
print("="*80 + "\n")
# Return original video in case of error
error_info = f"Error during tiled WanVideo processing: {str(e)}"
return (video, error_info)
def _extract_and_process_wanvideo_tiles(self, video, mask, temporal_tiles, spatial_tiles_h, spatial_tiles_w,
model, vae, WanVideoVACEEncode, WanVideoSampler, WanVideoDecode,
steps, cfg, shift, seed, scheduler, vace_strength, vace_start_percent, vace_end_percent,
decode_enable_vae_tiling, decode_tile_x, decode_tile_y, decode_tile_stride_x, decode_tile_stride_y,
debug_color_shift, force_offload_between_tiles, debug_mode, frame_overlap, spatial_overlap, kwargs):
"""
Extract all tiles and process through WanVideo VACE pipeline with frame-wise temporal consistency
"""
all_tiles = []
# Store previous temporal chunk's COMPLETE STITCHED FRAME for reference
previous_chunk_stitched_frame = None
for temporal_idx, (t_start, t_end) in enumerate(temporal_tiles):
if debug_mode:
print(f" 🎬 Processing temporal chunk {temporal_idx + 1}/{len(temporal_tiles)} (frames {t_start}-{t_end-1})")
# Extract temporal chunk
video_chunk = video[t_start:t_end]
mask_chunk = mask[t_start:t_end]
# Store current chunk tiles for stitching the reference frame
current_chunk_tiles = []
# Process each spatial tile within this temporal chunk
for h_idx, (h_start, h_end) in enumerate(spatial_tiles_h):
for w_idx, (w_start, w_end) in enumerate(spatial_tiles_w):
tile_idx = len(all_tiles) # Global tile index
if debug_mode:
print(f" 🧩 Tile {tile_idx + 1}: T{temporal_idx}:L{h_idx}:C{w_idx} "
f"H[{h_start}:{h_end}] × W[{w_start}:{w_end}]")
# Extract spatial tiles from temporal chunk
video_tile = video_chunk[:, h_start:h_end, w_start:w_end, :]
mask_tile = mask_chunk[:, h_start:h_end, w_start:w_end]
# ========== TEMPORAL OVERWRITING FOR CONSISTENCY ==========
if temporal_idx > 0 and previous_chunk_stitched_frame is not None:
# Determine how many frames to overwrite
frames_to_overwrite = frame_overlap
if temporal_idx == len(temporal_tiles) - 1:
# Last chunk: use more frames if chunk is large enough
available_frames = video_tile.shape[0]
frames_to_overwrite = min(available_frames - 1, max(frame_overlap, available_frames // 3))
# Extract corresponding frames from previous chunk's stitched result
prev_chunk_frames = previous_chunk_stitched_frame.shape[0]
source_start_idx = max(0, prev_chunk_frames - frames_to_overwrite)
source_frames = previous_chunk_stitched_frame[source_start_idx:, h_start:h_end, w_start:w_end, :]
# Overwrite first frames of current tile with last frames from previous chunk
actual_overwrite = min(frames_to_overwrite, source_frames.shape[0], video_tile.shape[0])
if actual_overwrite > 0:
video_tile[:actual_overwrite] = source_frames[:actual_overwrite]
# Zero out corresponding mask areas
mask_tile[:actual_overwrite] = 0.0
if debug_mode:
print(f" 🔄 Temporal overwrite: {actual_overwrite} frames from prev chunk")
# ========== SPATIAL OVERWRITING FOR CONSISTENCY ==========
# Look for already processed tiles to overwrite overlapping regions
# Check for LEFT neighbor (same temporal and line, previous column)
if w_idx > 0:
left_neighbor = None
for tile in current_chunk_tiles:
if (tile.temporal_index == temporal_idx and
tile.line_index == h_idx and
tile.column_index == w_idx - 1):
left_neighbor = tile
break
if left_neighbor is not None and hasattr(left_neighbor, 'content'):
# Calculate overlap region
left_tile_end = left_neighbor.spatial_range_w[1]
current_tile_start = w_start
overlap_width = left_tile_end - current_tile_start
if overlap_width > 0:
# Extract overlap from left neighbor's right edge
left_content = left_neighbor.content
left_overlap = left_content[:, :, -overlap_width:, :] # Last overlap_width columns
# Overwrite left edge of current tile
video_tile[:, :, :overlap_width, :] = left_overlap
# Zero out corresponding mask areas
mask_tile[:, :, :overlap_width] = 0.0
if debug_mode:
print(f" ↔️ Spatial overwrite LEFT: {overlap_width} pixels from neighbor")
# Check for TOP neighbor (same temporal and column, previous line)
if h_idx > 0:
top_neighbor = None
for tile in current_chunk_tiles:
if (tile.temporal_index == temporal_idx and
tile.line_index == h_idx - 1 and
tile.column_index == w_idx):
top_neighbor = tile
break
if top_neighbor is not None and hasattr(top_neighbor, 'content'):
# Calculate overlap region
top_tile_end = top_neighbor.spatial_range_h[1]
current_tile_start = h_start
overlap_height = top_tile_end - current_tile_start
if overlap_height > 0:
# Extract overlap from top neighbor's bottom edge
top_content = top_neighbor.content
top_overlap = top_content[:, -overlap_height:, :, :] # Last overlap_height rows
# Overwrite top edge of current tile
video_tile[:, :overlap_height, :, :] = top_overlap
# Zero out corresponding mask areas
mask_tile[:, :overlap_height, :] = 0.0
if debug_mode:
print(f" ↕️ Spatial overwrite TOP: {overlap_height} pixels from neighbor")
# Handle TOP-LEFT corner conflict (if both top and left neighbors exist)
if h_idx > 0 and w_idx > 0:
# Find top-left diagonal neighbor
topleft_neighbor = None
for tile in current_chunk_tiles:
if (tile.temporal_index == temporal_idx and
tile.line_index == h_idx - 1 and
tile.column_index == w_idx - 1):
topleft_neighbor = tile
break
if topleft_neighbor is not None and hasattr(topleft_neighbor, 'content'):
# Calculate corner overlap
top_tile_end_h = topleft_neighbor.spatial_range_h[1]
left_tile_end_w = topleft_neighbor.spatial_range_w[1]
current_start_h = h_start
current_start_w = w_start
overlap_h = top_tile_end_h - current_start_h
overlap_w = left_tile_end_w - current_start_w
if overlap_h > 0 and overlap_w > 0:
# Extract corner from top-left neighbor
topleft_content = topleft_neighbor.content
corner_overlap = topleft_content[:, -overlap_h:, -overlap_w:, :]
# Overwrite corner of current tile
video_tile[:, :overlap_h, :overlap_w, :] = corner_overlap
# Zero out corresponding mask areas
mask_tile[:, :overlap_h, :overlap_w] = 0.0
if debug_mode:
print(f" ↖️ Spatial overwrite CORNER: {overlap_h}x{overlap_w} from diagonal neighbor")
# FRAME-WISE TEMPORAL CONSISTENCY: Determine reference image for this tile
tile_ref_images = None
if temporal_idx == 0:
# First temporal chunk: Use user-provided ref_images (if any)
tile_ref_images = kwargs.get("vace_ref_images")
if debug_mode and h_idx == 0 and w_idx == 0: # Only print once per chunk
print(f" 🎯 First chunk: Using user-provided ref_images")
else:
# Subsequent chunks: Use COMPLETE STITCHED FRAME from previous chunk as reference
if previous_chunk_stitched_frame is not None:
# Extract reference frame: frame_overlap frames before the last frame
ref_frame_idx = previous_chunk_stitched_frame.shape[0] - frame_overlap - 1
ref_frame_idx = max(0, ref_frame_idx) # Ensure non-negative
# Extract single COMPLETE FRAME as reference image
tile_ref_images = previous_chunk_stitched_frame[ref_frame_idx:ref_frame_idx+1] # Shape: [1, H, W, C]
if debug_mode and h_idx == 0 and w_idx == 0: # Only print once per chunk
print(f" 🔗 Frame-wise temporal chain: Using complete stitched frame {ref_frame_idx} from previous chunk T{temporal_idx-1}")
print(f" Reference frame shape: {tile_ref_images.shape}")
else:
# Fallback: Use user-provided ref_images
tile_ref_images = kwargs.get("vace_ref_images")
if debug_mode and h_idx == 0 and w_idx == 0:
print(f" ⚠️ No previous stitched frame found, using user ref_images as fallback")
# Process tile through WanVideo VACE pipeline with frame-wise temporal reference
try:
# Create kwargs with tile-specific reference image (COMPLETE FRAME)
tile_kwargs = kwargs.copy()
tile_kwargs["vace_ref_images"] = tile_ref_images
processed_tile, tile_latents = self._process_tile_through_wanvideo(
video_tile, mask_tile, model, vae,
WanVideoVACEEncode, WanVideoSampler, WanVideoDecode,
steps, cfg, shift, seed + tile_idx, # Unique seed per tile
scheduler, vace_strength, vace_start_percent, vace_end_percent,
decode_enable_vae_tiling, decode_tile_x, decode_tile_y,
decode_tile_stride_x, decode_tile_stride_y, debug_color_shift, tile_kwargs
)
# Create Tile object with processed content
tile = Tile(
content=processed_tile,
temporal_index=temporal_idx,
line_index=h_idx,
column_index=w_idx,
temporal_range=(t_start, t_end),
spatial_range_h=(h_start, h_end),
spatial_range_w=(w_start, w_end)
)
# Store processing status and reference info for tracking
tile.processing_status = 'success'
tile.seed_used = seed + tile_idx
tile.latents = tile_latents
tile.ref_frame_source = f"complete_frame_T{temporal_idx-1}" if temporal_idx > 0 and previous_chunk_stitched_frame is not None else "user_provided"
all_tiles.append(tile)
current_chunk_tiles.append(tile)
# Force memory cleanup between tiles if requested
if force_offload_between_tiles:
self._force_memory_cleanup(model, vae)
except Exception as tile_error:
print(f" ❌ Error processing tile {tile_idx + 1}: {str(tile_error)}")
# Create fallback Tile object with original content
tile = Tile(
content=video_tile, # Use original video tile as fallback
temporal_index=temporal_idx,
line_index=h_idx,
column_index=w_idx,
temporal_range=(t_start, t_end),
spatial_range_h=(h_start, h_end),
spatial_range_w=(w_start, w_end)
)
# Store error status for tracking
tile.processing_status = f'failed: {str(tile_error)}'
tile.seed_used = seed + tile_idx
tile.latents = None
tile.ref_frame_source = "error_fallback"
all_tiles.append(tile)
current_chunk_tiles.append(tile)
# STITCH THE CURRENT CHUNK TO CREATE REFERENCE FOR NEXT CHUNK
if debug_mode:
print(f" 🔧 Stitching current chunk T{temporal_idx} for next chunk's reference...")
# Group current chunk tiles for stitching
current_chunk_tiles_for_stitching = [tile for tile in current_chunk_tiles]
# Perform dimension-wise stitching for this temporal chunk
try:
# Step 1: Column-wise stitching
column_strips = self._stitch_columns(current_chunk_tiles_for_stitching, spatial_overlap, False) # No debug for reference stitching
# Step 2: Line-wise stitching
temporal_chunks = self._stitch_lines(column_strips, spatial_overlap, False) # No debug for reference stitching
# Get the stitched chunk content (should be only one chunk)
if temporal_chunks and len(temporal_chunks) > 0:
current_chunk_stitched = temporal_chunks[0]['content']
# Store this stitched chunk as reference for next temporal chunk
previous_chunk_stitched_frame = current_chunk_stitched
if debug_mode:
print(f" ✅ Chunk T{temporal_idx} stitched for reference: {current_chunk_stitched.shape}")
else:
print(f" ⚠️ Warning: Failed to stitch chunk T{temporal_idx} for reference")
previous_chunk_stitched_frame = None
except Exception as stitching_error:
print(f" ❌ Error stitching chunk T{temporal_idx} for reference: {str(stitching_error)}")
previous_chunk_stitched_frame = None
if debug_mode:
chain_status = "user_ref" if temporal_idx == 0 else f"complete_frame_from_T{temporal_idx-1}"
print(f" ✅ Temporal chunk {temporal_idx} completed with {chain_status} references")
print(f" ✅ Extracted and processed {len(all_tiles)} tiles through WanVideo VACE pipeline")
successful_tiles = sum(1 for tile in all_tiles if tile.processing_status == 'success')
print(f" 🎯 Success rate: {successful_tiles}/{len(all_tiles)} ({(successful_tiles/len(all_tiles))*100:.1f}%)")
# Debug: Print overwriting statistics
if debug_mode:
temporal_overwrites = sum(1 for t_idx in range(1, len(temporal_tiles)))
spatial_overwrites = 0
corner_overwrites = 0
for temporal_idx in range(len(temporal_tiles)):
for h_idx in range(len(spatial_tiles_h)):
for w_idx in range(len(spatial_tiles_w)):
if w_idx > 0: # Has left neighbor
spatial_overwrites += 1
if h_idx > 0: # Has top neighbor
spatial_overwrites += 1
if h_idx > 0 and w_idx > 0: # Has corner conflict
corner_overwrites += 1
print(f" 🔄 Overwriting consistency enhancements:")
print(f" • Temporal overwrites: {temporal_overwrites * len(spatial_tiles_h) * len(spatial_tiles_w)} tile operations")
print(f" • Spatial overwrites: {spatial_overwrites} edge operations")
print(f" • Corner overwrites: {corner_overwrites} corner conflict resolutions")
# Debug: Print frame-wise temporal consistency chain summary
if debug_mode:
print(f" 🖼️ Frame-wise temporal consistency chain summary:")
print(f" • Chunk 0: Used user-provided reference images")
for t_idx in range(1, len(temporal_tiles)):
print(f" • Chunk {t_idx}: Used complete stitched frame from Chunk {t_idx-1}")
return all_tiles
def _process_tile_through_wanvideo(self, video_tile, mask_tile, model, vae,
WanVideoVACEEncode, WanVideoSampler, WanVideoDecode,
steps, cfg, shift, seed, scheduler, vace_strength,
vace_start_percent, vace_end_percent, decode_enable_vae_tiling,
decode_tile_x, decode_tile_y, decode_tile_stride_x,
decode_tile_stride_y, debug_color_shift, kwargs):
"""Process a single tile through the complete WanVideo VACE pipeline with tile-specific reference"""
tile_frames, tile_height, tile_width = video_tile.shape[:3]
# Get tile-specific reference images (from temporal consistency chain)
tile_ref_images = kwargs.get("vace_ref_images")
# Step 1: Create VACE embeds for this tile with its specific reference
vace_node = WanVideoVACEEncode()
vace_embeds = vace_node.process(
vae=vae,
width=tile_width,
height=tile_height,
num_frames=tile_frames,
strength=vace_strength,
vace_start_percent=vace_start_percent,
vace_end_percent=vace_end_percent,
input_frames=video_tile, # Use tile as input frames
ref_images=tile_ref_images, # Use tile-specific reference (from chain or user)
input_masks=mask_tile, # Use tile mask
tiled_vae=kwargs.get("vace_tiled_vae", False)
)[0]
# Step 2: Run WanVideoSampler on this tile
sampler_node = WanVideoSampler()
latent_samples = sampler_node.process(
model=model,
image_embeds=vace_embeds,
steps=steps,
cfg=cfg,
shift=shift,
seed=seed,
scheduler=scheduler,
riflex_freq_index=kwargs.get("riflex_freq_index", 0),
text_embeds=kwargs.get("text_embeds"),
samples=kwargs.get("samples"),
denoise_strength=kwargs.get("denoise_strength", 1.0),
force_offload=kwargs.get("force_offload", True),
# External arguments
cache_args=kwargs.get("cache_args"),
slg_args=kwargs.get("slg_args"),
experimental_args=kwargs.get("experimental_args"),
# Other arguments
feta_args=kwargs.get("feta_args"),
context_options=kwargs.get("context_options"),
flowedit_args=None,
batched_cfg=kwargs.get("batched_cfg", False),
rope_function=kwargs.get("rope_function", "comfy"),
loop_args=kwargs.get("loop_args"),
sigmas=kwargs.get("sigmas"),
unianimate_poses=kwargs.get("unianimate_poses"),
fantasytalking_embeds=kwargs.get("fantasytalking_embeds"),
uni3c_embeds=kwargs.get("uni3c_embeds"),
multitalk_embeds=kwargs.get("multitalk_embeds"),
freeinit_args=kwargs.get("freeinit_args")
)[0]
# Step 3: Decode latents back to video for this tile
decode_node = WanVideoDecode()
processed_tile = decode_node.decode(
vae=vae,
samples=latent_samples,
enable_vae_tiling=decode_enable_vae_tiling,
tile_x=decode_tile_x,
tile_y=decode_tile_y,
tile_stride_x=decode_tile_stride_x,
tile_stride_y=decode_tile_stride_y,
normalization=kwargs.get("decode_normalization", "default")
)[0]
# Step 4: Apply debug color shift to identify tiles visually (if enabled)
if debug_color_shift:
debug_color_shifted_tile = self._apply_debug_color_shift(processed_tile, seed)
return debug_color_shifted_tile, latent_samples
else:
return processed_tile, latent_samples
def _apply_debug_color_shift(self, tile, seed):
"""Apply a random color shift to each tile for debugging visualization"""
import random
# Generate truly random RGB shifts (different each time, not based on seed)
r_shift = (random.random() - 0.5) * 0.3 # ±0.15 range
g_shift = (random.random() - 0.5) * 0.3 # ±0.15 range
b_shift = (random.random() - 0.5) * 0.3 # ±0.15 range
print(f" 🎨 Debug color shift - Tile: R{r_shift:+.3f}, G{g_shift:+.3f}, B{b_shift:+.3f}")
# Apply color shift to the tile
shifted_tile = tile.clone()
if tile.shape[-1] >= 3: # Ensure we have RGB channels
shifted_tile[:, :, :, 0] = torch.clamp(shifted_tile[:, :, :, 0] + r_shift, 0, 1)
shifted_tile[:, :, :, 1] = torch.clamp(shifted_tile[:, :, :, 1] + g_shift, 0, 1)
shifted_tile[:, :, :, 2] = torch.clamp(shifted_tile[:, :, :, 2] + b_shift, 0, 1)
return shifted_tile
def _stitch_columns(self, all_tiles, spatial_overlap, debug_mode):
"""
Stitch tiles vertically (same column, same temporal chunk) to create column strips
"""
# Group tiles by temporal_index and column_index
column_groups = {}
for tile in all_tiles:
key = (tile.temporal_index, tile.column_index)
if key not in column_groups:
column_groups[key] = []
column_groups[key].append(tile)
# Sort each group by line_index
for key in column_groups:
column_groups[key].sort(key=lambda t: t.line_index)
column_strips = []
for (temporal_idx, col_idx), column_tiles in column_groups.items():
if debug_mode and temporal_idx == 0:
print(f" 🔄 Column {col_idx}: {len(column_tiles)} tiles")
# Stitch tiles in this column vertically
if len(column_tiles) == 1:
column_strip = column_tiles[0].content
else:
column_strip = self._stitch_tiles_vertically(column_tiles, spatial_overlap)
# Store column strip with its position info
strip_info = {
'content': column_strip,
'temporal_index': temporal_idx,
'column_index': col_idx,
'spatial_range_w': column_tiles[0].spatial_range_w,
'spatial_range_h': (column_tiles[0].spatial_range_h[0], column_tiles[-1].spatial_range_h[1])
}
column_strips.append(strip_info)
print(f" ✅ Created {len(column_strips)} column strips")
return column_strips
def _stitch_lines(self, column_strips, spatial_overlap, debug_mode):
"""
Stitch column strips horizontally (same temporal chunk) to create complete temporal chunks
"""
# Group column strips by temporal_index
temporal_groups = {}
for strip in column_strips:
temporal_idx = strip['temporal_index']
if temporal_idx not in temporal_groups:
temporal_groups[temporal_idx] = []
temporal_groups[temporal_idx].append(strip)
# Sort each group by column_index
for temporal_idx in temporal_groups:
temporal_groups[temporal_idx].sort(key=lambda s: s['column_index'])
temporal_chunks = []
for temporal_idx, strips in temporal_groups.items():
if debug_mode and temporal_idx == 0:
print(f" 🔄 Temporal chunk {temporal_idx}: {len(strips)} strips")
# Stitch column strips horizontally
if len(strips) == 1:
chunk_content = strips[0]['content']
else:
chunk_content = self._stitch_strips_horizontally(strips, spatial_overlap)
# Store temporal chunk with its info
chunk_info = {
'content': chunk_content,
'temporal_index': temporal_idx,
'spatial_range_h': strips[0]['spatial_range_h'],
'spatial_range_w': (strips[0]['spatial_range_w'][0], strips[-1]['spatial_range_w'][1])
}
temporal_chunks.append(chunk_info)
# Sort by temporal index
temporal_chunks.sort(key=lambda c: c['temporal_index'])
print(f" ✅ Created {len(temporal_chunks)} temporal chunks")
return temporal_chunks
def _stitch_tiles_vertically(self, column_tiles, spatial_overlap):
"""
Stitch tiles vertically (in height dimension) with proper fade blending.
Always respects spatial_overlap limit - starts blending from the correct position
within the overlapping region when actual overlap exceeds spatial_overlap.
"""
if len(column_tiles) == 1:
return column_tiles[0].content
# Get dimensions
first_tile = column_tiles[0].content
total_height = sum(tile.spatial_range_h[1] - tile.spatial_range_h[0] for tile in column_tiles)
# Subtract overlaps
total_height -= spatial_overlap * (len(column_tiles) - 1)
width = first_tile.shape[2]
channels = first_tile.shape[3]
frames = first_tile.shape[0]
# Create result tensor
result = torch.zeros((frames, total_height, width, channels),
dtype=first_tile.dtype, device=first_tile.device)
current_h = 0
for i, tile in enumerate(column_tiles):
tile_h = tile.content.shape[1]
if i == 0:
# First tile - place entirely
result[:, current_h:current_h+tile_h, :, :] = tile.content
current_h += tile_h
else:
# Subsequent tiles - ALWAYS limit overlap to spatial_overlap maximum
# Any extra pixels are ignored (they were just for generation context)
# Calculate the effective overlap, but limit it to spatial_overlap
expected_start = tile.spatial_range_h[0]
actual_overlap = current_h - expected_start
actual_overlap = max(0, actual_overlap) # Ensure non-negative
# STRICT LIMIT: Never exceed spatial_overlap
overlap_h = min(spatial_overlap, actual_overlap, tile_h // 2, current_h)
if actual_overlap > spatial_overlap:
print(f"🎯 Tile {i}: Large overlap {actual_overlap} detected, limiting to {spatial_overlap} (extra pixels ignored)")
if overlap_h > 0:
# Calculate where to start within the tile based on actual overlap
tile_start_offset = actual_overlap - overlap_h # Skip extra pixels at the beginning
# Get regions for blending (limited to spatial_overlap size)
result_overlap = result[:, current_h-overlap_h:current_h, :, :]
tile_overlap = tile.content[:, tile_start_offset:tile_start_offset+overlap_h, :, :]
# Create vertical fade mask
fade_mask = self._create_vertical_fade_mask(overlap_h, result_overlap.dtype, result_overlap.device)
# Apply fade blending only within the limited overlap region
blended_overlap = result_overlap * (1.0 - fade_mask) + tile_overlap * fade_mask
result[:, current_h-overlap_h:current_h, :, :] = blended_overlap
# Place the non-overlapping part (skip the overlap region + offset)
remaining_start = tile_start_offset + overlap_h
if tile_h > remaining_start:
result[:, current_h:current_h+tile_h-remaining_start, :, :] = tile.content[:, remaining_start:, :, :]
current_h += tile_h - remaining_start
else:
# No overlap - place directly
result[:, current_h:current_h+tile_h, :, :] = tile.content
current_h += tile_h
return result
def _stitch_strips_horizontally(self, strips, spatial_overlap):
"""
Stitch column strips horizontally (in width dimension) with proper fade blending.
Always respects spatial_overlap limit - starts blending from the correct position
within the overlapping region when actual overlap exceeds spatial_overlap.
"""
if len(strips) == 1:
return strips[0]['content']
# Get dimensions
first_strip = strips[0]['content']
total_width = sum(s['spatial_range_w'][1] - s['spatial_range_w'][0] for s in strips)
# Subtract overlaps
total_width -= spatial_overlap * (len(strips) - 1)
height = first_strip.shape[1]
channels = first_strip.shape[3]
frames = first_strip.shape[0]
# Create result tensor
result = torch.zeros((frames, height, total_width, channels),
dtype=first_strip.dtype, device=first_strip.device)
current_w = 0
for i, strip in enumerate(strips):
strip_content = strip['content']
strip_w = strip_content.shape[2]
if i == 0:
# First strip - place entirely
result[:, :, current_w:current_w+strip_w, :] = strip_content
current_w += strip_w
else:
# Subsequent strips - ALWAYS limit overlap to spatial_overlap maximum
# Any extra pixels are ignored (they were just for generation context)
# Calculate the effective overlap, but limit it to spatial_overlap
expected_start = strip['spatial_range_w'][0]
actual_overlap = current_w - expected_start
actual_overlap = max(0, actual_overlap) # Ensure non-negative
# STRICT LIMIT: Never exceed spatial_overlap
overlap_w = min(spatial_overlap, actual_overlap, strip_w // 2, current_w)
if actual_overlap > spatial_overlap:
print(f"🎯 Strip {i}: Large overlap {actual_overlap} detected, limiting to {spatial_overlap} (extra pixels ignored)")
if overlap_w > 0:
# Calculate where to start within the strip based on actual overlap
strip_start_offset = actual_overlap - overlap_w # Skip extra pixels at the beginning
# Get regions for blending (limited to spatial_overlap size)
result_overlap = result[:, :, current_w-overlap_w:current_w, :]
strip_overlap = strip_content[:, :, strip_start_offset:strip_start_offset+overlap_w, :]
# Create horizontal fade mask
fade_mask = self._create_horizontal_fade_mask(overlap_w, result_overlap.dtype, result_overlap.device)
# Apply fade blending only within the limited overlap region
blended_overlap = result_overlap * (1.0 - fade_mask) + strip_overlap * fade_mask
result[:, :, current_w-overlap_w:current_w, :] = blended_overlap
# Place the non-overlapping part (skip the overlap region + offset)
remaining_start = strip_start_offset + overlap_w
if strip_w > remaining_start:
result[:, :, current_w:current_w+strip_w-remaining_start, :] = strip_content[:, :, remaining_start:, :]
current_w += strip_w - remaining_start
else:
# No overlap - place directly
result[:, :, current_w:current_w+strip_w, :] = strip_content
current_w += strip_w
return result
def _create_vertical_fade_mask(self, fade_size, dtype, device):
"""Create a vertical fade mask for blending tiles vertically."""
fade_mask = torch.linspace(0, 1, fade_size, dtype=dtype, device=device)
# Shape: (1, fade_size, 1, 1) for broadcasting
return fade_mask.view(1, fade_size, 1, 1)
def _create_horizontal_fade_mask(self, fade_size, dtype, device):
"""Create a horizontal fade mask for blending tiles horizontally."""
fade_mask = torch.linspace(0, 1, fade_size, dtype=dtype, device=device)
# Shape: (1, 1, fade_size, 1) for broadcasting
return fade_mask.view(1, 1, fade_size, 1)
def _stitch_temporal_chunks_new(self, temporal_chunks, temporal_tiles, frame_overlap):
"""
Stitch temporal chunks to create the final output with temporal blending.
Always respects frame_overlap limit - starts blending from the correct position
within the overlapping region when actual overlap exceeds frame_overlap.
"""
if not temporal_chunks:
return None
if len(temporal_chunks) == 1:
return temporal_chunks[0]['content']
# Get dimensions from first chunk
first_chunk_content = temporal_chunks[0]['content']
# Calculate total frames accounting for overlaps
total_frames = sum(chunk['content'].shape[0] for chunk in temporal_chunks)
if frame_overlap > 0:
total_frames -= (len(temporal_chunks) - 1) * frame_overlap
# Get spatial dimensions from the stitched chunks (not original input)
height = first_chunk_content.shape[1]
width = first_chunk_content.shape[2]
channels = first_chunk_content.shape[3]
print(f" 📏 Temporal stitching dimensions:")
print(f" • Total frames: {total_frames}")
print(f" • Spatial size: {height}×{width}")
print(f" • Channels: {channels}")
print(f" • Chunk sizes: {[chunk['content'].shape for chunk in temporal_chunks]}")
# Initialize result tensor with correct dimensions
result = torch.zeros(
(total_frames, height, width, channels),
dtype=first_chunk_content.dtype,
device=first_chunk_content.device
)
current_t = 0
for i, chunk_info in enumerate(temporal_chunks):
chunk_content = chunk_info['content']
chunk_frames = chunk_content.shape[0]
if i == 0:
# First chunk - place entirely
result[current_t:current_t+chunk_frames] = chunk_content
current_t += chunk_frames
else:
# Subsequent chunks - ALWAYS limit overlap to frame_overlap maximum
# Any extra frames are ignored (they were just for generation context)
# Calculate the effective overlap, but limit it to frame_overlap
temporal_tile_info = temporal_tiles[i]
expected_start = temporal_tile_info[0]
actual_overlap = current_t - expected_start
actual_overlap = max(0, actual_overlap) # Ensure non-negative
# STRICT LIMIT: Never exceed frame_overlap
overlap_frames = min(frame_overlap, actual_overlap, chunk_frames // 2, current_t)
if actual_overlap > frame_overlap:
print(f"🎯 Chunk {i}: Large overlap {actual_overlap} detected, limiting to {frame_overlap} (extra frames ignored)")
if overlap_frames > 0:
# Calculate where to start within the chunk based on actual overlap
chunk_start_offset = actual_overlap - overlap_frames # Skip extra frames at the beginning
# Get regions for blending (limited to frame_overlap size)
result_overlap = result[current_t-overlap_frames:current_t]
chunk_overlap = chunk_content[chunk_start_offset:chunk_start_offset+overlap_frames]
# Create temporal fade mask
fade_mask = self._create_temporal_fade_mask(overlap_frames, result_overlap.dtype, result_overlap.device)
# Apply fade blending only within the limited overlap region
blended_overlap = result_overlap * (1.0 - fade_mask) + chunk_overlap * fade_mask
result[current_t-overlap_frames:current_t] = blended_overlap
# Place the non-overlapping part (skip the overlap region + offset)
remaining_start = chunk_start_offset + overlap_frames
if chunk_frames > remaining_start:
result[current_t:current_t+chunk_frames-remaining_start] = chunk_content[remaining_start:]
current_t += chunk_frames - remaining_start
else:
# No overlap - place directly
result[current_t:current_t+chunk_frames] = chunk_content
current_t += chunk_frames
return result
def _create_temporal_fade_mask(self, fade_size, dtype, device):
"""Create a temporal fade mask for blending across time dimension."""
fade_mask = torch.linspace(0, 1, fade_size, dtype=dtype, device=device)
# Shape: (fade_size, 1, 1, 1) for broadcasting
return fade_mask.view(fade_size, 1, 1, 1)
def _force_memory_cleanup(self, model, vae):
"""Force memory cleanup between tile processing"""
try:
import gc
# Force garbage collection
gc.collect()
# Clear CUDA cache if available
if torch.cuda.is_available():
torch.cuda.empty_cache()
torch.cuda.synchronize()
# Try to offload models if they have offload methods
if hasattr(model, 'cpu_offload'):
model.cpu_offload()
elif hasattr(model, 'to'):
model.to('cpu')
if hasattr(vae, 'cpu_offload'):
vae.cpu_offload()
elif hasattr(vae, 'to'):
vae.to('cpu')
except Exception as cleanup_error:
print(f" ⚠️ Warning: Memory cleanup failed: {str(cleanup_error)}")
def _calculate_temporal_tiles(self, total_frames, target_frames, overlap):
"""Calculate temporal tile ranges with overlap handling"""
tiles = []
if total_frames <= target_frames:
tiles.append((0, total_frames))
else:
stride = target_frames - overlap
current = 0
while current < total_frames:
end = min(current + target_frames, total_frames)
remaining = total_frames - end
if remaining > 0:
tiles.append((current, end))
if remaining < stride:
final_start = total_frames - target_frames
tiles.append((final_start, total_frames))
break
else:
tiles.append((current, end))
break
current += stride
return tiles
def _calculate_spatial_tiles(self, total_size, target_size, overlap):
"""Calculate spatial tile ranges with overlap handling"""
tiles = []
if total_size <= target_size:
tiles.append((0, total_size))
else:
stride = target_size - overlap
current = 0
while current < total_size:
end = min(current + target_size, total_size)
remaining = total_size - end
if remaining > 0:
tiles.append((current, end))
if remaining < stride:
final_start = total_size - target_size
tiles.append((final_start, total_size))
break
else:
tiles.append((current, end))
break
current += stride
return tiles
def _place_tile_with_overlap(self, target, tile, h_start, h_end, w_start, w_end, spatial_overlap):
"""Place tile into target tensor, handling overlaps with spatial fade blending"""
tile_h, tile_w = tile.shape[1:3]
# Get the target region
target_region = target[:, h_start:h_start+tile_h, w_start:w_start+tile_w, :]
# Check if there's existing content (non-zero) in the target area
if target_region.sum() > 0:
# Use production-quality fade blending
fade_mask = self._create_spatial_fade_mask(tile_h, tile_w, target_region, tile, spatial_overlap)
# Apply fade blending: existing * (1 - fade_mask) + tile * fade_mask
blended = target_region * (1.0 - fade_mask) + tile * fade_mask
target[:, h_start:h_start+tile_h, w_start:w_start+tile_w, :] = blended
else:
# First tile in this area - place directly
target[:, h_start:h_start+tile_h, w_start:w_start+tile_w, :] = tile
def _create_spatial_fade_mask(self, tile_h, tile_w, existing, new_tile, spatial_overlap):
"""Create a spatial fade mask for smooth blending between tiles"""
# Create base mask (1.0 = use new tile, 0.0 = use existing)
mask = torch.ones(1, tile_h, tile_w, 1, dtype=existing.dtype, device=existing.device)
# Define fade distance using spatial_overlap parameter
fade_h = min(spatial_overlap, tile_h // 2) # Fade over spatial_overlap pixels or half tile height
fade_w = min(spatial_overlap, tile_w // 2) # Fade over spatial_overlap pixels or half tile width
# Check which edges have existing content
has_top = existing[:, :fade_h, :, :].sum() > 0
has_bottom = existing[:, -fade_h:, :, :].sum() > 0
has_left = existing[:, :, :fade_w, :].sum() > 0
has_right = existing[:, :, -fade_w:, :].sum() > 0
# Create fade gradients for each edge
if has_top:
for i in range(fade_h):
alpha = i / fade_h
mask[:, i, :, :] = alpha
if has_bottom:
for i in range(fade_h):
alpha = 1.0 - (i / fade_h)
mask[:, tile_h - 1 - i, :, :] = alpha
if has_left:
for i in range(fade_w):
alpha = i / fade_w
mask[:, :, i, :] = torch.minimum(mask[:, :, i, :], torch.tensor(alpha, dtype=mask.dtype, device=mask.device))
if has_right:
for i in range(fade_w):
alpha = 1.0 - (i / fade_w)
mask[:, :, tile_w - 1 - i, :] = torch.minimum(mask[:, :, tile_w - 1 - i, :], torch.tensor(alpha, dtype=mask.dtype, device=mask.device))
return mask.expand_as(new_tile)
def _stitch_temporal_chunks(self, chunks, temporal_tiles, total_frames, height, width, channels, overlap):
"""Stitch temporal chunks back together with temporal fade blending"""
result = torch.zeros((total_frames, height, width, channels), dtype=chunks[0].dtype, device=chunks[0].device)
for i, ((t_start, t_end), chunk) in enumerate(zip(temporal_tiles, chunks)):
chunk_frames = chunk.shape[0]
if i == 0:
result[t_start:t_start+chunk_frames] = chunk
else:
prev_end = temporal_tiles[i-1][1]
overlap_start = max(t_start, prev_end - overlap)
overlap_end = min(t_end, prev_end)
if overlap_start < overlap_end:
overlap_frames = overlap_end - overlap_start
chunk_overlap_start = overlap_start - t_start
existing_frames = result[overlap_start:overlap_end]
new_frames = chunk[chunk_overlap_start:chunk_overlap_start+overlap_frames]
fade_mask = self._create_temporal_fade_mask(overlap_frames, existing_frames.dtype, existing_frames.device)
blended_frames = existing_frames * (1.0 - fade_mask) + new_frames * fade_mask
result[overlap_start:overlap_end] = blended_frames
if overlap_end < t_start + chunk_frames:
non_overlap_start = overlap_end
chunk_offset = non_overlap_start - t_start
result[non_overlap_start:t_start+chunk_frames] = chunk[chunk_offset:]
else:
result[t_start:t_start+chunk_frames] = chunk
return result
def _create_temporal_fade_mask(self, overlap_frames, dtype, device):
"""Create a temporal fade mask for smooth frame transitions"""
fade_values = torch.linspace(0.0, 1.0, overlap_frames, dtype=dtype, device=device)
fade_mask = fade_values.view(overlap_frames, 1, 1, 1)
return fade_mask
def _generate_wanvideo_processing_summary(self, all_tiles, temporal_tiles, spatial_tiles_h, spatial_tiles_w, total_tiles):
"""Generate comprehensive summary of the WanVideo tiled processing"""
successful_tiles = sum(1 for tile in all_tiles if hasattr(tile, 'processing_status') and tile.processing_status == 'success')
failed_tiles = total_tiles - successful_tiles
summary = f"=== DIMENSION-WISE TILED WANVIDEO VACE PROCESSING SUMMARY ===\n"
summary += f"Total tiles processed: {total_tiles}\n"
summary += f"Successful WanVideo tiles: {successful_tiles}\n"
summary += f"Failed tiles: {failed_tiles}\n"
summary += f"WanVideo success rate: {(successful_tiles/total_tiles)*100:.1f}%\n\n"
# Temporal consistency chain information
summary += f"TEMPORAL CONSISTENCY CHAIN:\n"
summary += f"Temporal chunks: {len(temporal_tiles)}\n"
for i, (start, end) in enumerate(temporal_tiles):
if i == 0:
ref_source = "User-provided reference images"
else:
ref_source = f"Frame references from Chunk {i-1} (frame_overlap-based)"
summary += f" Chunk {i+1}: frames {start}-{end-1} ({end-start} frames) - {ref_source}\n"
summary += f"\nSpatial tiles per frame: {len(spatial_tiles_h)}×{len(spatial_tiles_w)}\n"
summary += f"Height tiles: {len(spatial_tiles_h)}\n"
for i, (start, end) in enumerate(spatial_tiles_h):
summary += f" Row {i+1}: pixels {start}-{end-1} ({end-start} pixels)\n"
summary += f"\nWidth tiles: {len(spatial_tiles_w)}\n"
for i, (start, end) in enumerate(spatial_tiles_w):
summary += f" Col {i+1}: pixels {start}-{end-1} ({end-start} pixels)\n"
# Analyze temporal consistency chain usage
chain_usage = {}
for tile in all_tiles:
if hasattr(tile, 'ref_frame_source'):
source = tile.ref_frame_source
if source not in chain_usage:
chain_usage[source] = 0
chain_usage[source] += 1
summary += f"\nTemporal reference chain usage:\n"
for source, count in chain_usage.items():
summary += f" {source}: {count} tiles\n"
# Group tiles by processing status
if failed_tiles > 0:
summary += f"\nFailed tiles details:\n"
for tile in all_tiles:
if hasattr(tile, 'processing_status') and tile.processing_status != 'success':
summary += f" T{tile.temporal_index}:L{tile.line_index}:C{tile.column_index}: {tile.processing_status}\n"
# Add dimension-wise processing information
summary += f"\nDimension-wise processing algorithm with temporal consistency:\n"
summary += f" 1. Temporal consistency chain: Each chunk uses references from previous chunk\n"
summary += f" 2. WanVideo VACE processing: Each tile processed with spatial-specific references\n"
summary += f" 3. Column-wise stitching: Vertical fade blending within temporal chunks\n"
summary += f" 4. Line-wise stitching: Horizontal fade blending across columns\n"
summary += f" 5. Temporal stitching: Frame-based fade blending across time\n"
summary += f" 6. Cropping: Final video cropped to original input dimensions\n"
summary += f"\nTemporal consistency ensures each tile position maintains coherent evolution across chunks\n"
summary += f"Reference frame extracted from frame_overlap position for smooth temporal transitions\n"
summary += f"Large video processed through {total_tiles} individual WanVideo VACE operations with temporal continuity\n"
return summary
def rescale_i(samples, width, height, algorithm: str):
samples = samples.movedim(-1, 1)
algorithm = getattr(Image, algorithm.upper()) # i.e. Image.BICUBIC
samples_pil: Image.Image = F.to_pil_image(samples[0].cpu()).resize((width, height), algorithm)
samples = F.to_tensor(samples_pil).unsqueeze(0)
samples = samples.movedim(1, -1)
return samples
def rescale_m(samples, width, height, algorithm: str):
samples = samples.unsqueeze(1)
algorithm = getattr(Image, algorithm.upper()) # i.e. Image.BICUBIC
samples_pil: Image.Image = F.to_pil_image(samples[0].cpu()).resize((width, height), algorithm)
samples = F.to_tensor(samples_pil).unsqueeze(0)
samples = samples.squeeze(1)
return samples
def preresize_imm(image, mask, optional_context_mask, downscale_algorithm, upscale_algorithm, preresize_mode, preresize_min_width, preresize_min_height, preresize_max_width, preresize_max_height):
current_width, current_height = image.shape[2], image.shape[1] # Image size [batch, height, width, channels]
if preresize_mode == "ensure minimum resolution":
if current_width >= preresize_min_width and current_height >= preresize_min_height:
return image, mask, optional_context_mask
scale_factor_min_width = preresize_min_width / current_width
scale_factor_min_height = preresize_min_height / current_height
scale_factor = max(scale_factor_min_width, scale_factor_min_height)
target_width = int(current_width * scale_factor)
target_height = int(current_height * scale_factor)
image = rescale_i(image, target_width, target_height, upscale_algorithm)
mask = rescale_m(mask, target_width, target_height, 'bilinear')
optional_context_mask = rescale_m(optional_context_mask, target_width, target_height, 'bilinear')
assert target_width >= preresize_min_width and target_height >= preresize_min_height, \
f"Internal error: After resizing, target size {target_width}x{target_height} is smaller than min size {preresize_min_width}x{preresize_min_height}"
elif preresize_mode == "ensure minimum and maximum resolution":
if preresize_min_width <= current_width <= preresize_max_width and preresize_min_height <= current_height <= preresize_max_height:
return image, mask, optional_context_mask
scale_factor_min_width = preresize_min_width / current_width
scale_factor_min_height = preresize_min_height / current_height
scale_factor_min = max(scale_factor_min_width, scale_factor_min_height)
scale_factor_max_width = preresize_max_width / current_width
scale_factor_max_height = preresize_max_height / current_height
scale_factor_max = min(scale_factor_max_width, scale_factor_max_height)
if scale_factor_min > 1 and scale_factor_max < 1:
assert False, "Cannot meet both minimum and maximum resolution requirements with aspect ratio preservation."
if scale_factor_min > 1: # We're upscaling to meet min resolution
scale_factor = scale_factor_min
rescale_algorithm = upscale_algorithm # Use upscale algorithm for min resolution
else: # We're downscaling to meet max resolution
scale_factor = scale_factor_max
rescale_algorithm = downscale_algorithm # Use downscale algorithm for max resolution
target_width = int(current_width * scale_factor)
target_height = int(current_height * scale_factor)
image = rescale_i(image, target_width, target_height, rescale_algorithm)
mask = rescale_m(mask, target_width, target_height, 'nearest') # Always nearest for efficiency
optional_context_mask = rescale_m(optional_context_mask, target_width, target_height, 'nearest') # Always nearest for efficiency
assert preresize_min_width <= target_width <= preresize_max_width, \
f"Internal error: Target width {target_width} is outside the range {preresize_min_width} - {preresize_max_width}"
assert preresize_min_height <= target_height <= preresize_max_height, \
f"Internal error: Target height {target_height} is outside the range {preresize_min_height} - {preresize_max_height}"
elif preresize_mode == "ensure maximum resolution":
if current_width <= preresize_max_width and current_height <= preresize_max_height:
return image, mask, optional_context_mask
scale_factor_max_width = preresize_max_width / current_width
scale_factor_max_height = preresize_max_height / current_height
scale_factor_max = min(scale_factor_max_width, scale_factor_max_height)
target_width = int(current_width * scale_factor_max)
target_height = int(current_height * scale_factor_max)
image = rescale_i(image, target_width, target_height, downscale_algorithm)
mask = rescale_m(mask, target_width, target_height, 'nearest') # Always nearest for efficiency
optional_context_mask = rescale_m(optional_context_mask, target_width, target_height, 'nearest') # Always nearest for efficiency
assert target_width <= preresize_max_width and target_height <= preresize_max_height, \
f"Internal error: Target size {target_width}x{target_height} is greater than max size {preresize_max_width}x{preresize_max_height}"
return image, mask, optional_context_mask
def fillholes_iterative_hipass_fill_m(samples):
thresholds = [1, 0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1]
mask_np = samples.squeeze(0).cpu().numpy()
for threshold in thresholds:
thresholded_mask = mask_np >= threshold
closed_mask = binary_closing(thresholded_mask, structure=np.ones((3, 3)), border_value=1)
filled_mask = binary_fill_holes(closed_mask)
mask_np = np.maximum(mask_np, np.where(filled_mask != 0, threshold, 0))
final_mask = torch.from_numpy(mask_np.astype(np.float32)).unsqueeze(0)
return final_mask
def hipassfilter_m(samples, threshold):
filtered_mask = samples.clone()
filtered_mask[filtered_mask < threshold] = 0
return filtered_mask
def expand_m(mask, pixels):
sigma = pixels / 4
mask_np = mask.squeeze(0).cpu().numpy()
kernel_size = math.ceil(sigma * 1.5 + 1)
kernel = np.ones((kernel_size, kernel_size), dtype=np.uint8)
dilated_mask = grey_dilation(mask_np, footprint=kernel)
dilated_mask = dilated_mask.astype(np.float32)
dilated_mask = torch.from_numpy(dilated_mask)
dilated_mask = torch.clamp(dilated_mask, 0.0, 1.0)
return dilated_mask.unsqueeze(0)
def invert_m(samples):
inverted_mask = samples.clone()
inverted_mask = 1.0 - inverted_mask
return inverted_mask
def blur_m(samples, pixels):
mask = samples.squeeze(0)
sigma = pixels / 4
mask_np = mask.cpu().numpy()
blurred_mask = gaussian_filter(mask_np, sigma=sigma)
blurred_mask = torch.from_numpy(blurred_mask).float()
blurred_mask = torch.clamp(blurred_mask, 0.0, 1.0)
return blurred_mask.unsqueeze(0)
def extend_imm(image, mask, optional_context_mask, extend_up_factor, extend_down_factor, extend_left_factor, extend_right_factor):
B, H, W, C = image.shape
new_H = int(H * (1.0 + extend_up_factor - 1.0 + extend_down_factor - 1.0))
new_W = int(W * (1.0 + extend_left_factor - 1.0 + extend_right_factor - 1.0))
assert new_H >= 0, f"Error: Trying to crop too much, height ({new_H}) must be >= 0"
assert new_W >= 0, f"Error: Trying to crop too much, width ({new_W}) must be >= 0"
expanded_image = torch.zeros(1, new_H, new_W, C, device=image.device)
expanded_mask = torch.ones(1, new_H, new_W, device=mask.device)
expanded_optional_context_mask = torch.zeros(1, new_H, new_W, device=optional_context_mask.device)
up_padding = int(H * (extend_up_factor - 1.0))
down_padding = new_H - H - up_padding
left_padding = int(W * (extend_left_factor - 1.0))
right_padding = new_W - W - left_padding
slice_target_up = max(0, up_padding)
slice_target_down = min(new_H, up_padding + H)
slice_target_left = max(0, left_padding)
slice_target_right = min(new_W, left_padding + W)
slice_source_up = max(0, -up_padding)
slice_source_down = min(H, new_H - up_padding)
slice_source_left = max(0, -left_padding)
slice_source_right = min(W, new_W - left_padding)
image = image.permute(0, 3, 1, 2) # [B, H, W, C] -> [B, C, H, W]
expanded_image = expanded_image.permute(0, 3, 1, 2) # [B, H, W, C] -> [B, C, H, W]
expanded_image[:, :, slice_target_up:slice_target_down, slice_target_left:slice_target_right] = image[:, :, slice_source_up:slice_source_down, slice_source_left:slice_source_right]
if up_padding > 0:
expanded_image[:, :, :up_padding, slice_target_left:slice_target_right] = image[:, :, 0:1, slice_source_left:slice_source_right].repeat(1, 1, up_padding, 1)
if down_padding > 0:
expanded_image[:, :, -down_padding:, slice_target_left:slice_target_right] = image[:, :, -1:, slice_source_left:slice_source_right].repeat(1, 1, down_padding, 1)
if left_padding > 0:
expanded_image[:, :, :, :left_padding] = expanded_image[:, :, :, left_padding:left_padding+1].repeat(1, 1, 1, left_padding)
if right_padding > 0:
expanded_image[:, :, :, -right_padding:] = expanded_image[:, :, :, -right_padding-1:-right_padding].repeat(1, 1, 1, right_padding)
expanded_mask[:, slice_target_up:slice_target_down, slice_target_left:slice_target_right] = mask[:, slice_source_up:slice_source_down, slice_source_left:slice_source_right]
expanded_optional_context_mask[:, slice_target_up:slice_target_down, slice_target_left:slice_target_right] = optional_context_mask[:, slice_source_up:slice_source_down, slice_source_left:slice_source_right]
expanded_image = expanded_image.permute(0, 2, 3, 1) # [B, C, H, W] -> [B, H, W, C]
image = image.permute(0, 2, 3, 1) # [B, C, H, W] -> [B, H, W, C]
return expanded_image, expanded_mask, expanded_optional_context_mask
def debug_context_location_in_image(image, x, y, w, h):
debug_image = image.clone()
debug_image[:, y:y+h, x:x+w, :] = 1.0 - debug_image[:, y:y+h, x:x+w, :]
return debug_image
def findcontextarea_m(mask):
mask_squeezed = mask[0] # Now shape is [H, W]
non_zero_indices = torch.nonzero(mask_squeezed)
H, W = mask_squeezed.shape
if non_zero_indices.numel() == 0:
x, y = -1, -1
w, h = -1, -1
else:
y = torch.min(non_zero_indices[:, 0]).item()
x = torch.min(non_zero_indices[:, 1]).item()
y_max = torch.max(non_zero_indices[:, 0]).item()
x_max = torch.max(non_zero_indices[:, 1]).item()
w = x_max - x + 1 # +1 to include the max index
h = y_max - y + 1 # +1 to include the max index
context = mask[:, y:y+h, x:x+w]
return context, x, y, w, h
def growcontextarea_m(context, mask, x, y, w, h, extend_factor):
img_h, img_w = mask.shape[1], mask.shape[2]
# Compute intended growth in each direction
grow_left = int(round(w * (extend_factor-1.0) / 2.0))
grow_right = int(round(w * (extend_factor-1.0) / 2.0))
grow_up = int(round(h * (extend_factor-1.0) / 2.0))
grow_down = int(round(h * (extend_factor-1.0) / 2.0))
# Try to grow left, but clamp at 0
new_x = x - grow_left
if new_x < 0:
new_x = 0
# Try to grow up, but clamp at 0
new_y = y - grow_up
if new_y < 0:
new_y = 0
# Right edge
new_x2 = x + w + grow_right
if new_x2 > img_w:
new_x2 = img_w
# Bottom edge
new_y2 = y + h + grow_down
if new_y2 > img_h:
new_y2 = img_h
# New width and height
new_w = new_x2 - new_x
new_h = new_y2 - new_y
# Extract the context
new_context = mask[:, new_y:new_y+new_h, new_x:new_x+new_w]
if new_h < 0 or new_w < 0:
new_x = 0
new_y = 0
new_w = mask.shape[2]
new_h = mask.shape[1]
return new_context, new_x, new_y, new_w, new_h
def combinecontextmask_m(context, mask, x, y, w, h, optional_context_mask):
_, x_opt, y_opt, w_opt, h_opt = findcontextarea_m(optional_context_mask)
if x == -1:
x, y, w, h = x_opt, y_opt, w_opt, h_opt
if x_opt == -1:
x_opt, y_opt, w_opt, h_opt = x, y, w, h
if x == -1:
return torch.zeros(1, 0, 0, device=mask.device), -1, -1, -1, -1
new_x = min(x, x_opt)
new_y = min(y, y_opt)
new_x_max = max(x + w, x_opt + w_opt)
new_y_max = max(y + h, y_opt + h_opt)
new_w = new_x_max - new_x
new_h = new_y_max - new_y
combined_context = mask[:, new_y:new_y+new_h, new_x:new_x+new_w]
return combined_context, new_x, new_y, new_w, new_h
def pad_to_multiple(value, multiple):
return int(math.ceil(value / multiple) * multiple)
def crop_magic_im(image, mask, x, y, w, h, target_w, target_h, padding, downscale_algorithm, upscale_algorithm):
image = image.clone()
mask = mask.clone()
# Ok this is the most complex function in this node. The one that does the magic after all the preparation done by the other nodes.
# Basically this function determines the right context area that encompasses the whole context area (mask+optional_context_mask),
# that is ideally within the bounds of the original image, and that has the right aspect ratio to match target width and height.
# It may grow the image if the aspect ratio wouldn't fit in the original image.
# It keeps track of that growing to then be able to crop the image in the stitch node.
# Finally, it crops the context area and resizes it to be exactly target_w and target_h.
# It keeps track of that resize to be able to revert it in the stitch node.
# Check for invalid inputs
if target_w <= 0 or target_h <= 0 or w == 0 or h == 0:
return image, 0, 0, image.shape[2], image.shape[1], image, mask, 0, 0, image.shape[2], image.shape[1]
# Step 1: Pad target dimensions to be multiples of padding
if padding != 0:
target_w = pad_to_multiple(target_w, padding)
target_h = pad_to_multiple(target_h, padding)
# Step 2: Calculate target aspect ratio
target_aspect_ratio = target_w / target_h
# Step 3: Grow current context area to meet the target aspect ratio
B, image_h, image_w, C = image.shape
context_aspect_ratio = w / h
if context_aspect_ratio < target_aspect_ratio:
# Grow width to meet aspect ratio
new_w = int(h * target_aspect_ratio)
new_h = h
new_x = x - (new_w - w) // 2
new_y = y
# Adjust new_x to keep within bounds
if new_x < 0:
shift = -new_x
if new_x + new_w + shift <= image_w:
new_x += shift
else:
overflow = (new_w - image_w) // 2
new_x = -overflow
elif new_x + new_w > image_w:
overflow = new_x + new_w - image_w
if new_x - overflow >= 0:
new_x -= overflow
else:
overflow = (new_w - image_w) // 2
new_x = -overflow
else:
# Grow height to meet aspect ratio
new_w = w
new_h = int(w / target_aspect_ratio)
new_x = x
new_y = y - (new_h - h) // 2
# Adjust new_y to keep within bounds
if new_y < 0:
shift = -new_y
if new_y + new_h + shift <= image_h:
new_y += shift
else:
overflow = (new_h - image_h) // 2
new_y = -overflow
elif new_y + new_h > image_h:
overflow = new_y + new_h - image_h
if new_y - overflow >= 0:
new_y -= overflow
else:
overflow = (new_h - image_h) // 2
new_y = -overflow
# Step 4: Grow the image to accommodate the new context area
up_padding, down_padding, left_padding, right_padding = 0, 0, 0, 0
expanded_image_w = image_w
expanded_image_h = image_h
# Adjust width for left overflow (x < 0) and right overflow (x + w > image_w)
if new_x < 0:
left_padding = -new_x
expanded_image_w += left_padding
if new_x + new_w > image_w:
right_padding = (new_x + new_w - image_w)
expanded_image_w += right_padding
# Adjust height for top overflow (y < 0) and bottom overflow (y + h > image_h)
if new_y < 0:
up_padding = -new_y
expanded_image_h += up_padding
if new_y + new_h > image_h:
down_padding = (new_y + new_h - image_h)
expanded_image_h += down_padding
# Step 5: Create the new image and mask
expanded_image = torch.zeros((image.shape[0], expanded_image_h, expanded_image_w, image.shape[3]), device=image.device)
expanded_mask = torch.ones((mask.shape[0], expanded_image_h, expanded_image_w), device=mask.device)
# Reorder the tensors to match the required dimension format for padding
image = image.permute(0, 3, 1, 2) # [B, H, W, C] -> [B, C, H, W]
expanded_image = expanded_image.permute(0, 3, 1, 2) # [B, H, W, C] -> [B, C, H, W]
# Ensure the expanded image has enough room to hold the padded version of the original image
expanded_image[:, :, up_padding:up_padding + image_h, left_padding:left_padding + image_w] = image
# Fill the new extended areas with the edge values of the image
if up_padding > 0:
expanded_image[:, :, :up_padding, left_padding:left_padding + image_w] = image[:, :, 0:1, left_padding:left_padding + image_w].repeat(1, 1, up_padding, 1)
if down_padding > 0:
expanded_image[:, :, -down_padding:, left_padding:left_padding + image_w] = image[:, :, -1:, left_padding:left_padding + image_w].repeat(1, 1, down_padding, 1)
if left_padding > 0:
expanded_image[:, :, up_padding:up_padding + image_h, :left_padding] = expanded_image[:, :, up_padding:up_padding + image_h, left_padding:left_padding+1].repeat(1, 1, 1, left_padding)
if right_padding > 0:
expanded_image[:, :, up_padding:up_padding + image_h, -right_padding:] = expanded_image[:, :, up_padding:up_padding + image_h, -right_padding-1:-right_padding].repeat(1, 1, 1, right_padding)
# Reorder the tensors back to [B, H, W, C] format
expanded_image = expanded_image.permute(0, 2, 3, 1) # [B, C, H, W] -> [B, H, W, C]
image = image.permute(0, 2, 3, 1) # [B, C, H, W] -> [B, H, W, C]
# Same for the mask
expanded_mask[:, up_padding:up_padding + image_h, left_padding:left_padding + image_w] = mask
# Record the cto values (canvas to original)
cto_x = left_padding
cto_y = up_padding
cto_w = image_w
cto_h = image_h
# The final expanded image and mask
canvas_image = expanded_image
canvas_mask = expanded_mask
# Step 6: Crop the image and mask around x, y, w, h
ctc_x = new_x+left_padding
ctc_y = new_y+up_padding
ctc_w = new_w
ctc_h = new_h
# Crop the image and mask
cropped_image = canvas_image[:, ctc_y:ctc_y + ctc_h, ctc_x:ctc_x + ctc_w]
cropped_mask = canvas_mask[:, ctc_y:ctc_y + ctc_h, ctc_x:ctc_x + ctc_w]
# Step 7: Resize image and mask to the target width and height
# Decide which algorithm to use based on the scaling direction
if target_w > ctc_w or target_h > ctc_h: # Upscaling
cropped_image = rescale_i(cropped_image, target_w, target_h, upscale_algorithm)
cropped_mask = rescale_m(cropped_mask, target_w, target_h, upscale_algorithm)
else: # Downscaling
cropped_image = rescale_i(cropped_image, target_w, target_h, downscale_algorithm)
cropped_mask = rescale_m(cropped_mask, target_w, target_h, downscale_algorithm)
return canvas_image, cto_x, cto_y, cto_w, cto_h, cropped_image, cropped_mask, ctc_x, ctc_y, ctc_w, ctc_h
def stitch_magic_im(canvas_image, inpainted_image, mask, ctc_x, ctc_y, ctc_w, ctc_h, cto_x, cto_y, cto_w, cto_h, downscale_algorithm, upscale_algorithm):
canvas_image = canvas_image.clone()
inpainted_image = inpainted_image.clone()
mask = mask.clone()
# Resize inpainted image and mask to match the context size
_, h, w, _ = inpainted_image.shape
if ctc_w > w or ctc_h > h: # Upscaling
resized_image = rescale_i(inpainted_image, ctc_w, ctc_h, upscale_algorithm)
resized_mask = rescale_m(mask, ctc_w, ctc_h, upscale_algorithm)
else: # Downscaling
resized_image = rescale_i(inpainted_image, ctc_w, ctc_h, downscale_algorithm)
resized_mask = rescale_m(mask, ctc_w, ctc_h, downscale_algorithm)
# Clamp mask to [0, 1] and expand to match image channels
resized_mask = resized_mask.clamp(0, 1).unsqueeze(-1) # shape: [1, H, W, 1]
# Extract the canvas region we're about to overwrite
canvas_crop = canvas_image[:, ctc_y:ctc_y + ctc_h, ctc_x:ctc_x + ctc_w]
# Blend: new = mask * inpainted + (1 - mask) * canvas
blended = resized_mask * resized_image + (1.0 - resized_mask) * canvas_crop
# Paste the blended region back onto the canvas
canvas_image[:, ctc_y:ctc_y + ctc_h, ctc_x:ctc_x + ctc_w] = blended
# Final crop to get back the original image area
output_image = canvas_image[:, cto_y:cto_y + cto_h, cto_x:cto_x + cto_w]
return output_image
class InpaintCropImproved:
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
# Required inputs
"image": ("IMAGE", {"tooltip": "Input image(s) to be processed. Can be a single image or batch of images for video processing."}),
# Resize algorithms
"downscale_algorithm": (["nearest", "bilinear", "bicubic", "lanczos", "box", "hamming"], {"default": "bilinear", "tooltip": "Algorithm used when downscaling images. Bilinear provides good balance of quality and speed."}),
"upscale_algorithm": (["nearest", "bilinear", "bicubic", "lanczos", "box", "hamming"], {"default": "bicubic", "tooltip": "Algorithm used when upscaling images. Bicubic provides better quality for upscaling."}),
# Pre-resize input image
"preresize": ("BOOLEAN", {"default": False, "tooltip": "Enable to resize the original image before processing. Useful for normalizing input sizes."}),
"preresize_mode": (["ensure minimum resolution", "ensure maximum resolution", "ensure minimum and maximum resolution"], {"default": "ensure minimum resolution", "tooltip": "Mode for pre-resizing: minimum ensures image is at least the specified size, maximum caps the size, both enforces a range."}),
"preresize_min_width": ("INT", {"default": 1024, "min": 0, "max": nodes.MAX_RESOLUTION, "step": 1, "tooltip": "Minimum width for pre-resize operation. Image will be upscaled if smaller."}),
"preresize_min_height": ("INT", {"default": 1024, "min": 0, "max": nodes.MAX_RESOLUTION, "step": 1, "tooltip": "Minimum height for pre-resize operation. Image will be upscaled if smaller."}),
"preresize_max_width": ("INT", {"default": nodes.MAX_RESOLUTION, "min": 0, "max": nodes.MAX_RESOLUTION, "step": 1, "tooltip": "Maximum width for pre-resize operation. Image will be downscaled if larger."}),
"preresize_max_height": ("INT", {"default": nodes.MAX_RESOLUTION, "min": 0, "max": nodes.MAX_RESOLUTION, "step": 1, "tooltip": "Maximum height for pre-resize operation. Image will be downscaled if larger."}),
# Mask manipulation
"mask_fill_holes": ("BOOLEAN", {"default": True, "tooltip": "Fill holes in the mask using iterative morphological operations. Helps create more complete masked regions."}),
"mask_expand_pixels": ("INT", {"default": 0, "min": 0, "max": nodes.MAX_RESOLUTION, "step": 1, "tooltip": "Expand the mask by this many pixels before processing. Useful for including more context around masked areas."}),
"mask_invert": ("BOOLEAN", {"default": False, "tooltip": "Invert the mask so that masked areas become unmasked and vice versa."}),
"mask_blend_pixels": ("INT", {"default": 32, "min": 0, "max": 64, "step": 1, "tooltip": "Create a soft transition zone around mask edges. Higher values create smoother blending during stitching."}),
"mask_hipass_filter": ("FLOAT", {"default": 0.1, "min": 0, "max": 1, "step": 0.01, "tooltip": "Remove mask values below this threshold. Helps eliminate weak mask areas and noise."}),
# Extend image for outpainting
"extend_for_outpainting": ("BOOLEAN", {"default": False, "tooltip": "Extend the image canvas for outpainting. Adds padding around the image for generating content beyond original boundaries."}),
"extend_up_factor": ("FLOAT", {"default": 1.0, "min": 0.01, "max": 100.0, "step": 0.01, "tooltip": "Factor to extend image upward. 1.0 = no extension, 1.5 = 50% extension upward."}),
"extend_down_factor": ("FLOAT", {"default": 1.0, "min": 0.01, "max": 100.0, "step": 0.01, "tooltip": "Factor to extend image downward. 1.0 = no extension, 1.5 = 50% extension downward."}),
"extend_left_factor": ("FLOAT", {"default": 1.0, "min": 0.01, "max": 100.0, "step": 0.01, "tooltip": "Factor to extend image leftward. 1.0 = no extension, 1.5 = 50% extension leftward."}),
"extend_right_factor": ("FLOAT", {"default": 1.0, "min": 0.01, "max": 100.0, "step": 0.01, "tooltip": "Factor to extend image rightward. 1.0 = no extension, 1.5 = 50% extension rightward."}),
# Context
"context_from_mask_extend_factor": ("FLOAT", {"default": 1.2, "min": 1.0, "max": 100.0, "step": 0.01, "tooltip": "Expand the context area around the mask by this factor. 1.2 = 20% expansion in all directions. Larger values include more surrounding context for better inpainting."}),
# Output
"output_resize_to_target_size": ("BOOLEAN", {"default": True, "tooltip": "Resize the output to specific dimensions. When disabled, output size depends on mask area and extend factor."}),
"output_target_width": ("INT", {"default": 512, "min": 64, "max": nodes.MAX_RESOLUTION, "step": 1, "tooltip": "Target width for output image when resize to target size is enabled. Should match your inpainting model's preferred resolution."}),
"output_target_height": ("INT", {"default": 512, "min": 64, "max": nodes.MAX_RESOLUTION, "step": 1, "tooltip": "Target height for output image when resize to target size is enabled. Should match your inpainting model's preferred resolution."}),
"output_padding": (["0", "8", "16", "32", "64", "128", "256", "512"], {"default": "32", "tooltip": "Padding to ensure output dimensions are multiples of this value. Important for models that require specific dimension alignment (e.g., 8 for VAE, 32 for some diffusion models)."}),
# Batch consistency
"keep_window_size": ("BOOLEAN", {"default": False, "tooltip": "Maintain consistent crop window size across all images in batch. Essential for video processing to avoid flickering. Uses maximum dimensions found and interpolates missing coordinates."}),
},
"optional": {
# Optional inputs
"mask": ("MASK", {"tooltip": "Mask defining areas to be inpainted. White areas will be inpainted, black areas will be preserved. If not provided, the entire image will be processed."}),
"optional_context_mask": ("MASK", {"tooltip": "Additional mask defining extra context areas to include in the crop. Useful for ensuring important surrounding details are preserved during inpainting."}),
}
}
FUNCTION = "inpaint_crop"
CATEGORY = "inpaint"
DESCRIPTION = """
Advanced inpainting crop node that intelligently crops images around masked areas for optimal inpainting results.
Features:
- Batch processing with temporal consistency for video sequences
- Automatic context area detection and expansion
- Smart resizing with aspect ratio preservation
- Optional pre-processing (resize, mask manipulation, outpainting extension)
- Window size consistency across batches for video processing
- Linear interpolation for smooth mask transitions in video sequences
The node processes masks to find the optimal crop area, applies various preprocessing steps,
and outputs cropped images ready for inpainting along with stitcher data for reconstruction.
"""
# Remove the following # to turn on debug mode (extra outputs, print statements)
#'''
DEBUG_MODE = False
RETURN_TYPES = ("STITCHER", "IMAGE", "MASK")
RETURN_NAMES = ("stitcher", "cropped_image", "cropped_mask")
OUTPUT_TOOLTIPS = ("Stitcher data containing all information needed to reconstruct the original image after inpainting. Pass this to InpaintStitchImproved.",
"Cropped and processed image ready for inpainting. Contains the masked area plus surrounding context.",
"Processed mask corresponding to the cropped image. Shows which areas need to be inpainted.")
'''
DEBUG_MODE = True # TODO
RETURN_TYPES = ("STITCHER", "IMAGE", "MASK",
# DEBUG
"IMAGE",
"MASK",
"MASK",
"MASK",
"MASK",
"MASK",
"MASK",
"IMAGE",
"MASK",
"MASK",
"IMAGE",
"MASK",
"IMAGE",
"MASK",
"IMAGE",
"MASK",
"IMAGE",
"IMAGE",
"MASK",
"IMAGE",
"IMAGE",
"IMAGE",
"MASK",
)
RETURN_NAMES = ("stitcher", "cropped_image", "cropped_mask",
# DEBUG
"DEBUG_preresize_image",
"DEBUG_preresize_mask",
"DEBUG_fillholes_mask",
"DEBUG_expand_mask",
"DEBUG_invert_mask",
"DEBUG_blur_mask",
"DEBUG_hipassfilter_mask",
"DEBUG_extend_image",
"DEBUG_extend_mask",
"DEBUG_context_from_mask",
"DEBUG_context_from_mask_location",
"DEBUG_context_expand",
"DEBUG_context_expand_location",
"DEBUG_context_with_context_mask",
"DEBUG_context_with_context_mask_location",
"DEBUG_context_to_target",
"DEBUG_context_to_target_location",
"DEBUG_context_to_target_image",
"DEBUG_context_to_target_mask",
"DEBUG_canvas_image",
"DEBUG_orig_in_canvas_location",
"DEBUG_cropped_in_canvas_location",
"DEBUG_cropped_mask_blend",
)
#'''
def inpaint_crop(self, image, downscale_algorithm, upscale_algorithm, preresize, preresize_mode, preresize_min_width, preresize_min_height, preresize_max_width, preresize_max_height, extend_for_outpainting, extend_up_factor, extend_down_factor, extend_left_factor, extend_right_factor, mask_hipass_filter, mask_fill_holes, mask_expand_pixels, mask_invert, mask_blend_pixels, context_from_mask_extend_factor, output_resize_to_target_size, output_target_width, output_target_height, output_padding, keep_window_size, mask=None, optional_context_mask=None):
image = image.clone()
if mask is not None:
mask = mask.clone()
if optional_context_mask is not None:
optional_context_mask = optional_context_mask.clone()
output_padding = int(output_padding)
# Check that some parameters make sense
if preresize and preresize_mode == "ensure minimum and maximum resolution":
assert preresize_max_width >= preresize_min_width, "Preresize maximum width must be greater than or equal to minimum width"
assert preresize_max_height >= preresize_min_height, "Preresize maximum height must be greater than or equal to minimum height"
if self.DEBUG_MODE:
print('Inpaint Crop Batch input')
print(image.shape, type(image), image.dtype)
if mask is not None:
print(mask.shape, type(mask), mask.dtype)
if optional_context_mask is not None:
print(optional_context_mask.shape, type(optional_context_mask), optional_context_mask.dtype)
if image.shape[0] > 1 and not keep_window_size:
assert output_resize_to_target_size, "output_resize_to_target_size must be enabled when input is a batch of images without keep_window_size enabled, given all images in the batch output have to be the same size (unless keep_window_size is enabled)"
# When a LoadImage node passes a mask without user editing, it may be the wrong shape.
# Detect and fix that to avoid shape mismatch errors.
if mask is not None and (image.shape[0] == 1 or mask.shape[0] == 1 or mask.shape[0] == image.shape[0]):
if mask.shape[1] != image.shape[1] or mask.shape[2] != image.shape[2]:
if torch.count_nonzero(mask) == 0:
mask = torch.zeros((mask.shape[0], image.shape[1], image.shape[2]), device=image.device, dtype=image.dtype)
if optional_context_mask is not None and (image.shape[0] == 1 or optional_context_mask.shape[0] == 1 or optional_context_mask.shape[0] == image.shape[0]):
if optional_context_mask.shape[1] != image.shape[1] or optional_context_mask.shape[2] != image.shape[2]:
if torch.count_nonzero(optional_context_mask) == 0:
optional_context_mask = torch.zeros((optional_context_mask.shape[0], image.shape[1], image.shape[2]), device=image.device, dtype=image.dtype)
# If no mask is provided, create one with the shape of the image
if mask is None:
mask = torch.zeros_like(image[:, :, :, 0])
# If there is only one image for many masks, replicate it for all masks
if mask.shape[0] > 1 and image.shape[0] == 1:
assert image.dim() == 4, f"Expected 4D BHWC image tensor, got {image.shape}"
image = image.expand(mask.shape[0], -1, -1, -1).clone()
# If there is only one mask for many images, replicate it for all images
if image.shape[0] > 1 and mask.shape[0] == 1:
assert mask.dim() == 3, f"Expected 3D BHW mask tensor, got {mask.shape}"
mask = mask.expand(image.shape[0], -1, -1).clone()
# If no optional_context_mask is provided, create one with the shape of the image
if optional_context_mask is None:
optional_context_mask = torch.zeros_like(image[:, :, :, 0])
# If there is only one optional_context_mask for many images, replicate it for all images
if image.shape[0] > 1 and optional_context_mask.shape[0] == 1:
assert optional_context_mask.dim() == 3, f"Expected 3D BHW optional_context_mask tensor, got {optional_context_mask.shape}"
optional_context_mask = optional_context_mask.expand(image.shape[0], -1, -1).clone()
if self.DEBUG_MODE:
print('Inpaint Crop Batch ready')
print(image.shape, type(image), image.dtype)
print(mask.shape, type(mask), mask.dtype)
print(optional_context_mask.shape, type(optional_context_mask), optional_context_mask.dtype)
# Validate data
assert image.ndimension() == 4, f"Expected 4 dimensions for image, got {image.ndimension()}"
assert mask.ndimension() == 3, f"Expected 3 dimensions for mask, got {mask.ndimension()}"
assert optional_context_mask.ndimension() == 3, f"Expected 3 dimensions for optional_context_mask, got {optional_context_mask.ndimension()}"
assert mask.shape[1:] == image.shape[1:3], f"Mask dimensions do not match image dimensions. Expected {image.shape[1:3]}, got {mask.shape[1:]}"
assert optional_context_mask.shape[1:] == image.shape[1:3], f"optional_context_mask dimensions do not match image dimensions. Expected {image.shape[1:3]}, got {optional_context_mask.shape[1:]}"
assert mask.shape[0] == image.shape[0], f"Mask batch does not match image batch. Expected {image.shape[0]}, got {mask.shape[0]}"
assert optional_context_mask.shape[0] == image.shape[0], f"Optional context mask batch does not match image batch. Expected {image.shape[0]}, got {optional_context_mask.shape[0]}"
# Run for each image separately
result_stitcher = {
'downscale_algorithm': downscale_algorithm,
'upscale_algorithm': upscale_algorithm,
'blend_pixels': mask_blend_pixels,
'canvas_to_orig_x': [],
'canvas_to_orig_y': [],
'canvas_to_orig_w': [],
'canvas_to_orig_h': [],
'canvas_image': [],
'cropped_to_canvas_x': [],
'cropped_to_canvas_y': [],
'cropped_to_canvas_w': [],
'cropped_to_canvas_h': [],
'cropped_mask_for_blend': [],
}
result_image = []
result_mask = []
debug_outputs = {name: [] for name in self.RETURN_NAMES if name.startswith("DEBUG_")}
batch_size = image.shape[0]
# Compute all contexts first
batch_contexts, max_w, max_h = self.compute_batch_contexts(
image, mask, optional_context_mask, downscale_algorithm, upscale_algorithm,
preresize, preresize_mode, preresize_min_width, preresize_min_height,
preresize_max_width, preresize_max_height, extend_for_outpainting,
extend_up_factor, extend_down_factor, extend_left_factor, extend_right_factor,
mask_hipass_filter, mask_fill_holes, mask_expand_pixels, mask_invert,
mask_blend_pixels, context_from_mask_extend_factor, keep_window_size
)
# Process each image with pre-computed contexts
for b in range(batch_size):
ctx = batch_contexts[b]
outputs = self.inpaint_crop_single_image_with_context(
ctx['processed_image'], ctx['processed_mask'], ctx['processed_optional_context_mask'],
ctx['x'], ctx['y'], ctx['w'], ctx['h'], ctx['context'],
downscale_algorithm, upscale_algorithm, mask_blend_pixels,
output_resize_to_target_size, output_target_width, output_target_height, output_padding,
keep_window_size, max_w, max_h, context_from_mask_extend_factor)
stitcher, cropped_image, cropped_mask = outputs[:3]
for key in ['canvas_to_orig_x', 'canvas_to_orig_y', 'canvas_to_orig_w', 'canvas_to_orig_h', 'canvas_image', 'cropped_to_canvas_x', 'cropped_to_canvas_y', 'cropped_to_canvas_w', 'cropped_to_canvas_h', 'cropped_mask_for_blend']:
result_stitcher[key].append(stitcher[key])
cropped_image = cropped_image.clone().squeeze(0)
result_image.append(cropped_image)
cropped_mask = cropped_mask.clone().squeeze(0)
result_mask.append(cropped_mask)
# Handle the DEBUG_ fields dynamically
for name, output in zip(self.RETURN_NAMES[3:], outputs[3:]): # Start from index 3 since first 3 are fixed
if name.startswith("DEBUG_"):
output_array = output.squeeze(0) # Assuming output needs to be squeezed similar to image/mask
debug_outputs[name].append(output_array)
result_image = torch.stack(result_image, dim=0)
result_mask = torch.stack(result_mask, dim=0)
if self.DEBUG_MODE:
print('Inpaint Crop Batch output')
print(result_image.shape, type(result_image), result_image.dtype)
print(result_mask.shape, type(result_mask), result_mask.dtype)
debug_outputs = {name: torch.stack(values, dim=0) for name, values in debug_outputs.items()}
return result_stitcher, result_image, result_mask, [debug_outputs[name] for name in self.RETURN_NAMES if name.startswith("DEBUG_")]
def compute_batch_contexts(self, image, mask, optional_context_mask, downscale_algorithm, upscale_algorithm,
preresize, preresize_mode, preresize_min_width, preresize_min_height,
preresize_max_width, preresize_max_height, extend_for_outpainting,
extend_up_factor, extend_down_factor, extend_left_factor, extend_right_factor,
mask_hipass_filter, mask_fill_holes, mask_expand_pixels, mask_invert,
mask_blend_pixels, context_from_mask_extend_factor, keep_window_size):
"""
Compute contexts for all images in the batch before processing.
Returns processed masks, contexts, and coordinates for each image.
"""
batch_size = image.shape[0]
batch_contexts = []
# Step 1: Apply mask preprocessing for all images
for b in range(batch_size):
one_image = image[b].unsqueeze(0)
one_mask = mask[b].unsqueeze(0)
one_optional_context_mask = optional_context_mask[b].unsqueeze(0)
# Apply all mask preprocessing steps
if preresize:
one_image, one_mask, one_optional_context_mask = preresize_imm(
one_image, one_mask, one_optional_context_mask, downscale_algorithm,
upscale_algorithm, preresize_mode, preresize_min_width, preresize_min_height,
preresize_max_width, preresize_max_height)
if mask_fill_holes:
one_mask = fillholes_iterative_hipass_fill_m(one_mask)
if mask_expand_pixels > 0:
one_mask = expand_m(one_mask, mask_expand_pixels)
if mask_invert:
one_mask = invert_m(one_mask)
if mask_blend_pixels > 0:
one_mask = expand_m(one_mask, mask_blend_pixels)
one_mask = blur_m(one_mask, mask_blend_pixels*0.5)
if mask_hipass_filter >= 0.01:
one_mask = hipassfilter_m(one_mask, mask_hipass_filter)
one_optional_context_mask = hipassfilter_m(one_optional_context_mask, mask_hipass_filter)
if extend_for_outpainting:
one_image, one_mask, one_optional_context_mask = extend_imm(
one_image, one_mask, one_optional_context_mask, extend_up_factor,
extend_down_factor, extend_left_factor, extend_right_factor)
batch_contexts.append({
'processed_image': one_image,
'processed_mask': one_mask,
'processed_optional_context_mask': one_optional_context_mask
})
# Step 2: Find initial contexts without applying context operations yet
initial_contexts = []
for ctx in batch_contexts:
context, x, y, w, h = findcontextarea_m(ctx['processed_mask'])
initial_contexts.append({'x': x, 'y': y, 'w': w, 'h': h})
# Step 3: Apply keep_window_size logic if enabled
max_w = None
max_h = None
if keep_window_size and batch_size > 1:
# Find maximum dimensions from initial contexts
max_w = max(ctx['w'] for ctx in initial_contexts)
max_h = max(ctx['h'] for ctx in initial_contexts)
# Update all initial contexts to use consistent window size
# First, interpolate x and y coordinates for frames with invalid values (-1)
def interpolate_coordinates(coords):
"""Interpolate missing coordinates (-1) using linear interpolation between valid values"""
coords = coords.copy()
n = len(coords)
# Find all valid indices (where coord != -1)
valid_indices = [i for i, coord in enumerate(coords) if coord != -1]
if not valid_indices:
# No valid coordinates, fill with zeros
return [0] * n
# Handle leading -1s: use first valid value
first_valid_idx = valid_indices[0]
first_valid_value = coords[first_valid_idx]
for i in range(first_valid_idx):
coords[i] = first_valid_value
# Handle trailing -1s: use last valid value
last_valid_idx = valid_indices[-1]
last_valid_value = coords[last_valid_idx]
for i in range(last_valid_idx + 1, n):
coords[i] = last_valid_value
# Interpolate between valid values
for i in range(len(valid_indices) - 1):
start_idx = valid_indices[i]
end_idx = valid_indices[i + 1]
start_val = coords[start_idx]
end_val = coords[end_idx]
# Linear interpolation for indices between start_idx and end_idx
for j in range(start_idx + 1, end_idx):
alpha = (j - start_idx) / (end_idx - start_idx)
coords[j] = int(start_val + alpha * (end_val - start_val))
return coords
# Extract x and y coordinates
x_coords = [ctx['x'] for ctx in initial_contexts]
y_coords = [ctx['y'] for ctx in initial_contexts]
# Interpolate invalid coordinates
x_coords = interpolate_coordinates(x_coords)
y_coords = interpolate_coordinates(y_coords)
# Update contexts with interpolated coordinates
for i, ctx in enumerate(initial_contexts):
ctx['x'] = x_coords[i]
ctx['y'] = y_coords[i]
# Update dimensions to maximum
ctx['w'] = max_w
ctx['h'] = max_h
# Ensure we don't go outside image boundaries
img_h, img_w = batch_contexts[i]['processed_image'].shape[1], batch_contexts[i]['processed_image'].shape[2]
if ctx['x'] + ctx['w'] > img_w:
ctx['x'] = max(0, img_w - ctx['w'])
if ctx['y'] + ctx['h'] > img_h:
ctx['y'] = max(0, img_h - ctx['h'])
# Step 4: Now apply context operations (grow context, combine with optional mask) after window size keeping
for i, ctx in enumerate(batch_contexts):
x, y, w, h = initial_contexts[i]['x'], initial_contexts[i]['y'], initial_contexts[i]['w'], initial_contexts[i]['h']
# Find initial context
context, x, y, w, h = findcontextarea_m(ctx['processed_mask'])
if x == -1 or w == -1 or h == -1 or y == -1:
x, y, w, h = initial_contexts[i]['x'], initial_contexts[i]['y'], initial_contexts[i]['w'], initial_contexts[i]['h']
context = ctx['processed_mask'][:, y:y+h, x:x+w]
else:
# Use the coordinates from keep_window_size if enabled
if keep_window_size and batch_size > 1:
x, y, w, h = initial_contexts[i]['x'], initial_contexts[i]['y'], initial_contexts[i]['w'], initial_contexts[i]['h']
context = ctx['processed_mask'][:, y:y+h, x:x+w]
# Grow context if needed
if context_from_mask_extend_factor >= 1.01:
context, x, y, w, h = growcontextarea_m(context, ctx['processed_mask'], x, y, w, h, context_from_mask_extend_factor)
if x == -1 or w == -1 or h == -1 or y == -1:
x, y, w, h = 0, 0, ctx['processed_image'].shape[2], ctx['processed_image'].shape[1]
context = ctx['processed_mask'][:, y:y+h, x:x+w]
# Combine with optional context mask
context, x, y, w, h = combinecontextmask_m(context, ctx['processed_mask'], x, y, w, h, ctx['processed_optional_context_mask'])
if x == -1 or w == -1 or h == -1 or y == -1:
x, y, w, h = 0, 0, ctx['processed_image'].shape[2], ctx['processed_image'].shape[1]
context = ctx['processed_mask'][:, y:y+h, x:x+w]
# Update the context with final values
ctx['context'] = context
ctx['x'] = x
ctx['y'] = y
ctx['w'] = w
ctx['h'] = h
return batch_contexts, max_w, max_h
def inpaint_crop_single_image_with_context(self, image, mask, optional_context_mask, x, y, w, h, context,
downscale_algorithm, upscale_algorithm, mask_blend_pixels,
output_resize_to_target_size, output_target_width, output_target_height, output_padding,
keep_window_size=False, max_w=None, max_h=None, context_from_mask_extend_factor=1.0):
"""
Process a single image with pre-computed context coordinates.
This is the simplified version that skips all the mask preprocessing since it's already done.
"""
if self.DEBUG_MODE:
DEBUG_preresize_image = image.clone()
DEBUG_preresize_mask = mask.clone()
DEBUG_fillholes_mask = mask.clone()
DEBUG_expand_mask = mask.clone()
DEBUG_invert_mask = mask.clone()
DEBUG_blur_mask = mask.clone()
DEBUG_hipassfilter_mask = mask.clone()
DEBUG_extend_image = image.clone()
DEBUG_extend_mask = mask.clone()
DEBUG_context_from_mask = context.clone()
DEBUG_context_from_mask_location = debug_context_location_in_image(image, x, y, w, h)
DEBUG_context_expand = context.clone()
DEBUG_context_expand_location = debug_context_location_in_image(image, x, y, w, h)
DEBUG_context_with_context_mask = context.clone()
DEBUG_context_with_context_mask_location = debug_context_location_in_image(image, x, y, w, h)
if not output_resize_to_target_size:
# If keep_window_size is enabled and we have max dimensions, use them instead of w,h
if keep_window_size and max_w is not None and max_h is not None:
# Dont't forget to consider context_from_mask_extend_factor
extended_w = int(max_w * context_from_mask_extend_factor)
extended_h = int(max_h * context_from_mask_extend_factor)
canvas_image, cto_x, cto_y, cto_w, cto_h, cropped_image, cropped_mask, ctc_x, ctc_y, ctc_w, ctc_h = crop_magic_im(image, mask, x, y, w, h, extended_w, extended_h, output_padding, downscale_algorithm, upscale_algorithm)
else:
# Use the actual context dimensions w,h which already include the extend factor
canvas_image, cto_x, cto_y, cto_w, cto_h, cropped_image, cropped_mask, ctc_x, ctc_y, ctc_w, ctc_h = crop_magic_im(image, mask, x, y, w, h, w, h, output_padding, downscale_algorithm, upscale_algorithm)
else: # if output_resize_to_target_size:
canvas_image, cto_x, cto_y, cto_w, cto_h, cropped_image, cropped_mask, ctc_x, ctc_y, ctc_w, ctc_h = crop_magic_im(image, mask, x, y, w, h, output_target_width, output_target_height, output_padding, downscale_algorithm, upscale_algorithm)
if self.DEBUG_MODE:
DEBUG_context_to_target = context.clone()
DEBUG_context_to_target_location = debug_context_location_in_image(image, x, y, w, h)
DEBUG_context_to_target_image = image.clone()
DEBUG_context_to_target_mask = mask.clone()
DEBUG_canvas_image = canvas_image.clone()
DEBUG_orig_in_canvas_location = debug_context_location_in_image(canvas_image, cto_x, cto_y, cto_w, cto_h)
DEBUG_cropped_in_canvas_location = debug_context_location_in_image(canvas_image, ctc_x, ctc_y, ctc_w, ctc_h)
# For blending, grow the mask even further and make it blurrier.
cropped_mask_blend = cropped_mask.clone()
if mask_blend_pixels > 0:
cropped_mask_blend = blur_m(cropped_mask_blend, mask_blend_pixels*0.5)
if self.DEBUG_MODE:
DEBUG_cropped_mask_blend = cropped_mask_blend.clone()
stitcher = {
'canvas_to_orig_x': cto_x,
'canvas_to_orig_y': cto_y,
'canvas_to_orig_w': cto_w,
'canvas_to_orig_h': cto_h,
'canvas_image': canvas_image,
'cropped_to_canvas_x': ctc_x,
'cropped_to_canvas_y': ctc_y,
'cropped_to_canvas_w': ctc_w,
'cropped_to_canvas_h': ctc_h,
'cropped_mask_for_blend': cropped_mask_blend,
}
if not self.DEBUG_MODE:
return stitcher, cropped_image, cropped_mask
else:
return stitcher, cropped_image, cropped_mask, DEBUG_preresize_image, DEBUG_preresize_mask, DEBUG_fillholes_mask, DEBUG_expand_mask, DEBUG_invert_mask, DEBUG_blur_mask, DEBUG_hipassfilter_mask, DEBUG_extend_image, DEBUG_extend_mask, DEBUG_context_from_mask, DEBUG_context_from_mask_location, DEBUG_context_expand, DEBUG_context_expand_location, DEBUG_context_with_context_mask, DEBUG_context_with_context_mask_location, DEBUG_context_to_target, DEBUG_context_to_target_location, DEBUG_context_to_target_image, DEBUG_context_to_target_mask, DEBUG_canvas_image, DEBUG_orig_in_canvas_location, DEBUG_cropped_in_canvas_location, DEBUG_cropped_mask_blend
def inpaint_crop_single_image(self, image, downscale_algorithm, upscale_algorithm, preresize, preresize_mode, preresize_min_width, preresize_min_height, preresize_max_width, preresize_max_height, extend_for_outpainting, extend_up_factor, extend_down_factor, extend_left_factor, extend_right_factor, mask_hipass_filter, mask_fill_holes, mask_expand_pixels, mask_invert, mask_blend_pixels, context_from_mask_extend_factor, output_resize_to_target_size, output_target_width, output_target_height, output_padding, mask, optional_context_mask):
if preresize:
image, mask, optional_context_mask = preresize_imm(image, mask, optional_context_mask, downscale_algorithm, upscale_algorithm, preresize_mode, preresize_min_width, preresize_min_height, preresize_max_width, preresize_max_height)
if self.DEBUG_MODE:
DEBUG_preresize_image = image.clone()
DEBUG_preresize_mask = mask.clone()
if mask_fill_holes:
mask = fillholes_iterative_hipass_fill_m(mask)
if self.DEBUG_MODE:
DEBUG_fillholes_mask = mask.clone()
if mask_expand_pixels > 0:
mask = expand_m(mask, mask_expand_pixels)
if self.DEBUG_MODE:
DEBUG_expand_mask = mask.clone()
if mask_invert:
mask = invert_m(mask)
if self.DEBUG_MODE:
DEBUG_invert_mask = mask.clone()
if mask_blend_pixels > 0:
mask = expand_m(mask, mask_blend_pixels)
mask = blur_m(mask, mask_blend_pixels*0.5)
if self.DEBUG_MODE:
DEBUG_blur_mask = mask.clone()
if mask_hipass_filter >= 0.01:
mask = hipassfilter_m(mask, mask_hipass_filter)
optional_context_mask = hipassfilter_m(optional_context_mask, mask_hipass_filter)
if self.DEBUG_MODE:
DEBUG_hipassfilter_mask = mask.clone()
if extend_for_outpainting:
image, mask, optional_context_mask = extend_imm(image, mask, optional_context_mask, extend_up_factor, extend_down_factor, extend_left_factor, extend_right_factor)
if self.DEBUG_MODE:
DEBUG_extend_image = image.clone()
DEBUG_extend_mask = mask.clone()
context, x, y, w, h = findcontextarea_m(mask)
# If no mask, mask everything for some inpainting.
if x == -1 or w == -1 or h == -1 or y == -1:
x, y, w, h = 0, 0, image.shape[2], image.shape[1]
context = mask[:, y:y+h, x:x+w]
if self.DEBUG_MODE:
DEBUG_context_from_mask = context.clone()
DEBUG_context_from_mask_location = debug_context_location_in_image(image, x, y, w, h)
if context_from_mask_extend_factor >= 1.01:
context, x, y, w, h = growcontextarea_m(context, mask, x, y, w, h, context_from_mask_extend_factor)
# If no mask, mask everything for some inpainting.
if x == -1 or w == -1 or h == -1 or y == -1:
x, y, w, h = 0, 0, image.shape[2], image.shape[1]
context = mask[:, y:y+h, x:x+w]
if self.DEBUG_MODE:
DEBUG_context_expand = context.clone()
DEBUG_context_expand_location = debug_context_location_in_image(image, x, y, w, h)
context, x, y, w, h = combinecontextmask_m(context, mask, x, y, w, h, optional_context_mask)
# If no mask, mask everything for some inpainting.
if x == -1 or w == -1 or h == -1 or y == -1:
x, y, w, h = 0, 0, image.shape[2], image.shape[1]
context = mask[:, y:y+h, x:x+w]
if self.DEBUG_MODE:
DEBUG_context_with_context_mask = context.clone()
DEBUG_context_with_context_mask_location = debug_context_location_in_image(image, x, y, w, h)
if not output_resize_to_target_size:
canvas_image, cto_x, cto_y, cto_w, cto_h, cropped_image, cropped_mask, ctc_x, ctc_y, ctc_w, ctc_h = crop_magic_im(image, mask, x, y, w, h, w, h, output_padding, downscale_algorithm, upscale_algorithm)
else: # if output_resize_to_target_size:
canvas_image, cto_x, cto_y, cto_w, cto_h, cropped_image, cropped_mask, ctc_x, ctc_y, ctc_w, ctc_h = crop_magic_im(image, mask, x, y, w, h, output_target_width, output_target_height, output_padding, downscale_algorithm, upscale_algorithm)
if self.DEBUG_MODE:
DEBUG_context_to_target = context.clone()
DEBUG_context_to_target_location = debug_context_location_in_image(image, x, y, w, h)
DEBUG_context_to_target_image = image.clone()
DEBUG_context_to_target_mask = mask.clone()
DEBUG_canvas_image = canvas_image.clone()
DEBUG_orig_in_canvas_location = debug_context_location_in_image(canvas_image, cto_x, cto_y, cto_w, cto_h)
DEBUG_cropped_in_canvas_location = debug_context_location_in_image(canvas_image, ctc_x, ctc_y, ctc_w, ctc_h)
# For blending, grow the mask even further and make it blurrier.
cropped_mask_blend = cropped_mask.clone()
if mask_blend_pixels > 0:
cropped_mask_blend = blur_m(cropped_mask_blend, mask_blend_pixels*0.5)
if self.DEBUG_MODE:
DEBUG_cropped_mask_blend = cropped_mask_blend.clone()
stitcher = {
'canvas_to_orig_x': cto_x,
'canvas_to_orig_y': cto_y,
'canvas_to_orig_w': cto_w,
'canvas_to_orig_h': cto_h,
'canvas_image': canvas_image,
'cropped_to_canvas_x': ctc_x,
'cropped_to_canvas_y': ctc_y,
'cropped_to_canvas_w': ctc_w,
'cropped_to_canvas_h': ctc_h,
'cropped_mask_for_blend': cropped_mask_blend,
}
if not self.DEBUG_MODE:
return stitcher, cropped_image, cropped_mask
else:
return stitcher, cropped_image, cropped_mask, DEBUG_preresize_image, DEBUG_preresize_mask, DEBUG_fillholes_mask, DEBUG_expand_mask, DEBUG_invert_mask, DEBUG_blur_mask, DEBUG_hipassfilter_mask, DEBUG_extend_image, DEBUG_extend_mask, DEBUG_context_from_mask, DEBUG_context_from_mask_location, DEBUG_context_expand, DEBUG_context_expand_location, DEBUG_context_with_context_mask, DEBUG_context_with_context_mask_location, DEBUG_context_to_target, DEBUG_context_to_target_location, DEBUG_context_to_target_image, DEBUG_context_to_target_mask, DEBUG_canvas_image, DEBUG_orig_in_canvas_location, DEBUG_cropped_in_canvas_location, DEBUG_cropped_mask_blend
class InpaintStitchImproved:
"""
Advanced inpainting stitch node that reconstructs full images from cropped inpainted regions.
This node takes the output from InpaintCropImproved and stitches the inpainted results back
into the original image context. It supports multiple blending modes and handles batch processing
for video sequences while maintaining temporal consistency.
"""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"stitcher": ("STITCHER", {"tooltip": "Stitcher data from InpaintCropImproved containing reconstruction information including canvas coordinates, dimensions, and blending masks."}),
"inpainted_image": ("IMAGE", {"tooltip": "The inpainted image(s) to be stitched back. Should correspond to the cropped output from InpaintCropImproved after inpainting."}),
"stitch_mode": (["mask_only", "entire_crop", "blend_entire"], {"default": "mask_only", "tooltip": "Stitching method: 'mask_only' blends only masked areas, 'entire_crop' replaces the entire cropped region, 'blend_entire' smoothly blends the entire crop with surrounding areas."}),
"blend_strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01, "tooltip": "Strength of blending operation. 1.0 = full inpainted result, 0.0 = original image preserved. Only affects blend modes."}),
}
}
CATEGORY = "inpaint"
DESCRIPTION = """
Reconstructs full-resolution images by stitching inpainted crops back into their original context.
Features:
- Multiple stitching modes for different use cases
- Batch processing support for video sequences
- Smart blending to avoid seams and artifacts
- Automatic coordinate transformation and scaling
- Preserves original image quality in non-inpainted areas
The node handles all the complex coordinate transformations, scaling, and blending
needed to seamlessly integrate inpainted content back into the source images.
"""
RETURN_TYPES = ("IMAGE",)
RETURN_NAMES = ("image",)
OUTPUT_TOOLTIPS = ("Reconstructed full-resolution image(s) with inpainted content seamlessly integrated. Maintains original dimensions and quality.",)
FUNCTION = "inpaint_stitch"
def inpaint_stitch(self, stitcher, inpainted_image, stitch_mode, blend_strength):
inpainted_image = inpainted_image.clone()
results = []
batch_size = inpainted_image.shape[0]
assert len(stitcher['cropped_to_canvas_x']) == batch_size or len(stitcher['cropped_to_canvas_x']) == 1, "Stitch batch size doesn't match image batch size"
override = False
if len(stitcher['cropped_to_canvas_x']) != batch_size and len(stitcher['cropped_to_canvas_x']) == 1:
override = True
for b in range(batch_size):
one_image = inpainted_image[b]
one_stitcher = {}
for key in ['downscale_algorithm', 'upscale_algorithm', 'blend_pixels']:
one_stitcher[key] = stitcher[key]
for key in ['canvas_to_orig_x', 'canvas_to_orig_y', 'canvas_to_orig_w', 'canvas_to_orig_h', 'canvas_image', 'cropped_to_canvas_x', 'cropped_to_canvas_y', 'cropped_to_canvas_w', 'cropped_to_canvas_h', 'cropped_mask_for_blend']:
if override: # One stitcher for many images, always read 0.
one_stitcher[key] = stitcher[key][0]
else:
one_stitcher[key] = stitcher[key][b]
one_image = one_image.unsqueeze(0)
one_image, = self.inpaint_stitch_single_image(one_stitcher, one_image, stitch_mode, blend_strength)
one_image = one_image.squeeze(0)
one_image = one_image.clone()
results.append(one_image)
result_batch = torch.stack(results, dim=0)
return (result_batch,)
def inpaint_stitch_single_image(self, stitcher, inpainted_image, stitch_mode, blend_strength):
downscale_algorithm = stitcher['downscale_algorithm']
upscale_algorithm = stitcher['upscale_algorithm']
canvas_image = stitcher['canvas_image']
ctc_x = stitcher['cropped_to_canvas_x']
ctc_y = stitcher['cropped_to_canvas_y']
ctc_w = stitcher['cropped_to_canvas_w']
ctc_h = stitcher['cropped_to_canvas_h']
cto_x = stitcher['canvas_to_orig_x']
cto_y = stitcher['canvas_to_orig_y']
cto_w = stitcher['canvas_to_orig_w']
cto_h = stitcher['canvas_to_orig_h']
mask = stitcher['cropped_mask_for_blend'] # shape: [1, H, W]
if stitch_mode == "mask_only":
# Original behavior - use mask for blending
output_image = stitch_magic_im(canvas_image, inpainted_image, mask, ctc_x, ctc_y, ctc_w, ctc_h, cto_x, cto_y, cto_w, cto_h, downscale_algorithm, upscale_algorithm)
elif stitch_mode == "entire_crop":
# Replace entire cropped region - create a full mask
full_mask = torch.ones_like(mask)
output_image = stitch_magic_im(canvas_image, inpainted_image, full_mask, ctc_x, ctc_y, ctc_w, ctc_h, cto_x, cto_y, cto_w, cto_h, downscale_algorithm, upscale_algorithm)
elif stitch_mode == "blend_entire":
# Blend entire cropped region with adjustable strength
full_mask = torch.ones_like(mask) * blend_strength
output_image = stitch_magic_im(canvas_image, inpainted_image, full_mask, ctc_x, ctc_y, ctc_w, ctc_h, cto_x, cto_y, cto_w, cto_h, downscale_algorithm, upscale_algorithm)
return (output_image,)
# A dictionary that contains all nodes you want to export with their names
# NOTE: names should be globally unique
NODE_CLASS_MAPPINGS = {
"TiledWanImageToMask": ImageToMask,
"TiledWanImageStatistics": ImageStatistics,
"TiledWanMaskStatistics": MaskStatistics,
"TileAndStitchBack": TileAndStitchBack,
"TiledWanVideoVACEpipe": TiledWanVideoVACEpipe,
"TiledWanInpaintCropImproved": InpaintCropImproved,
"TiledWanInpaintStitchImproved": InpaintStitchImproved,
}
# A dictionary that contains the friendly/humanly readable titles for the nodes
NODE_DISPLAY_NAME_MAPPINGS = {
"TiledWanImageToMask": "TiledWan Image To Mask",
"TiledWanImageStatistics": "TiledWan Image Statistics",
"TiledWanMaskStatistics": "TiledWan Mask Statistics",
"TileAndStitchBack": "Tile and Stitch Back",
"TiledWanVideoVACEpipe": "TiledWan Video VACE Pipeline",
"TiledWanInpaintCropImproved": "TiledWan Inpaint Crop Improved",
"TiledWanInpaintStitchImproved": "TiledWan Inpaint Stitch Improved"
}