Compare commits

..
Author SHA1 Message Date
Mel Massadian 2441f19db3 ⬆️ Bump version: 0.2.0 → 0.2.1 2024-12-30 18:46:35 +01:00
Mel Massadian b01e027ec0 Merge branch 'main' into pr-227 2024-12-30 18:40:13 +01:00
Mel Massadian 9a4ecb2b90 fix: 🐛 handle missing submodules
the nodes should never fail to load completely.
I still need to remove the few remaining side effects like this one.
2024-12-30 18:37:43 +01:00
Robin Huang 885688e7c7 Checkout submodules before publishing. 2024-12-27 14:39:48 -08:00
5 changed files with 37 additions and 664 deletions
+10 -34
View File
@@ -44,22 +44,14 @@ class MTB_ToDevice:
if torch.backends.mps.is_available(): if torch.backends.mps.is_available():
devices.append("mps") devices.append("mps")
if torch.cuda.is_available(): if torch.cuda.is_available():
devices.append("cuda:0")
for i in range(1, torch.cuda.device_count()):
devices.append(f"cuda:{i}")
devices.append("cuda") devices.append("cuda")
for i in range(torch.cuda.device_count()):
devices.append(f"cuda{i}")
return { return {
"required": { "required": {
"ignore_errors": ("BOOLEAN", {"default": False}), "ignore_errors": ("BOOLEAN", {"default": False}),
"device": ( "device": (devices, {"default": "cpu"}),
devices,
{
"default": "cuda"
if torch.cuda.is_available()
else "cpu"
},
),
}, },
"optional": { "optional": {
"image": ("IMAGE",), "image": ("IMAGE",),
@@ -75,36 +67,20 @@ class MTB_ToDevice:
def to_device( def to_device(
self, self,
*, *,
ignore_errors: bool = False, ignore_errors=False,
device: str = "cuda", device="cuda",
image: torch.Tensor | None = None, image: torch.Tensor | None = None,
mask: torch.Tensor | None = None, mask: torch.Tensor | None = None,
): ):
if not ignore_errors and image is None and mask is None: if not ignore_errors and image is None and mask is None:
raise ValueError( raise ValueError(
"You must either provide an image or a mask," "You must either provide an image or a mask,"
+ " use ignore_error to passthrough" " use ignore_error to passthrough"
)
if (
device.startswith("cuda")
and ":" not in device
and device != "cuda"
):
device = f"cuda:{device[4:]}"
try:
if image is not None:
image = image.to(device)
if mask is not None:
mask = mask.to(device)
except RuntimeError as e:
if not ignore_errors:
raise RuntimeError(
f"Failed to move tensor to device {device}: {str(e)}"
) from e
log.warning(
f"Failed to move tensor to device {device}, ignoring: {str(e)}"
) )
if image is not None:
image = image.to(device)
if mask is not None:
mask = mask.to(device)
return (image, mask) return (image, mask)
-458
View File
@@ -1,458 +0,0 @@
import comfy.utils
import torch
import torch.nn.functional as F
from ..log import log
class MTB_SceneCutDetector:
"""Detects scene cuts in a video using various methods (content, histogram, hash, or adaptive)"""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"frames": (
"IMAGE",
{"tooltip": "The frames used for processing"},
),
"method": (
["content", "histogram", "hash", "adaptive"],
{
"default": "histogram",
"tooltip": "only histogram works properly for now",
},
),
"downsample": (
["0.1x", "0.25x", "0.5x", "0.75x", "1.0x"],
{
"default": "0.1x",
"tooltip": "Downsample 'frames' (only for processing)",
},
),
"min_scene_length": (
"INT",
{
"default": 15,
"min": 1,
"max": 1000,
"tooltip": "the minimum number of frames a cut can be",
},
),
# content
"content_threshold": (
"FLOAT",
{"default": 0.1, "min": 0.0, "max": 1.0, "step": 0.001},
),
# histogram
"histogram_threshold": (
"FLOAT",
{"default": 0.20, "min": 0.0, "max": 1.0, "step": 0.001},
),
"histogram_bins": (
"INT",
{"default": 32, "min": 2, "max": 256},
),
# hash
"hash_threshold": (
"FLOAT",
{"default": 0.395, "min": 0.0, "max": 1.0, "step": 0.001},
),
"hash_size": ("INT", {"default": 16, "min": 8, "max": 64}),
# adaptive
"adaptive_threshold": (
"FLOAT",
{"default": 3.0, "min": 0.0, "max": 10.0, "step": 0.001},
),
"window_width": ("INT", {"default": 2, "min": 1, "max": 10}),
"min_content_val": (
"FLOAT",
{"default": 15.0, "min": 0.0, "max": 100.0},
),
},
"optional": {
"original_frames": (
"IMAGE",
{
"tooltip": "If provided the returned list will use these frames."
},
),
},
}
FUNCTION = "detect_cuts"
RETURN_TYPES = ("IMAGE",)
RETURN_NAMES = ("sequences",)
OUTPUT_IS_LIST = (True,)
CATEGORY = "mtb/video"
def detect_cuts(
self,
frames: torch.Tensor,
method: str,
min_scene_length: int,
content_threshold: float = 27.0,
histogram_threshold: float = 0.05,
histogram_bins: int = 64,
hash_threshold: float = 0.395,
hash_size: int = 16,
adaptive_threshold: float = 3.0,
window_width: int = 2,
min_content_val: float = 15.0,
downsample: str = "1.0x",
original_frames: torch.Tensor | None = None,
) -> tuple[list[torch.Tensor]]:
processing_frames = frames
frames_to_split = (
original_frames if original_frames is not None else frames
)
if downsample != "1.0x":
scale = float(downsample.replace("x", ""))
h, w = frames.shape[1:3]
new_h, new_w = int(h * scale), int(w * scale)
processing_frames = F.interpolate(
frames.permute(0, 3, 1, 2), # [B,C,H,W] for interpolate
size=(new_h, new_w),
mode="bilinear",
align_corners=False,
).permute(0, 2, 3, 1) # Back to [B,H,W,C]
cuts = []
if method == "content":
cuts = self.detect_content_cuts(
processing_frames, content_threshold, min_scene_length
)
elif method == "histogram":
cuts = self.detect_histogram_cuts(
processing_frames,
histogram_threshold,
histogram_bins,
min_scene_length,
)
elif method == "hash":
cuts = self.detect_hash_cuts(
processing_frames, hash_threshold, hash_size, min_scene_length
)
elif method == "adaptive":
cuts = self.detect_adaptive_cuts(
processing_frames,
adaptive_threshold,
window_width,
min_content_val,
min_scene_length,
)
# always include end
cuts.append(frames.shape[0])
# split into list
sequences = [
frames_to_split[cuts[i] : cuts[i + 1]]
for i in range(len(cuts) - 1)
]
log.debug(f"Found {len(sequences)} cuts")
return (sequences,)
def detect_content_cuts(
self,
frames: torch.Tensor,
threshold: float,
min_scene_length: int,
) -> list[int]:
"""Content-based cut detection using frame differences"""
num_frames = frames.shape[0]
device = frames.device
cuts = [0]
last_cut = 0
total = (
max(0, (num_frames - min_scene_length) - min_scene_length)
+ num_frames
)
pbar = comfy.utils.ProgressBar(total)
differences = torch.zeros(num_frames - 1, device=device)
for i in range(num_frames - 1):
differences[i] = self.compute_content_difference(
frames[i], frames[i + 1]
)
pbar.update(1)
# temporal smoothing
kernel_size = 3
differences = F.pad(
differences.unsqueeze(0).unsqueeze(0),
((kernel_size - 1) // 2, (kernel_size - 1) // 2),
mode="replicate",
)
differences = F.avg_pool1d(
differences, kernel_size, stride=1
).squeeze()
for i in range(min_scene_length, num_frames - min_scene_length):
pbar.update(1)
if i - last_cut >= min_scene_length and differences[i] > threshold:
cuts.append(i)
last_cut = i
return cuts
def detect_histogram_cuts(
self,
frames: torch.Tensor,
threshold: float,
bins: int,
min_scene_length: int,
) -> list[int]:
"""Histogram-based cut detection"""
num_frames = frames.shape[0]
# device = frames.device
cuts = [0]
last_cut = 0
pbar = comfy.utils.ProgressBar(num_frames)
for i in range(1, num_frames):
pbar.update(1)
if i - last_cut < min_scene_length:
continue
# Convert to YUV and get Y channel
yuv1 = (
0.299 * frames[i - 1, ..., 0]
+ 0.587 * frames[i - 1, ..., 1]
+ 0.114 * frames[i - 1, ..., 2]
)
yuv2 = (
0.299 * frames[i, ..., 0]
+ 0.587 * frames[i, ..., 1]
+ 0.114 * frames[i, ..., 2]
)
# Compute histograms
hist1 = torch.histc(yuv1, bins=bins, min=0, max=1)
hist2 = torch.histc(yuv2, bins=bins, min=0, max=1)
# Normalize histograms
hist1 = hist1 / hist1.sum()
hist2 = hist2 / hist2.sum()
# Compute histogram difference
diff = torch.sum(torch.abs(hist1 - hist2))
if diff > threshold:
cuts.append(i)
last_cut = i
return cuts
def detect_hash_cuts(
self,
frames: torch.Tensor,
threshold: float,
hash_size: int,
min_scene_length: int,
) -> list[int]:
"""Perceptual hash based cut detection"""
num_frames = frames.shape[0]
# device = frames.device
cuts = [0]
last_cut = 0
pbar = comfy.utils.ProgressBar(num_frames)
def compute_frame_hash(frame):
# Convert to grayscale
gray = (
0.299 * frame[..., 0]
+ 0.587 * frame[..., 1]
+ 0.114 * frame[..., 2]
)
gray = F.interpolate(
gray.unsqueeze(0).unsqueeze(0),
size=(hash_size, hash_size),
mode="bilinear",
align_corners=False,
).squeeze()
dct = torch.fft.rfft2(gray)
dct = dct[: hash_size // 2, : hash_size // 2]
return dct > dct.median()
for i in range(1, num_frames):
pbar.update(1)
if i - last_cut < min_scene_length:
continue
hash1 = compute_frame_hash(frames[i - 1])
hash2 = compute_frame_hash(frames[i])
diff = torch.mean((hash1 != hash2).float())
if diff > threshold:
cuts.append(i)
last_cut = i
return cuts
def detect_adaptive_cuts(
self,
frames: torch.Tensor,
adaptive_threshold: float,
window_width: int,
min_content_val: float,
min_scene_length: int,
) -> list[int]:
"""Adaptive threshold based cut detection"""
num_frames = frames.shape[0]
device = frames.device
cuts = [0]
last_cut = 0
total = num_frames + max(0, (num_frames - window_width) - window_width)
pbar = comfy.utils.ProgressBar(total)
content_vals = torch.zeros(num_frames - 1, device=device)
for i in range(num_frames - 1):
content_vals[i] = self.compute_content_difference(
frames[i], frames[i + 1]
)
pbar.update(1)
for i in range(window_width, num_frames - window_width):
pbar.update(1)
if i - last_cut < min_scene_length:
continue
target_score = content_vals[i]
window_scores = content_vals[
i - window_width : i + window_width + 1
]
surrounding_scores = torch.cat(
[
window_scores[:window_width],
window_scores[window_width + 1 :],
]
)
average_score = surrounding_scores.mean()
if average_score > 1e-5:
adaptive_ratio = min(target_score / average_score, 255.0)
elif target_score >= min_content_val:
adaptive_ratio = 255.0
else:
adaptive_ratio = 0.0
if (
adaptive_ratio >= adaptive_threshold
and target_score >= min_content_val
):
cuts.append(i)
last_cut = i
return cuts
def compute_content_difference(
self, frame1: torch.Tensor, frame2: torch.Tensor
) -> torch.Tensor:
"""
Computes content difference between frames using multiple metrics:
- Structural similarity
- Color distribution changes
- Edge differences
"""
device = frame1.device
if frame1.dtype != torch.float32:
frame1 = frame1.float()
frame2 = frame2.float()
def ssim(x, y):
c1, c2 = 0.01**2, 0.03**2
mu_x = F.avg_pool2d(x, kernel_size=11, stride=1, padding=5)
mu_y = F.avg_pool2d(y, kernel_size=11, stride=1, padding=5)
mu_x_sq = mu_x.pow(2)
mu_y_sq = mu_y.pow(2)
mu_xy = mu_x * mu_y
sigma_x = (
F.avg_pool2d(x.pow(2), kernel_size=11, stride=1, padding=5)
- mu_x_sq
)
sigma_y = (
F.avg_pool2d(y.pow(2), kernel_size=11, stride=1, padding=5)
- mu_y_sq
)
sigma_xy = (
F.avg_pool2d(x * y, kernel_size=11, stride=1, padding=5)
- mu_xy
)
ssim_map = ((2 * mu_xy + c1) * (2 * sigma_xy + c2)) / (
(mu_x_sq + mu_y_sq + c1) * (sigma_x + sigma_y + c2)
)
return 1 - ssim_map.mean()
def color_change(x, y):
bins = 64
x_hist = torch.stack(
[
torch.histc(x[..., i], bins=bins, min=0, max=1)
for i in range(3)
]
)
y_hist = torch.stack(
[
torch.histc(y[..., i], bins=bins, min=0, max=1)
for i in range(3)
]
)
x_hist = x_hist / x_hist.sum(dim=1, keepdim=True).clamp(min=1e-6)
y_hist = y_hist / y_hist.sum(dim=1, keepdim=True).clamp(min=1e-6)
return torch.mean(torch.abs(x_hist - y_hist))
def edge_change(x, y):
sobel_x = torch.tensor(
[[-1, 0, 1], [-2, 0, 2], [-1, 0, 1]], device=device
).float()
sobel_y = torch.tensor(
[[-1, -2, -1], [0, 0, 0], [1, 2, 1]], device=device
).float()
def detect_edges(img):
gray = (
0.2989 * img[..., 0]
+ 0.5870 * img[..., 1]
+ 0.1140 * img[..., 2]
)
gray = gray.unsqueeze(0).unsqueeze(0)
gx = F.conv2d(gray, sobel_x.view(1, 1, 3, 3), padding=1)
gy = F.conv2d(gray, sobel_y.view(1, 1, 3, 3), padding=1)
return torch.sqrt(gx.pow(2) + gy.pow(2)).squeeze()
edges1 = detect_edges(frame1)
edges2 = detect_edges(frame2)
return torch.mean(torch.abs(edges1 - edges2))
struct_diff = ssim(frame1, frame2)
color_diff = color_change(frame1, frame2)
edge_diff = edge_change(frame1, frame2)
weights = torch.tensor([0.4, 0.3, 0.3], device=device)
combined_diff = (
weights[0] * struct_diff
+ weights[1] * color_diff
+ weights[2] * edge_diff
)
return combined_diff
__nodes__ = [MTB_SceneCutDetector]
+14 -29
View File
@@ -14,7 +14,6 @@ import { app } from '../../scripts/app.js'
import * as shared from './comfy_shared.js' import * as shared from './comfy_shared.js'
import { MtbWidgets } from './mtb_widgets.js' import { MtbWidgets } from './mtb_widgets.js'
import * as mtb_ui from './mtb_ui.js'
// TODO: respect inputs order... // TODO: respect inputs order...
@@ -37,10 +36,12 @@ app.registerExtension({
async beforeRegisterNodeDef(nodeType, nodeData, app) { async beforeRegisterNodeDef(nodeType, nodeData, app) {
if (nodeData.name === 'Debug (mtb)') { if (nodeData.name === 'Debug (mtb)') {
const onNodeCreated = nodeType.prototype.onNodeCreated const onNodeCreated = nodeType.prototype.onNodeCreated
nodeType.prototype.onNodeCreated = function (...args) { nodeType.prototype.onNodeCreated = function () {
this.options = {} this.options = {}
const r = onNodeCreated ? onNodeCreated.apply(this, args) : undefined const r = onNodeCreated
this.addInput('anything_1', '*') ? onNodeCreated.apply(this, arguments)
: undefined
this.addInput(`anything_1`, '*')
return r return r
} }
@@ -80,16 +81,14 @@ app.registerExtension({
} }
const onExecuted = nodeType.prototype.onExecuted const onExecuted = nodeType.prototype.onExecuted
nodeType.prototype.onExecuted = function (...args) { nodeType.prototype.onExecuted = function (data) {
onExecuted?.apply(this, args) onExecuted?.apply(this, arguments)
const [data, ..._rest] = args
const prefix = 'anything_' const prefix = 'anything_'
if (this.widgets) { if (this.widgets) {
for (let i = 0; i < this.widgets.length; i++) { for (let i = 0; i < this.widgets.length; i++) {
if (this.widgets[i].name !== 'output_to_console') { if (this.widgets[i].name !== 'output_to_console') {
this.widgets[i].onRemove?.()
this.widgets[i].onRemoved?.() this.widgets[i].onRemoved?.()
} }
} }
@@ -99,32 +98,19 @@ app.registerExtension({
// console.log(message) // console.log(message)
if (data.text) { if (data.text) {
for (const txt of data.text) { for (const txt of data.text) {
const textDom = mtb_ui.makeElement('p', { fontFamily: 'monospace' }) const w = this.addCustomWidget(
textDom.innerHTML = txt MtbWidgets.DEBUG_STRING(`${prefix}_${widgetI}`, escapeHtml(txt)),
this.addDOMWidget(
`${prefix}_${widgetI}`,
'CUSTOM_TEXT',
textDom,
{},
) )
w.parent = this
widgetI++ widgetI++
} }
} }
if (data.b64_images) { if (data.b64_images) {
for (const img of data.b64_images) { for (const img of data.b64_images) {
const imgDom = mtb_ui.makeElement('img', { width: '100%' }) const w = this.addCustomWidget(
imgDom.src = img MtbWidgets.DEBUG_IMG(`${prefix}_${widgetI}`, img),
this.addDOMWidget(
`${prefix}_${widgetI}`,
'CUSTOM_IMG_B64',
mtb_ui.wrapElement(imgDom, {
overflow: 'hidden',
}),
{},
) )
w.parent = this
widgetI++ widgetI++
} }
} }
@@ -133,13 +119,12 @@ app.registerExtension({
this.onRemoved = function () { this.onRemoved = function () {
// When removing this node we need to remove the input from the DOM // When removing this node we need to remove the input from the DOM
for (const y in this.widgets) { for (let y in this.widgets) {
if (this.widgets[y].canvas) { if (this.widgets[y].canvas) {
this.widgets[y].canvas.remove() this.widgets[y].canvas.remove()
} }
shared.cleanupNode(this) shared.cleanupNode(this)
this.widgets[y].onRemoved?.() this.widgets[y].onRemoved?.()
this.widgets[y].onRemove?.()
} }
} }
} }
-12
View File
@@ -184,18 +184,6 @@ ${inputs}
) )
} }
/**
* Wrap an element with a div
*
* @param {Object} [style] - CSS styles to apply to the element.
* @returns {HTMLElement} - The created DOM element.
*/
export const wrapElement = (element, style = {}) => {
const container = makeElement('div', style)
container.appendChild(element)
return container
}
/** /**
* Creates a DOM element with optional styles, class, and id. * Creates a DOM element with optional styles, class, and id.
* *
+13 -131
View File
@@ -21,7 +21,7 @@ import { infoLogger } from './comfy_shared.js'
import { NumberInputWidget } from './numberInput.js' import { NumberInputWidget } from './numberInput.js'
// NOTE: new widget types registered by MTB Widgets // NOTE: new widget types registered by MTB Widgets
const newTypes = [/*'BOOL'*/ 'COLOR', 'BBOX'] const newTypes = [/*'BOOL'*/ , 'COLOR', 'BBOX']
const deprecated_nodes = { const deprecated_nodes = {
// 'Animation Builder': // 'Animation Builder':
@@ -1283,141 +1283,23 @@ const mtb_widgets = {
}) })
break break
} }
case 'Scene Detect (mtb)': { case 'Save Tensors (mtb)': {
break
}
case 'Loop Start (mtb)': {
const onDrawBackground = nodeType.prototype.onDrawBackground const onDrawBackground = nodeType.prototype.onDrawBackground
nodeType.prototype.onDrawBackground = function (...args) { nodeType.prototype.onDrawBackground = function (ctx, canvas) {
const r = onDrawBackground const r = onDrawBackground
? onDrawBackground.apply(this, args) ? onDrawBackground.apply(this, arguments)
: undefined : undefined
const [ctx, /*canvas,*/ ..._rest] = args // // draw a circle on the top right of the node, with text inside
if (this.flags.collapsed) return r // ctx.fillStyle = "#fff";
if (!this.computed_flow) { // ctx.beginPath();
const related = new Set([this.id]) // ctx.arc(this.size[0] - this.node_width * 0.5, this.size[1] - this.node_height * 0.5, this.node_width * 0.5, 0, Math.PI * 2);
const visited = new Set() // ctx.fill();
if (this.outputs[0].links) {
const initLink = this.outputs[0].links[0]
const { to: loopEnd } = shared.nodesFromLink(this, initLink)
const canReachEnd = (node, visited = new Set()) => {
if (node === loopEnd) return true
if (visited.has(node.id)) return false
visited.add(node.id)
for (const output of node.outputs || []) {
if (!output.links) continue
for (const linkId of output.links) {
const { to: nextNode } = shared.nodesFromLink(node, linkId)
if (!nextNode) continue
if (canReachEnd(nextNode, visited)) {
return true
}
}
}
return false
}
const traverseNodes = (node) => {
if (visited.has(node.id)) return
visited.add(node.id)
// can reach the end // ctx.fillStyle = "#000";
if (node !== this && node !== loopEnd && !canReachEnd(node)) { // ctx.textAlign = "center";
return // ctx.font = "bold 12px Arial";
} // ctx.fillText("Save Tensors", this.size[0] - this.node_width * 0.5, this.size[1] - this.node_height * 0.5);
related.add(node.id)
for (const output of node.outputs || []) {
if (!output.links) continue
for (const linkId of output.links) {
const { to: nextNode } = shared.nodesFromLink(node, linkId)
if (!nextNode) continue
traverseNodes(nextNode)
}
}
}
traverseNodes(this)
}
this.related_to_flow = Array.from(related)
this.computed_flow = true
}
if (this.related_to_flow) {
ctx.save()
const points = []
const padding = 20
const graph = this.graph
const offset = this._pos
for (const nodeId of this.related_to_flow) {
const node = graph.getNodeById(nodeId)
if (!node) continue
const scale = 1.0
const x = node._pos[0] * scale - offset[0]
const y = node._pos[1] * scale - offset[1]
const width = node.size[0] * scale
const height = node.size[1] * scale
const scaledPadding = padding * scale
// console.log({ main: this, x, y, width, height })
points.push(
[x - scaledPadding, y - scaledPadding],
[x + width + scaledPadding, y - scaledPadding],
[x + width + scaledPadding, y + height + scaledPadding],
[x - scaledPadding, y + height + scaledPadding],
)
}
// console.log({ points })
const hull = shared.getConvexHull(points)
ctx.beginPath()
ctx.moveTo(hull[0][0], hull[0][1])
for (let i = 1; i < hull.length; i++) {
ctx.lineTo(hull[i][0], hull[i][1])
}
ctx.closePath()
ctx.fillStyle = 'rgba(255, 0, 0, 0.1)'
ctx.strokeStyle = 'rgba(255, 0, 0, 0.5)'
ctx.lineWidth = 2
ctx.fill()
ctx.stroke()
ctx.restore()
} else {
ctx.save()
ctx.fillStyle = 'red'
ctx.fillRect(-50, -50, this.size[0] + 100, this.size[1] + 100)
ctx.fillStyle = 'white'
ctx.font = 'bold 12px Arial'
ctx.fillText(
`pos: ${this.x}x${this.y}`,
this.size[0] / 2,
this.size[1],
)
ctx.fillText(
`size:${this._posSize}`,
this.size[0] / 2,
this.size[1] - 30,
)
ctx.fillText(
`dpi: ${window.devicePixelRatio}`,
this.size[0] / 2,
this.size[1] - 60,
)
ctx.fillText(
`next: ${graph.getNodeById(this.related_to_flow[1])._posSize}`,
this.size[0] / 2,
this.size[1] - 90,
)
ctx.restore()
}
return r return r
} }
break break