From 77dd7dbb91bf67b8238d4c858a9753acb108b55b Mon Sep 17 00:00:00 2001 From: Tung Nguyen Date: Thu, 30 Nov 2023 10:35:23 +0700 Subject: [PATCH] improve preview code --- web/js/vid_preview.js | 449 +++++++++++++++++++++++++++--------------- web/js/vid_upload.js | 267 +++++++++---------------- 2 files changed, 392 insertions(+), 324 deletions(-) diff --git a/web/js/vid_preview.js b/web/js/vid_preview.js index 7b37aaa..58a90b8 100644 --- a/web/js/vid_preview.js +++ b/web/js/vid_preview.js @@ -1,171 +1,310 @@ import { app } from "../../../scripts/app.js"; import { api } from "../../../scripts/api.js"; -function offsetDOMWidget(widget, ctx, node, widgetWidth, widgetY, height) { - const margin = 10; - const elRect = ctx.canvas.getBoundingClientRect(); - const transform = new DOMMatrix() - .scaleSelf( - elRect.width / ctx.canvas.width, - elRect.height / ctx.canvas.height - ) - .multiplySelf(ctx.getTransform()) - .translateSelf(0, widgetY + margin); - - const scale = new DOMMatrix().scaleSelf(transform.a, transform.d); - Object.assign(widget.inputEl.style, { - transformOrigin: "0 0", - transform: scale, - left: `${transform.e}px`, - top: `${transform.d + transform.f}px`, - width: `${widgetWidth}px`, - height: `${(height || widget.parent?.inputHeight || 32) - margin}px`, - position: "absolute", - background: !node.color ? "" : node.color, - color: !node.color ? "" : "white", - zIndex: 5, //app.graph._nodes.indexOf(node), - }); -} - -export const hasWidgets = (node) => { - if (!node.widgets || !node.widgets?.[Symbol.iterator]) { - return false; - } - return true; -}; - -export const cleanupNode = (node) => { - if (!hasWidgets(node)) { +export const chainCallback = (object, property, callback) => { + if (object == undefined) { + //This should not happen. + console.error("Tried to add callback to non-existant object"); return; } - - for (const w of node.widgets) { - if (w.canvas) { - w.canvas.remove(); - } - if (w.inputEl) { - w.inputEl.remove(); - } - // calls the widget remove callback - w.onRemoved?.(); - } -}; - -export const CreatePreviewElement = (name, val, format, callback) => { - const [type] = format.split("/"); - - const w = { - name, - type, - value: val, - draw: function (ctx, node, widgetWidth, widgetY, height) { - const [cw, ch] = this.computeSize(widgetWidth); - offsetDOMWidget(this, ctx, node, widgetWidth, widgetY, ch); - }, - computeSize: function (_) { - const ratio = this.inputRatio || 1; - const width = Math.max(220, this.parent.size[0]); - return [width, width / ratio + 10]; - }, - onRemoved: function () { - if (this.inputEl) { - this.inputEl.remove(); - } - }, - }; - - w.inputEl = document.createElement(type === "video" ? "video" : "img"); - w.inputEl.src = w.value; - if (type === "video") { - w.inputEl.setAttribute("type", "video/webm"); - w.inputEl.autoplay = true; - w.inputEl.loop = true; - w.inputEl.controls = false; - w.inputEl.addEventListener("loadedmetadata", function (e) { - w.inputRatio = this.videoWidth / this.videoHeight; - callback?.(); - }, false); + if (property in object) { + const callback_orig = object[property]; + object[property] = function () { + const r = callback_orig.apply(this, arguments); + callback.apply(this, arguments); + return r; + }; } else { - w.inputEl.onload = function () { - w.inputRatio = w.inputEl.naturalWidth / w.inputEl.naturalHeight; - callback?.(); - }; + object[property] = callback; } - document.body.appendChild(w.inputEl); - return w; }; -const videoPreview = { +export const formatUploadedUrl = (params) => { + if (params.url) { + return params.url; + } + + params = { ...params }; + + if (!params.filename && params.name) { + params.filename = params.name; + delete params.name; + } + + return api.apiURL("/view?" + new URLSearchParams(params)); +}; + +export const fitHeight = (node) => { + node.setSize([ + node.size[0], + node.computeSize([node.size[0], node.size[1]])[1], + ]); + node.graph.setDirtyCanvas(true); +}; + +export const addVideoPreview = (nodeType) => { + chainCallback(nodeType.prototype, "onNodeCreated", function () { + let previewNode = this; + //preview is a made up widget type to enable user defined functions + //videopreview is widget name + //The previous implementation used type to distinguish between a video and gif, + //but the type is not serialized and would not survive a reload + var previewWidget = { + name: "videopreview", + type: "preview", + value: "", + draw: function (ctx, node, widgetWidth, widgetY, height) { + //update widget position, hide if off-screen + const transform = ctx.getTransform(); + const scale = app.canvas.ds.scale; //gets the litegraph zoom + //calculate coordinates with account for browser zoom + const x = (transform.e * scale) / transform.a; + const y = (transform.f * scale) / transform.a; + Object.assign(this.parentEl.style, { + left: x + 15 * scale + "px", + top: y + widgetY * scale + "px", + width: (widgetWidth - 30) * scale + "px", + zIndex: 2 + (node.is_selected ? 1 : 0), + position: "absolute", + }); + this._boundingCount = 0; + }, + computeSize: function (width) { + if (this.aspectRatio && !this.parentEl.hidden) { + let height = (previewNode.size[0] - 30) / this.aspectRatio; + if (!(height > 0)) { + height = 0; + } + return [width, height]; + } + return [width, -4]; //no loaded src, widget should not display + }, + _value: { hidden: false, paused: false }, + }; + //onRemoved isn't a litegraph supported function on widgets + //Given that onremoved widget and node callbacks are sparse, this + //saves the required iteration. + chainCallback(this, "onRemoved", () => { + previewWidget?.parentEl?.remove(); + }); + this.addCustomWidget(previewWidget); + previewWidget.parentEl = document.createElement("div"); + previewWidget.parentEl.className = "animatediff_preview"; + previewWidget.parentEl.style["pointer-events"] = "none"; + + previewWidget.videoEl = document.createElement("video"); + previewWidget.videoEl.controls = false; + previewWidget.videoEl.loop = true; + previewWidget.videoEl.muted = true; + previewWidget.videoEl.style["width"] = "100%"; + previewWidget.videoEl.addEventListener("loadedmetadata", () => { + previewWidget.aspectRatio = + previewWidget.videoEl.videoWidth / previewWidget.videoEl.videoHeight; + fitHeight(this); + }); + previewWidget.videoEl.addEventListener("error", () => { + //TODO: consider a way to properly notify the user why a preview isn't shown. + previewWidget.parentEl.hidden = true; + fitHeight(this); + }); + + previewWidget.imgEl = document.createElement("img"); + previewWidget.imgEl.style["width"] = "100%"; + previewWidget.imgEl.hidden = true; + previewWidget.imgEl.onload = () => { + previewWidget.aspectRatio = + previewWidget.imgEl.naturalWidth / previewWidget.imgEl.naturalHeight; + fitHeight(this); + }; + + this.setPreviewsrc = (params) => { + previewWidget._value.params = params; + this._setPreviewsrc(params); + }; + this._setPreviewsrc = function (params) { + if (params == undefined) { + return; + } + previewWidget.parentEl.hidden = previewWidget._value.hidden; + if (params?.format?.split("/")[0] == "video") { + previewWidget.videoEl.autoplay = + !previewWidget._value.paused && !previewWidget._value.hidden; + previewWidget.videoEl.src = formatUploadedUrl(params); + previewWidget.videoEl.hidden = false; + previewWidget.imgEl.hidden = true; + } else { + // Is animated image + previewWidget.imgEl.src = formatUploadedUrl(params); + previewWidget.videoEl.hidden = true; + previewWidget.imgEl.hidden = false; + } + }; + Object.defineProperty(previewWidget, "value", { + set: (value) => { + if (value) { + previewWidget._value = value; + this._setPreviewsrc(value.params); + } + }, + get: () => { + return previewWidget._value; + }, + }); + //Hide video element if offscreen + //The multiline input implementation moves offscreen every frame + //and doesn't apply until a node with an actual inputEl is loaded + this._boundingCount = 0; + this.onBounding = function () { + if (this._boundingCount++ > 5) { + previewWidget.parentEl.style.left = "-8000px"; + } + }; + previewWidget.parentEl.appendChild(previewWidget.videoEl); + previewWidget.parentEl.appendChild(previewWidget.imgEl); + document.body.appendChild(previewWidget.parentEl); + }); +}; + +export function addVideoPreviewOptions(nodeType) { + chainCallback( + nodeType.prototype, + "getExtraMenuOptions", + function (_, options) { + // The intended way of appending options is returning a list of extra options, + // but this isn't used in widgetInputs.js and would require + // less generalization of chainCallback + let optNew = []; + const previewWidget = this.widgets.find((w) => w.name === "videopreview"); + + let url = null; + if (previewWidget.videoEl?.hidden == false && previewWidget.videoEl.src) { + url = previewWidget.videoEl.src; + } else if ( + previewWidget.imgEl?.hidden == false && + previewWidget.imgEl.src + ) { + url = previewWidget.imgEl.src; + } + if (url) { + url = new URL(url); + //placeholder from Save Image, will matter once preview functionality is implemented + //url.searchParams.delete('preview') + optNew.push( + { + content: "Open preview", + callback: () => { + window.open(url, "_blank"); + }, + }, + { + content: "Save preview", + callback: () => { + const a = document.createElement("a"); + a.href = url; + a.setAttribute( + "download", + new URLSearchParams(url.search).get("filename") + ); + document.body.append(a); + a.click(); + requestAnimationFrame(() => a.remove()); + }, + } + ); + } + const PauseDesc = + (previewWidget._value.paused ? "Resume" : "Pause") + " preview"; + if (previewWidget.videoEl.hidden == false) { + optNew.push({ + content: PauseDesc, + callback: () => { + //animated images can't be paused and are more likely to cause performance issues. + //changing src to a single keyframe is possible, + //For now, the option is disabled if an animated image is being displayed + if (previewWidget._value.paused) { + previewWidget.videoEl?.play(); + } else { + previewWidget.videoEl?.pause(); + } + previewWidget._value.paused = !previewWidget._value.paused; + }, + }); + } + //TODO: Consider hiding elements if video no preview is available yet. + //It would reduce confusion at the cost of functionality + //(if a video preview lags the computer, the user should be able to hide in advance) + const visDesc = + (previewWidget._value.hidden ? "Show" : "Hide") + " preview"; + optNew.push({ + content: visDesc, + callback: () => { + if (!previewWidget.videoEl.hidden && !previewWidget._value.hidden) { + previewWidget.videoEl.pause(); + } else if ( + previewWidget._value.hidden && + !previewWidget.videoEl.hidden && + !previewWidget._value.paused + ) { + previewWidget.videoEl.play(); + } + previewWidget._value.hidden = !previewWidget._value.hidden; + previewWidget.parentEl.hidden = previewWidget._value.hidden; + fitHeight(this); + }, + }); + optNew.push({ + content: "Sync preview", + callback: () => { + //TODO: address case where videos have varying length + //Consider a system of sync groups which are opt-in? + for (let p of document.getElementsByClassName("vhs_preview")) { + for (let child of p.children) { + if (child.tagName == "VIDEO") { + child.currentTime = 0; + } else if (child.tagName == "IMG") { + child.src = child.src; + } + } + } + }, + }); + if (options.length > 0 && options[0] != null && optNew.length > 0) { + optNew.push(null); + } + options.unshift(...optNew); + } + ); +} + +app.registerExtension({ name: "AnimateDiff.VideoPreview", - async beforeRegisterNodeDef(nodeType, nodeData, app) { + async beforeRegisterNodeDef(nodeType, nodeData) { if (nodeData.name !== "AnimateDiffCombine") { - return + return; } - const onExecuted = nodeType.prototype.onExecuted; - nodeType.prototype.onExecuted = function (message) { - const r = onExecuted ? onExecuted.apply(this, message) : undefined; - + addVideoPreview(nodeType); + addVideoPreviewOptions(nodeType); + chainCallback(nodeType.prototype, "onNodeCreated", function () { + this._outputs = this.outputs; + Object.defineProperty(this, "outputs", { + set: function (value) { + this._outputs = value; + requestAnimationFrame(() => { + if (app.nodeOutputs[this.id + ""]) { + this.setPreviewsrc(app.nodeOutputs[this.id + ""].videos[0]); + } + }); + }, + get: function () { + return []; + }, + }); + }); + chainCallback(nodeType.prototype, "onExecuted", function (message) { if (message?.videos) { - this.videos = message.videos; + this.setPreviewsrc(message.videos[0]); } - - return r; - }; - - const onDrawBackground = nodeType.prototype.onDrawBackground; - nodeType.prototype.onDrawBackground = function (ctx) { - const r = onDrawBackground ? onDrawBackground.apply(this, arguments) : undefined; - const node = this; - const prefix = "ad_video_preview_"; - - if (node.videos_rendered === node.videos) { - return r; - } - - if (node.widgets) { - const pos = node.widgets.findIndex((w) => w.name === `${prefix}_0`); - if (pos !== -1) { - for (let i = pos; i < node.widgets.length; i++) { - node.widgets[i].onRemoved?.(); - } - node.widgets.length = pos; - } - } - if (node.videos) { - node.videos.forEach((params, i) => { - const previewUrl = api.apiURL( - "/view?" + new URLSearchParams(params).toString() - ); - const w = node.addCustomWidget( - CreatePreviewElement( - `${prefix}_${i}`, - previewUrl, - params.format || "image/gif", - node.computeSizeKeepWidth.bind(node) - ) - ); - w.parent = node; - }); - node.videos_rendered = node.videos; - } - - return r; - }; - - const onRemoved = nodeType.prototype.onRemoved; - nodeType.prototype.onRemoved = function () { - cleanupNode(this); - return onRemoved ? onRemoved.apply(this, arguments) : undefined; - }; - - nodeType.prototype.computeSizeKeepWidth = function () { - this.setSize([ - this.size[0], - this.computeSize([this.size[0], this.size[1]])[1], - ]); - }; + }); }, -}; - -app.registerExtension(videoPreview); +}); diff --git a/web/js/vid_upload.js b/web/js/vid_upload.js index 014a1b0..73150e1 100644 --- a/web/js/vid_upload.js +++ b/web/js/vid_upload.js @@ -1,188 +1,117 @@ import { app } from "../../../scripts/app.js"; import { api } from "../../../scripts/api.js"; -import { ComfyWidgets } from "../../../scripts/widgets.js"; -const supportedVideoTypes = [ - "image/gif", - "video/webm", - "video/mp4", - "video/mov", -]; +import { + chainCallback, + addVideoPreview, + addVideoPreviewOptions, +} from "./vid_preview.js"; -const VIDEOUPLOAD = (node, inputName, inputData, app) => { - const previewWidget = "ad_video_preview"; - const videoWidget = node.widgets.find((w) => w.name === "video"); - let uploadWidget; +async function uploadFile(file) { + try { + // Wrap file in formdata so it includes filename + const body = new FormData(); + const new_file = new File([file], file.name, { + type: file.type, + lastModified: file.lastModified, + }); + body.append("image", new_file); + body.append("subfolder", "video"); + const resp = await api.fetchApi("/upload/image", { + method: "POST", + body, + }); - const showVideo = (name) => { - let folder_separator = name.lastIndexOf("/"); - let subfolder = ""; - if (folder_separator > -1) { - subfolder = name.substring(0, folder_separator); - name = name.substring(folder_separator + 1); - } - const ext = name.substring(name.lastIndexOf(".") + 1); - const format = supportedVideoTypes.find((t) => t.endsWith(ext)); - node.videos = [ - { - filename: name, - type: "input", - subfolder: subfolder, - format, - }, - ]; - }; - - var default_value = videoWidget.value; - Object.defineProperty(videoWidget, "value", { - set: function (value) { - this._real_value = value; - }, - - get: function () { - let value = ""; - if (this._real_value) { - value = this._real_value; - } else { - return default_value; - } - - if (value.filename) { - let real_value = value; - value = ""; - if (real_value.subfolder) { - value = real_value.subfolder + "/"; - } - - value += real_value.filename; - - if (real_value.type && real_value.type !== "input") - value += ` [${real_value.type}]`; - } - return value; - }, - }); - - // Add our own callback to the combo widget to render an image when it changes - const cb = node.callback; - videoWidget.callback = function () { - showVideo(videoWidget.value); - if (cb) { - return cb.apply(this, arguments); - } - }; - - // On load if we have a value then render the image - // The value isnt set immediately so we need to wait a moment - // No change callbacks seem to be fired on initial setting of the value - requestAnimationFrame(() => { - if (videoWidget.value) { - showVideo(videoWidget.value); - } - }); - - async function uploadFile(file, updateNode, pasted = false) { - try { - // Wrap file in formdata so it includes filename - const body = new FormData(); - body.append("image", file); - body.append("subfolder", "video"); - const resp = await api.fetchApi("/upload/image", { - method: "POST", - body, - }); - - if (resp.status === 200) { - const data = await resp.json(); - // Add the file to the dropdown list and update the widget value - let path = data.name; - if (data.subfolder) path = data.subfolder + "/" + path; - - if (!videoWidget.options.values.includes(path)) { - videoWidget.options.values.push(path); - } - - if (updateNode) { - showVideo(path); - videoWidget.value = path; - } - } else { - alert(resp.status + " - " + resp.statusText); - } - } catch (error) { - alert(error); + if (resp.status === 200 || resp.status === 201) { + return resp.json(); + } else { + alert(`Upload failed: ${resp.statusText}`); } + } catch (error) { + alert(`Upload failed: ${error}`); } +} - const fileInput = document.createElement("input"); - Object.assign(fileInput, { - type: "file", - accept: supportedVideoTypes.join(","), - style: "display: none", - onchange: async () => { - if (fileInput.files.length) { - await uploadFile(fileInput.files[0], true); +function addUploadWidget(nodeType, callback) { + chainCallback(nodeType.prototype, "onNodeCreated", function () { + const fileInput = document.createElement("input"); + chainCallback(this, "onRemoved", () => { + fileInput?.remove(); + }); + + Object.assign(fileInput, { + type: "file", + accept: "video/webm,video/mp4,video/mkv,image/gif,image/webp", + style: "display: none", + onchange: async () => { + if (fileInput.files.length) { + const params = await uploadFile(fileInput.files[0]); + if (!params) { + // upload failed and file can not be added to options + return; + } + + fileInput.value = ""; + callback.call(this, params) + } + }, + }); + + document.body.append(fileInput); + let uploadWidget = this.addWidget( + "button", + "choose video to upload", + "image", + () => { + app.canvas.node_widget = null; + fileInput.click(); } - }, + ); + uploadWidget.options.serialize = false; }); - document.body.append(fileInput); - - // Create the button widget for selecting the files - uploadWidget = node.addWidget( - "button", - "choose file to upload", - "image", - () => { - fileInput.click(); - } - ); - uploadWidget.serialize = false; - - // Add handler to check if an image is being dragged over our node - node.onDragOver = function (e) { - if (e.dataTransfer && e.dataTransfer.items) { - const image = [...e.dataTransfer.items].find((f) => f.kind === "file"); - return !!image; - } - - return false; - }; - - // On drop upload files - node.onDragDrop = function (e) { - console.log("onDragDrop called"); - let handled = false; - for (const file of e.dataTransfer.files) { - if (file.type.startsWith("image/")) { - uploadFile(file, !handled); // Dont await these, any order is fine, only update on first one - handled = true; - } - } - - return handled; - }; - - node.pasteFile = function (file) { - if (supportedVideoTypes.indexOf(file.type) > -1) { - const is_pasted = - file.name === "image.png" && file.lastModified - Date.now() < 2000; - uploadFile(file, true, is_pasted); - return true; - } - return false; - }; - - return { widget: uploadWidget }; -}; - -ComfyWidgets["VIDEOUPLOAD"] = VIDEOUPLOAD; +} // Adds an upload button to the nodes app.registerExtension({ name: "AnimateDiff.UploadVideo", async beforeRegisterNodeDef(nodeType, nodeData, app) { if (nodeData?.input?.required?.video?.[1]?.video_upload === true) { - nodeData.input.required.upload = ["VIDEOUPLOAD"]; + addUploadWidget(nodeType, function (params) { + const pathWidget = this.widgets.find((w) => w.name === 'video'); + const filename = [params.subfolder, params.name || params.filename].filter(Boolean).join('/') + pathWidget.value = filename; + pathWidget.options.values.push(filename); + this.setPreviewsrc(params); + }); + addVideoPreview(nodeType); + addVideoPreviewOptions(nodeType); + chainCallback(nodeType.prototype, "onNodeCreated", function () { + const pathWidget = this.widgets.find((w) => w.name === "video"); + pathWidget._value = pathWidget.value; + Object.defineProperty(pathWidget, "value", { + set: (value) => { + pathWidget._value = value; + if (!value) { + return + } + + const parts = value.split("/") + const filename = parts.pop() + const subfolder = parts.join("/") + const extension = filename.split(".").pop(); + let format = "video" + if (["gif", "webp", "avif"].includes(extension)) { + format = "image" + } + this.setPreviewsrc({ filename, subfolder, type: "input", format: format }); + }, + get: () => { + return pathWidget._value; + } + }); + //Set value to ensure preview displays on initial add. + pathWidget.value = pathWidget._value; + }); } }, });