From 8734d593660e30a1c6ef781a97ee11e126b4ae22 Mon Sep 17 00:00:00 2001 From: bruefire Date: Sun, 31 Mar 2024 10:38:43 +0900 Subject: [PATCH] added new logics for masking/sketching with the magic wand. --- js/MagicWand.js | 49 ++++-- js/MaskEditorForVframes.js | 309 +++++++++++++++++++++---------------- 2 files changed, 211 insertions(+), 147 deletions(-) diff --git a/js/MagicWand.js b/js/MagicWand.js index c4f24d2..c61f196 100644 --- a/js/MagicWand.js +++ b/js/MagicWand.js @@ -6,11 +6,11 @@ class MagicWand { const rDiff = Math.abs(color1.r - color2.r); const gDiff = Math.abs(color1.g - color2.g); const bDiff = Math.abs(color1.b - color2.b); - return (rDiff + gDiff + bDiff) / 3; // simple average calculation + return (rDiff + gDiff + bDiff) / 3.0; // simple average calculation } static magicWand(canvas, startX, startY, threshold) { - const ctx = canvas.getContext('2d'); + const ctx = canvas.getContext('2d', {willReadFrequently:true}); const imgData = ctx.getImageData(0, 0, canvas.width, canvas.height); const pixels = imgData.data; const startIdx = (startY * canvas.width + startX) * 4; @@ -20,14 +20,20 @@ class MagicWand { b: pixels[startIdx + 2] }; let selectedPixels = new Set([`${startX},${startY}`]); + let visitedPixels = new Set([`${startX},${startY}`]); + let targetPixels = [[startX, startY]]; + + while (targetPixels.length != 0) { + let coords = targetPixels.shift(); + const x = coords[0]; + const y = coords[1]; - function selectSimilarPixels(x, y) { [[x-1, y], [x+1, y], [x, y-1], [x, y+1]].forEach(([nx, ny]) => { if (nx < 0 || ny < 0 || nx >= canvas.width || ny >= canvas.height) return; const key = `${nx},${ny}`; - if (selectedPixels.has(key)) + if (visitedPixels.has(key)) return; const idx = (ny * canvas.width + nx) * 4; @@ -36,19 +42,21 @@ class MagicWand { g: pixels[idx + 1], b: pixels[idx + 2] }; - if (colorDiff(startColor, pixelColor) <= threshold) { + + visitedPixels.add(key); + if (MagicWand.colorDiff(startColor, pixelColor) <= threshold) { selectedPixels.add(key); - selectSimilarPixels(nx, ny); + targetPixels.push([nx, ny]); } }); } - selectSimilarPixels(startX, startY); return Array.from(selectedPixels).map(p => p.split(',').map(Number)); } - execute(startX, startY, threshold) { - let selectedIndices = MagicWand.magicWand(this.targetCanvas, startX, startY, threshold); + execute(startX, startY, deviation, overrivedCvs) { + let selectedIndices = MagicWand.magicWand( + this.targetCanvas, Math.floor(startX), Math.floor(startY), this.threshold + deviation); let newMaskCvs = document.createElement('canvas'); let newMaskCtx = newMaskCvs.getContext('2d'); @@ -59,21 +67,27 @@ class MagicWand { const newMaskData = newMaskCtx.getImageData(0, 0, newMaskCvs.width, newMaskCvs.height); selectedIndices.forEach(coords => { const index = (coords[1] * newMaskCvs.width + coords[0]) * 4; - newMaskData.data[index + 0] = this.maskingColor[0]; - newMaskData.data[index + 1] = this.maskingColor[1]; - newMaskData.data[index + 2] = this.maskingColor[2]; + newMaskData.data[index + 0] = parseInt(this.maskingColor.substr(1,2), 16); + newMaskData.data[index + 1] = parseInt(this.maskingColor.substr(3,2), 16); + newMaskData.data[index + 2] = parseInt(this.maskingColor.substr(5,2), 16); + newMaskData.data[index + 3] = 255; }); newMaskCtx.putImageData(newMaskData, 0, 0); + if (overrivedCvs) { + overrivedCvs.getContext('2d').clearRect(0, 0, newMaskCvs.width, newMaskCvs.height); + overrivedCvs.getContext('2d').drawImage(newMaskCvs, 0, 0); + } + return newMaskCvs; } - constructor(imageCanvas, maskCanvas, sketchCanvas, color) { + constructor(imageCanvas, maskCanvas, sketchCanvas, colorStr, threshold) { this.targetCanvas = document.createElement('canvas'); let trgCvs = this.targetCanvas; - trgCvs.width = imageCanvas.width; - trgCvs.height = imageCanvas.height; - trgCvs.getContext('2d').drawImage(imageCanvas, 0, 0, trgCvs.width, trgCvs.height); + trgCvs.width = maskCanvas.width; + trgCvs.height = maskCanvas.height; + trgCvs.getContext('2d').drawImage(imageCanvas, 0, 0); trgCvs.getContext('2d').drawImage(sketchCanvas, 0, 0, trgCvs.width, trgCvs.height); this.defMaskCanvas = document.createElement('canvas'); @@ -82,7 +96,8 @@ class MagicWand { dmCvs.height = maskCanvas.height; dmCvs.getContext('2d').drawImage(maskCanvas, 0, 0, dmCvs.width, dmCvs.height); - this.maskingColor = color; + this.maskingColor = colorStr; + this.threshold = threshold; } } diff --git a/js/MaskEditorForVframes.js b/js/MaskEditorForVframes.js index 3d274be..08e3c91 100644 --- a/js/MaskEditorForVframes.js +++ b/js/MaskEditorForVframes.js @@ -2,7 +2,7 @@ import { app } from "../../scripts/app.js"; import { ComfyDialog, $el } from "../../scripts/ui.js"; import { ComfyApp } from "../../scripts/app.js"; import { api } from "../../scripts/api.js" -import { MagicWand } from "./MagicWand.js"; +import MagicWand from "./MagicWand.js"; //import { ClipspaceDialog } from "./clipspace.js"; // Helper function to convert a data URL to a Blob object @@ -292,6 +292,10 @@ class MaskEditorDialog extends ComfyDialog { () => { this.unstackWorkCanvas(); }); + var fillModeButton = this.createLeftButton("✨", + (ev) => { + this.changeFillMode(ev.target); + }); var prevButton = this.createLeftButton("<", () => this.moveToPrev()); var nextButton = this.createLeftButton(">", () => this.moveToNext()); var cancelButton = this.createRightButton("Cancel", () => { @@ -312,6 +316,7 @@ class MaskEditorDialog extends ComfyDialog { this.element.appendChild(placeholder); // must below z-index than bottom_panel to avoid covering button this.element.appendChild(bottom_panel); + top_panel.appendChild(fillModeButton); top_panel.appendChild(modeButton); top_panel.appendChild(colorPicker); top_right_sub_panel.appendChild(this.frameNumberText); @@ -370,6 +375,7 @@ class MaskEditorDialog extends ComfyDialog { this.is_layout_created = true; this.is_sketch = false; + this.is_magicWand = false; // replacement of onClose hook since close is not real close const self = this; @@ -565,6 +571,8 @@ class MaskEditorDialog extends ComfyDialog { self.moveToPrev(); } else if (event.key == 'ArrowRight') { self.moveToNext(); + } else if (event.ctrlKey && (event.key == 'z' || event.key == 'Z')) { + self.unstackWorkCanvas(); } self.updateBrushPreview(self); @@ -606,151 +614,180 @@ class MaskEditorDialog extends ComfyDialog { self.updateBrushPreview(self); - if (window.TouchEvent && event instanceof TouchEvent || event.buttons == 1) { - var diff = performance.now() - self.lasttime; + const maskRect = self.maskCanvas.getBoundingClientRect(); + const x = event.offsetX || event.targetTouches[0].clientX - maskRect.left; + const y = event.offsetY || event.targetTouches[0].clientY - maskRect.top; - const maskRect = self.maskCanvas.getBoundingClientRect(); + if (this.is_magicWand) { + if (window.TouchEvent && event instanceof TouchEvent || event.buttons == 1) { + var diff = performance.now() - self.lasttime; - var x = event.offsetX; - var y = event.offsetY - - if(event.offsetX == null) { - x = event.targetTouches[0].clientX - maskRect.left; - } - - if(event.offsetY == null) { - y = event.targetTouches[0].clientY - maskRect.top; - } - - var brush_size = this.brush_size; - if(event instanceof PointerEvent && event.pointerType == 'pen') { - brush_size *= event.pressure; - this.last_pressure = event.pressure; - } - else if(window.TouchEvent && event instanceof TouchEvent && diff < 20){ - // The firing interval of PointerEvents in Pen is unreliable, so it is supplemented by TouchEvents. - brush_size *= this.last_pressure; - } - else { - brush_size = this.brush_size; - } - - if(diff > 20 && !this.drawing_mode) - requestAnimationFrame(() => { - self.maskCtx.beginPath(); - self.maskCtx.fillStyle = this.getActiveBrushColor(); - self.maskCtx.globalCompositeOperation = "source-over"; - self.maskCtx.arc(x, y, brush_size, 0, Math.PI * 2, false); - self.maskCtx.fill(); - self.lastx = x; - self.lasty = y; - }); - else - requestAnimationFrame(() => { - self.maskCtx.beginPath(); - self.maskCtx.fillStyle = this.getActiveBrushColor(); - self.maskCtx.globalCompositeOperation = "source-over"; - - var dx = x - self.lastx; - var dy = y - self.lasty; - - var distance = Math.sqrt(dx * dx + dy * dy); - var directionX = dx / distance; - var directionY = dy / distance; - - for (var i = 0; i < distance; i+=5) { - var px = self.lastx + (directionX * i); - var py = self.lasty + (directionY * i); - self.maskCtx.arc(px, py, brush_size, 0, Math.PI * 2, false); - self.maskCtx.fill(); - } - self.lastx = x; - self.lasty = y; - }); - - self.lasttime = performance.now(); - } - else if(event.buttons == 2 || event.buttons == 5 || event.buttons == 32) { - const maskRect = self.maskCanvas.getBoundingClientRect(); - const x = event.offsetX || event.targetTouches[0].clientX - maskRect.left; - const y = event.offsetY || event.targetTouches[0].clientY - maskRect.top; - - var brush_size = this.brush_size; - if(event instanceof PointerEvent && event.pointerType == 'pen') { - brush_size *= event.pressure; - this.last_pressure = event.pressure; - } - else if(window.TouchEvent && event instanceof TouchEvent && diff < 20){ - brush_size *= this.last_pressure; - } - else { - brush_size = this.brush_size; - } - - if(diff > 20 && !drawing_mode) // cannot tracking drawing_mode for touch event - requestAnimationFrame(() => { - self.maskCtx.beginPath(); - self.maskCtx.globalCompositeOperation = "destination-out"; - self.maskCtx.arc(x, y, brush_size, 0, Math.PI * 2, false); - self.maskCtx.fill(); - self.lastx = x; - self.lasty = y; - }); - else - requestAnimationFrame(() => { - self.maskCtx.beginPath(); - self.maskCtx.globalCompositeOperation = "destination-out"; + if(diff > 20 && !this.drawing_mode) { + } else { var dx = x - self.lastx; var dy = y - self.lasty; + var deviation = Math.sqrt(dx * dx + dy * dy) * Math.sign(dx * dy); - var distance = Math.sqrt(dx * dx + dy * dy); - var directionX = dx / distance; - var directionY = dy / distance; + this.magicWandInst?.execute(self.lastx, self.lasty, deviation, this.maskCanvas); + } + } - for (var i = 0; i < distance; i+=5) { - var px = self.lastx + (directionX * i); - var py = self.lasty + (directionY * i); - self.maskCtx.arc(px, py, brush_size, 0, Math.PI * 2, false); + } else { + if (window.TouchEvent && event instanceof TouchEvent || event.buttons == 1) { + var diff = performance.now() - self.lasttime; + + var brush_size = this.brush_size; + if(event instanceof PointerEvent && event.pointerType == 'pen') { + brush_size *= event.pressure; + this.last_pressure = event.pressure; + } + else if(window.TouchEvent && event instanceof TouchEvent && diff < 20){ + // The firing interval of PointerEvents in Pen is unreliable, so it is supplemented by TouchEvents. + brush_size *= this.last_pressure; + } + else { + brush_size = this.brush_size; + } + + if(diff > 20 && !this.drawing_mode) + requestAnimationFrame(() => { + self.maskCtx.beginPath(); + self.maskCtx.fillStyle = this.getActiveBrushColor(); + self.maskCtx.globalCompositeOperation = "source-over"; + self.maskCtx.arc(x, y, brush_size, 0, Math.PI * 2, false); self.maskCtx.fill(); - } - self.lastx = x; - self.lasty = y; - }); - + self.lastx = x; + self.lasty = y; + }); + else + requestAnimationFrame(() => { + self.maskCtx.beginPath(); + self.maskCtx.fillStyle = this.getActiveBrushColor(); + self.maskCtx.globalCompositeOperation = "source-over"; + + var dx = x - self.lastx; + var dy = y - self.lasty; + + var distance = Math.sqrt(dx * dx + dy * dy); + var directionX = dx / distance; + var directionY = dy / distance; + + for (var i = 0; i < distance; i+=5) { + var px = self.lastx + (directionX * i); + var py = self.lasty + (directionY * i); + self.maskCtx.arc(px, py, brush_size, 0, Math.PI * 2, false); + self.maskCtx.fill(); + } + self.lastx = x; + self.lasty = y; + }); + self.lasttime = performance.now(); + } + else if(event.buttons == 2 || event.buttons == 5 || event.buttons == 32) { + + var brush_size = this.brush_size; + if(event instanceof PointerEvent && event.pointerType == 'pen') { + brush_size *= event.pressure; + this.last_pressure = event.pressure; + } + else if(window.TouchEvent && event instanceof TouchEvent && diff < 20){ + brush_size *= this.last_pressure; + } + else { + brush_size = this.brush_size; + } + + if(diff > 20 && !drawing_mode) // cannot tracking drawing_mode for touch event + requestAnimationFrame(() => { + self.maskCtx.beginPath(); + self.maskCtx.globalCompositeOperation = "destination-out"; + self.maskCtx.arc(x, y, brush_size, 0, Math.PI * 2, false); + self.maskCtx.fill(); + self.lastx = x; + self.lasty = y; + }); + else + requestAnimationFrame(() => { + self.maskCtx.beginPath(); + self.maskCtx.globalCompositeOperation = "destination-out"; + + var dx = x - self.lastx; + var dy = y - self.lasty; + + var distance = Math.sqrt(dx * dx + dy * dy); + var directionX = dx / distance; + var directionY = dy / distance; + + for (var i = 0; i < distance; i+=5) { + var px = self.lastx + (directionX * i); + var py = self.lasty + (directionY * i); + self.maskCtx.arc(px, py, brush_size, 0, Math.PI * 2, false); + self.maskCtx.fill(); + } + self.lastx = x; + self.lasty = y; + }); + + self.lasttime = performance.now(); + } } } handlePointerDown(self, event) { - var brush_size = this.brush_size; - if(event instanceof PointerEvent && event.pointerType == 'pen') { - brush_size *= event.pressure; - this.last_pressure = event.pressure; - } + this.stackWorkCanvas(); - if ([0, 2, 5].includes(event.button)) { - self.drawing_mode = true; - - this.stackWorkCanvas(); + if (this.is_magicWand) { + if ([0, 2, 5].includes(event.button)) { + self.drawing_mode = true; + + event.preventDefault(); + const maskRect = self.maskCanvas.getBoundingClientRect(); + const x = event.offsetX || event.targetTouches[0].clientX - maskRect.left; + const y = event.offsetY || event.targetTouches[0].clientY - maskRect.top; - event.preventDefault(); - const maskRect = self.maskCanvas.getBoundingClientRect(); - const x = event.offsetX || event.targetTouches[0].clientX - maskRect.left; - const y = event.offsetY || event.targetTouches[0].clientY - maskRect.top; + this.magicWandInst = new MagicWand( + this.imgCanvas, + this.maskCanvas, + this.sketchCanvas, + this.getActiveBrushColor(), + 16); + this.magicWandInst.execute(x, y, 0, this.maskCanvas); + + self.lastx = x; + self.lasty = y; + self.lasttime = performance.now(); + } - self.maskCtx.beginPath(); - if (event.button == 0) { - self.maskCtx.fillStyle = this.getActiveBrushColor(); - self.maskCtx.globalCompositeOperation = "source-over"; - } else { - self.maskCtx.globalCompositeOperation = "destination-out"; + } else { + var brush_size = this.brush_size; + if(event instanceof PointerEvent && event.pointerType == 'pen') { + brush_size *= event.pressure; + this.last_pressure = event.pressure; + } + + if ([0, 2, 5].includes(event.button)) { + self.drawing_mode = true; + + event.preventDefault(); + const maskRect = self.maskCanvas.getBoundingClientRect(); + const x = event.offsetX || event.targetTouches[0].clientX - maskRect.left; + const y = event.offsetY || event.targetTouches[0].clientY - maskRect.top; + + self.maskCtx.beginPath(); + if (event.button == 0) { + self.maskCtx.fillStyle = this.getActiveBrushColor(); + self.maskCtx.globalCompositeOperation = "source-over"; + } else { + self.maskCtx.globalCompositeOperation = "destination-out"; + } + self.maskCtx.arc(x, y, brush_size, 0, Math.PI * 2, false); + self.maskCtx.fill(); + self.lastx = x; + self.lasty = y; + self.lasttime = performance.now(); } - self.maskCtx.arc(x, y, brush_size, 0, Math.PI * 2, false); - self.maskCtx.fill(); - self.lastx = x; - self.lasty = y; - self.lasttime = performance.now(); } } @@ -769,11 +806,23 @@ class MaskEditorDialog extends ComfyDialog { if (this.workCanvases.length == 0) return; + this.maskCtx.globalCompositeOperation = "source-over"; let lastCanvas = this.workCanvases.pop(); this.maskCanvas.getContext('2d').clearRect(0, 0, lastCanvas.width, lastCanvas.height); this.maskCanvas.getContext('2d').drawImage(lastCanvas, 0, 0); } + changeFillMode(button) { + if (this.is_magicWand) { + button.innerText = "✨"; + + } else { + button.innerText = "🖌"; + } + + this.is_magicWand = !this.is_magicWand; + } + getBackCanvasForCurrentMode(index) { return this.is_sketch ? this.backSketchCanvases[index] @@ -789,7 +838,7 @@ class MaskEditorDialog extends ComfyDialog { getActiveBrushColor() { return this.is_sketch ? this.colorPicker.value - : "rgb(255, 255, 255)"; + : "#ffffff"; } prepareSketchLayer() {