first commit

2024-05-23 11:43:12 +00:00
parent fca2ff3695
commit d50737b7c2
10 changed files with 1676 additions and 0 deletions
@@ -0,0 +1,203 @@
+import { app } from "../../../scripts/app.js";
+import { api } from '../../../scripts/api.js'
+import { ComfyWidgets } from "../../../scripts/widgets.js"
+
+function fitHeight(node) {
+    node.setSize([node.size[0], node.computeSize([node.size[0], node.size[1]])[1]])
+    node?.graph?.setDirtyCanvas(true);
+}
+
+function previewAudio(node,file){
+    while (node.widgets.length > 2){
+        node.widgets.pop();
+    }
+    try {
+        var el = document.getElementById("uploadAudio");
+        el.remove();
+    } catch (error) {
+        console.log(error);
+    }
+    var element = document.createElement("div");
+    element.id = "uploadAudio";
+    const previewNode = node;
+    var previewWidget = node.addDOMWidget("audiopreview", "preview", element, {
+        serialize: false,
+        hideOnZoom: false,
+        getValue() {
+            return element.value;
+        },
+        setValue(v) {
+            element.value = v;
+        },
+    });
+    previewWidget.computeSize = function(width) {
+        if (this.aspectRatio && !this.parentEl.hidden) {
+            let height = (previewNode.size[0]-20)/ this.aspectRatio + 10;
+            if (!(height > 0)) {
+                height = 0;
+            }
+            this.computedHeight = height + 10;
+            return [width, height];
+        }
+        return [width, -4];//no loaded src, widget should not display
+    }
+    // element.style['pointer-events'] = "none"
+    previewWidget.value = {hidden: false, paused: false, params: {}}
+    previewWidget.parentEl = document.createElement("div");
+    previewWidget.parentEl.className = "audio_preview";
+    previewWidget.parentEl.style['width'] = "100%"
+    element.appendChild(previewWidget.parentEl);
+    previewWidget.audioEl = document.createElement("audio");
+    previewWidget.audioEl.controls = true;
+    previewWidget.audioEl.loop = false;
+    previewWidget.audioEl.muted = false;
+    previewWidget.audioEl.style['width'] = "100%"
+    previewWidget.audioEl.addEventListener("loadedmetadata", () => {
+
+        previewWidget.aspectRatio = previewWidget.audioEl.audioWidth / previewWidget.audioEl.audioHeight;
+        fitHeight(this);
+    });
+    previewWidget.audioEl.addEventListener("error", () => {
+        //TODO: consider a way to properly notify the user why a preview isn't shown.
+        previewWidget.parentEl.hidden = true;
+        fitHeight(this);
+    });
+
+    let params =  {
+        "filename": file,
+        "type": "input",
+    }
+    
+    previewWidget.parentEl.hidden = previewWidget.value.hidden;
+    previewWidget.audioEl.autoplay = !previewWidget.value.paused && !previewWidget.value.hidden;
+    let target_width = 256
+    if (element.style?.width) {
+        //overscale to allow scrolling. Endpoint won't return higher than native
+        target_width = element.style.width.slice(0,-2)*2;
+    }
+    if (!params.force_size || params.force_size.includes("?") || params.force_size == "Disabled") {
+        params.force_size = target_width+"x?"
+    } else {
+        let size = params.force_size.split("x")
+        let ar = parseInt(size[0])/parseInt(size[1])
+        params.force_size = target_width+"x"+(target_width/ar)
+    }
+    
+    previewWidget.audioEl.src = api.apiURL('/view?' + new URLSearchParams(params));
+
+    previewWidget.audioEl.hidden = false;
+    previewWidget.parentEl.appendChild(previewWidget.audioEl)
+}
+
+function audioUpload(node, inputName, inputData, app) {
+    const audioWidget = node.widgets.find((w) => w.name === "audio");
+    let uploadWidget;
+    /* 
+    A method that returns the required style for the html 
+    */
+    var default_value = audioWidget.value;
+    Object.defineProperty(audioWidget, "value", {
+        set : function(value) {
+            this._real_value = value;
+        },
+
+        get : function() {
+            let value = "";
+            if (this._real_value) {
+                value = this._real_value;
+            } else {
+                return default_value;
+            }
+
+            if (value.filename) {
+                let real_value = value;
+                value = "";
+                if (real_value.subfolder) {
+                    value = real_value.subfolder + "/";
+                }
+
+                value += real_value.filename;
+
+                if(real_value.type && real_value.type !== "input")
+                    value += ` [${real_value.type}]`;
+            }
+            return value;
+        }
+    });
+    async function uploadFile(file, updateNode, pasted = false) {
+        try {
+            // Wrap file in formdata so it includes filename
+            const body = new FormData();
+            body.append("image", file);
+            if (pasted) body.append("subfolder", "pasted");
+            const resp = await api.fetchApi("/upload/image", {
+                method: "POST",
+                body,
+            });
+
+            if (resp.status === 200) {
+                const data = await resp.json();
+                // Add the file to the dropdown list and update the widget value
+                let path = data.name;
+                if (data.subfolder) path = data.subfolder + "/" + path;
+
+                if (!audioWidget.options.values.includes(path)) {
+                    audioWidget.options.values.push(path);
+                }
+
+                if (updateNode) {
+                    audioWidget.value = path;
+                    previewAudio(node,path)
+                    
+                }
+            } else {
+                alert(resp.status + " - " + resp.statusText);
+            }
+        } catch (error) {
+            alert(error);
+        }
+    }
+
+    const fileInput = document.createElement("input");
+    Object.assign(fileInput, {
+        type: "file",
+        accept: "audio/mp3,audio/wav,audio/flac,audio/m4a",
+        style: "display: none",
+        onchange: async () => {
+            if (fileInput.files.length) {
+                await uploadFile(fileInput.files[0], true);
+            }
+        },
+    });
+    document.body.append(fileInput);
+
+    // Create the button widget for selecting the files
+    uploadWidget = node.addWidget("button", "choose audio file to upload", "Audio", () => {
+        fileInput.click();
+    });
+
+    uploadWidget.serialize = false;
+
+    previewAudio(node, audioWidget.value);
+    const cb = node.callback;
+    audioWidget.callback = function () {
+        previewAudio(node,audioWidget.value);
+        if (cb) {
+            return cb.apply(this, arguments);
+        }
+    };
+
+    return { widget: uploadWidget };
+}
+
+ComfyWidgets.AUDIOPLOAD = audioUpload;
+
+app.registerExtension({
+	name: "WhispherX.UploadAudio",
+	async beforeRegisterNodeDef(nodeType, nodeData, app) {
+		if (nodeData?.name == "LoadAudio") {
+			nodeData.input.required.upload = ["AUDIOPLOAD"];
+		}
+	},
+});
+
@@ -0,0 +1,137 @@
+import { app } from "../../../scripts/app.js";
+import { api } from '../../../scripts/api.js'
+import { ComfyWidgets } from "../../../scripts/widgets.js"
+
+function draggable(model) {
+  model.buttonMode = true;
+  model.on("pointerdown", (e) => {
+    model.dragging = true;
+    model._pointerX = e.data.global.x - model.x;
+    model._pointerY = e.data.global.y - model.y;
+  });
+  model.on("pointermove", (e) => {
+    if (model.dragging) {
+      model.position.x = e.data.global.x - model._pointerX;
+      model.position.y = e.data.global.y - model._pointerY;
+    }
+  });
+  model.on("pointerupoutside", () => (model.dragging = false));
+  model.on("pointerup", () => (model.dragging = false));
+}
+
+function talk(model,audio_link){
+    var volume = 1; // [Optional arg, can be null or empty] [0.0 - 1.0]
+    var expression = 8; // [Optional arg, can be null or empty] [index|name of expression]
+    var resetExpression = true; // [Optional arg, can be null or empty] [true|false] [default: true] [if true, expression will be reset to default after animation is over]
+    var crossOrigin = "anonymous"; // [Optional arg, to use not same-origin audios] [DEFAULT: null]
+    
+    model.speak(audio_link, {volume: volume, expression:expression, resetExpression:resetExpression, crossOrigin: crossOrigin})
+    
+    // Or if you want to keep some things default
+    model.speak(audio_link)
+    model.speak(audio_link, {volume: volume})
+    model.speak(audio_link, {expression:expression, resetExpression:resetExpression})
+}
+
+function showVtuber(node, audio_name,audio_type, vtuber) {
+    let params =  {
+        "filename": audio_name,
+        "type": audio_type,
+    }
+    const audio_link = api.apiURL('/view?' + new URLSearchParams(params));
+
+    var element = document.createElement("div");
+   
+    const previewNode = node;
+    var previewWidget = node.addDOMWidget("vtuberpreview", "preview", element, {
+        serialize: false,
+        hideOnZoom: false,
+        getValue() {
+            return element.value;
+        },
+        setValue(v) {
+            element.value = v;
+        },
+    });
+
+    previewWidget.computeSize = function(width) {
+        if (this.aspectRatio && !this.parentEl.hidden) {
+            let height = (previewNode.size[0]-20)/ this.aspectRatio + 10;
+            if (!(height > 0)) {
+                height = 0;
+            }
+            this.computedHeight = height + 10;
+            return [width, height];
+        }
+        return [width, -4];//no loaded src, widget should not display
+    }
+    // element.style['pointer-events'] = "none"
+    previewWidget.value = {hidden: false, paused: false, params: {}}
+    previewWidget.parentEl = document.createElement("canvas");
+    previewWidget.parentEl.id = "canvas";
+    previewWidget.parentEl.style['width'] = "100%"
+    element.appendChild(previewWidget.parentEl);
+
+    const cubism4Model = `extensions/ComfyUI-Live2DViewer/vtuber/${vtuber}/runtime/${vtuber}.model3.json`;
+
+    (async function main() {
+
+        const app = new PIXI.Application({
+          view: document.getElementById("canvas"),
+          autoStart: true,
+          resizeTo: window,
+          backgroundColor: 0x333333
+        });
+      
+        const models = await Promise.all([
+          PIXI.live2d.Live2DModel.from(cubism4Model)
+        ]);
+      
+        models.forEach((model) => {
+          app.stage.addChild(model);
+      
+          const scaleX = (innerWidth ) / model.width;
+          const scaleY = (innerHeight ) / model.height;
+      
+          // fit the window
+          model.scale.set(Math.min(scaleX, scaleY));
+      
+          model.y = innerHeight * 0.1;
+          draggable(model);
+          talk(model,audio_link);
+        });
+      
+        const model4 = models[0];
+      
+      
+        model4.x = innerWidth  / 2;
+      
+      
+        model4.on("hit", (hitAreas) => {
+          if (hitAreas.includes("Body")) {
+            model4.motion("Tap");
+          }
+      
+          if (hitAreas.includes("Head")) {
+            model4.expression();
+          }
+        });
+        let btnWidget =  node.addWidget("button", "RELOAD", "Audio", () => {
+          talk(model4,audio_link);
+        });
+        element.appendChild(btnWidget);
+      })();
+      return { widget: previewWidget };
+}
+
+app.registerExtension({
+	name: "Live2DViewer.ShowVtuber",
+	async beforeRegisterNodeDef(nodeType, nodeData, app) {
+		if (nodeData?.name == "Live2DViewer") {
+      nodeType.prototype.onExecuted = function (data) {
+				showVtuber(this, data.vtuber[0],data.vtuber[1],data.vtuber[2]);
+        console.log(data.vtuber);
+			}
+		}
+	},
+});