chore: ✨ stash audio experiments

__init__.py

@@ -322,6 +322,14 @@ if hasattr(PromptServer, "instance"):
 
         return await endpoint.do_action(request)
 
+    @PromptServer.instance.routes.get("/mtb/audio")
+    async def get_audio(request):
+        from . import endpoint
+
+        reload(endpoint)
+
+        return await endpoint.get_audio(request)
+
 
 # - WAS Dictionary
 MANIFEST = {

endpoint.py

@@ -3,7 +3,16 @@ import csv
 from aiohttp import web
 
 from .log import mklog
-from .utils import backup_file, here, import_install, reqs_map, run_command, styles_dir
+from .utils import (
+    audioInputDir,
+    backup_file,
+    comfy_dir,
+    here,
+    import_install,
+    reqs_map,
+    run_command,
+    styles_dir,
+)
 
 endlog = mklog("mtb endpoint")
 
@@ -15,6 +24,29 @@ from pathlib import Path
 import_install("requirements")
 
 
+def ACTIONS_loadAudio(args):
+    if not audioInputDir.exists():
+        audioInputDir.mkdir()
+
+    endlog.debug(f"Received Load Audio request for {args}")
+
+    if not args.file:
+        return web.Response(status=400)
+
+    filename = args.filename
+    if not filename:
+        return web.Response(status=400)
+
+    target = audioInputDir / filename
+    if target.exists():
+        target.unlink()
+
+    with target.open("wb") as f:
+        f.write(args.file.read())
+
+    return {"name": filename}
+
+
 def ACTIONS_installDependency(dependency_names=None):
     if dependency_names is None:
         return {"error": "No dependency name provided"}
@@ -88,7 +120,7 @@ def ACTIONS_saveStyle(data):
 
 async def do_action(request) -> web.Response:
     endlog.debug("Init action request")
-    request_data = await request.json()
+    request_data = await request.post()
     name = request_data.get("name")
     args = request_data.get("args")
 
@@ -100,14 +132,33 @@ async def do_action(request) -> web.Response:
     if callable(method):
         result = method(args) if args else method()
         endlog.debug(f"Action result: {result}")
-        return web.json_response({"result": result})
+        return web.json_response({"result": result}, status=200)
 
     available_methods = [
         attr[len("ACTIONS_") :] for attr in globals() if attr.startswith("ACTIONS_")
     ]
 
     return web.json_response(
-        {"error": "Invalid method name.", "available_methods": available_methods}
+        {"error": "Invalid method name.", "available_methods": available_methods},
+        status=400,
+    )
+
+
+async def get_audio(request):
+    name = request.rel_url.query.get("filename")
+    if not name:
+        return web.json_response(
+            {"error": "No filename provided as url query."}, status=400
+        )
+
+    target = audioInputDir / name
+    if not target.exists():
+        return web.json_response(
+            {"error": f"File {name} (in {audioInputDir}) not found..."}, status=404
+        )
+
+    return web.FileResponse(
+        target, headers={"Content-Disposition": f'filename="{name}"'}
     )
 
 

node_list.json

@@ -42,14 +42,13 @@
     "Mask To Image (mtb)": "Converts a mask (alpha) to an RGB image with a color and background",
     "Math Expression (mtb)": "Node to evaluate a simple math expression string",
     "Model Patch Seamless (mtb)": "Uses the stable diffusion 'hack' to infer seamless images by setting the model layers padding mode to circular (experimental)",
-    "Pick From Batch (mtb)": "Pick a specific number of images from a batch, either from the start or end.",
     "Qr Code (mtb)": "Basic QR Code generator",
     "Restore Face (mtb)": "Uses GFPGan to restore faces",
     "Save Gif (mtb)": "Save the images from the batch as a GIF",
     "Save Image Grid (mtb)": "Save all the images in the input batch as a grid of images.",
     "Save Image Sequence (mtb)": "Save an image sequence to a folder. The current frame is used to determine which image to save.\n\n    This is merely a wrapper around the `save_images` function with formatting for the output folder and filename.\n    ",
     "Save Tensors (mtb)": "Save torch tensors (image, mask or latent) to disk, useful to debug things outside comfy",
-    "Sharpen (mtb)": "Sharpens an image using a Gaussian kernel.",
+    "Sharpen (mtb)": "Sharpens an image using a Gaussian kernel.",
     "Smart Step (mtb)": "Utils to control the steps start/stop of the KAdvancedSampler in percentage",
     "Stack Images (mtb)": "Stack the input images horizontally or vertically",
     "String Replace (mtb)": "Basic string replacement",

nodes/batch.py

@@ -1,17 +1,20 @@
-import math
-import os
-from pathlib import Path
-from typing import List
+from io import BytesIO
 
 import cv2
-import folder_paths
+import torchaudio
 import numpy as np
 import torch
+from PIL import Image
 
 from ..log import log
 from ..utils import apply_easing, pil2tensor
 from .transform import TransformImage
 
+try:
+    import librosa
+except ImportError:
+    log.warning("librosa not installed. Batch Audio features will not be available.")
+
 
 def hex_to_rgb(hex_color, bgr=False):
     hex_color = hex_color.lstrip("#")
@@ -613,9 +616,57 @@ class BatchShake:
         return (shaken_images, x_translations, y_translations, rotations)
 
 
+class BatchFloatsFromSound:
+    """Extracts a list of floats based on audio frequency band peaks."""
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "audio": ("AUDIO",),
+                "sensitivity": ("FLOAT", {"default": 1.0}),
+                "low_freq": ("FLOAT", {"default": 100.0}),
+                "high_freq": ("FLOAT", {"default": 2000.0}),
+                "hop_length": ("INT", {"default": 512}),
+            },
+        }
+
+    RETURN_TYPES = ("FLOATS",)
+    RETURN_NAMES = ("float_data",)
+    FUNCTION = "process_audio"
+    CATEGORY = "mtb/audio"
+
+    def process_audio(
+        self,
+        audio,
+        sensitivity=1.0,
+        low_freq=100,
+        high_freq=2000,
+        hop_length=512,
+    ):
+        # audio_data, _ = librosa.load(audio_file_path, sr=sample_rate)
+
+        # audio_data_tensor = audio.squeeze(1)  # Remove the channel dimension if present
+        # audio_tensor = audio_data_tensor.float()
+        audio_data = audio.to(device=torchaudio.transforms.Spectrogram().window.device)
+
+        hop_length = 512
+        stft = torchaudio.transforms.Spectrogram()(audio_data)
+        freqs = torchaudio.transforms.FrequencyMasking(low_freq, high_freq)(stft)
+        band_energy = torch.sum(freqs, dim=1)
+
+        min_val = torch.min(band_energy)
+        max_val = torch.max(band_energy)
+        normalized_peaks = (band_energy - min_val) / (max_val - min_val)
+        scaled_peaks = normalized_peaks * sensitivity
+
+        return (scaled_peaks.tolist(),)
+
+
 __nodes__ = [
     BatchFloat,
     Batch2dTransform,
+    BatchFloatsFromSound,
     BatchShape,
     BatchMake,
     BatchFloatAssemble,

nodes/conditions.py

@@ -1,4 +1,5 @@
-import csv, shutil
+import csv
+import shutil
 from pathlib import Path
 
 import folder_paths
@@ -156,21 +157,13 @@ class StylesLoader:
             for file in files:
                 with open(file, "r", encoding="utf8") as f:
                     parsed = csv.reader(f)
-                    for i, row in enumerate(parsed):
+                    for row in parsed:
                         log.debug(f"Adding style {row[0]}")
                         try:
-                            name, positive, negative = (row + [None] * 3)[:3]
-                            positive = positive or ""
-                            negative = negative or ""
-                            if name is not None:
-                                cls.options[name] = (positive, negative)
-                            else:
-                                # Handle the case where 'name' is None
-                                log.warning(f"Missing 'name' in row {i}.")
-
-                        except Exception as e:
+                            cls.options[row[0]] = (row[1], row[2])
+                        except Exception:
                             log.warning(
-                                f"There was an error while parsing {file}, make sure it respects A1111 format, i.e 3 columns name, positive, negative:\n{e}"
+                                f"There was an error while parsing {file}, make sure it respects A1111 format, i.e 3 columns name, positive, negative"
                             )
                             continue
 

nodes/crop.py

@@ -44,7 +44,6 @@ class BboxFromMask:
         return {
             "required": {
                 "mask": ("MASK",),
-                "invert": ("BOOLEAN", {"default": False}),
             },
             "optional": {
                 "image": ("IMAGE",),
@@ -62,7 +61,7 @@ class BboxFromMask:
     FUNCTION = "extract_bounding_box"
     CATEGORY = "mtb/crop"
 
-    def extract_bounding_box(self, mask: torch.Tensor, invert: bool, image=None):
+    def extract_bounding_box(self, mask: torch.Tensor, image=None):
         # if image != None:
         #     if mask.size(0) != image.size(0):
         #         if mask.size(0) != 1:
@@ -74,8 +73,9 @@ class BboxFromMask:
         #                 f"Batch count mismatch for mask and image, it can either be 1 mask for X images, or X masks for X images (mask: {mask.shape} | image: {image.shape})"
         #             )
 
+        _mask = tensor2pil(1.0 - mask)[0]
+
         # we invert it
-        _mask = tensor2pil(1.0 - mask)[0] if invert else tensor2pil(mask)[0]
         alpha_channel = np.array(_mask)
 
         non_zero_indices = np.nonzero(alpha_channel)
@@ -141,23 +141,19 @@ class Crop:
         self, image: torch.Tensor, mask=None, x=0, y=0, width=256, height=256, bbox=None
     ):
         image = image.numpy()
-        if mask is not None:
+        if mask:
             mask = mask.numpy()
 
-        if bbox is not None:
+        if bbox != None:
             x, y, width, height = bbox
 
         cropped_image = image[:, y : y + height, x : x + width, :]
-        cropped_mask = None
-        if mask is not None:
-            cropped_mask = (
-                mask[:, y : y + height, x : x + width] if mask is not None else None
-            )
+        cropped_mask = mask[y : y + height, x : x + width] if mask != None else None
         crop_data = (x, y, width, height)
 
         return (
             torch.from_numpy(cropped_image),
-            torch.from_numpy(cropped_mask) if cropped_mask is not None else None,
+            torch.from_numpy(cropped_mask) if mask != None else None,
             crop_data,
         )
 

nodes/generate.py

@@ -1,11 +1,9 @@
-import threading
-from typing import cast
-
 import qrcode
+from ..utils import pil2tensor
+from ..utils import comfy_dir
+from typing import cast
 from PIL import Image
-
 from ..log import log
-from ..utils import comfy_dir, pil2tensor
 
 # class MtbExamples:
 #     """MTB Example Images"""
@@ -76,9 +74,8 @@ class UnsplashImage:
     CATEGORY = "mtb/generate"
 
     def do_unsplash_image(self, width, height, random_seed, keyword=None):
-        import io
-
         import requests
+        import io
 
         base_url = "https://source.unsplash.com/random/"
 
@@ -204,13 +201,12 @@ class TextToImage:
 
         for font in fonts:
             log.debug(f"Adding font {font}")
-            TextToImage.fonts[font.stem] = font.as_posix()
+            cls.fonts[font.stem] = font.as_posix()
 
     @classmethod
     def INPUT_TYPES(cls):
         if not cls.fonts:
-            thread = threading.Thread(target=cls.CACHE_FONTS)
-            thread.start()
+            cls.CACHE_FONTS()
         else:
             log.debug(f"Using cached fonts (count: {len(cls.fonts)})")
         return {
@@ -236,6 +232,7 @@ class TextToImage:
                     "INT",
                     {"default": 512, "min": 1, "max": 8096, "step": 1},
                 ),
+                # "position": (["INT"], {"default": 0, "min": 0, "max": 100, "step": 1}),
                 "color": (
                     "COLOR",
                     {"default": "black"},
@@ -244,8 +241,6 @@ class TextToImage:
                     "COLOR",
                     {"default": "white"},
                 ),
-                "h_align": (("left", "center", "right"), {"default": "left"}),
-                "v_align": (("top", "center", "bottom"), {"default": "top"}),
             }
         }
 
@@ -255,62 +250,29 @@ class TextToImage:
     CATEGORY = "mtb/generate"
 
     def text_to_image(
-        self,
-        text,
-        font,
-        wrap,
-        font_size,
-        width,
-        height,
-        color,
-        background,
-        h_align="left",
-        v_align="top",
+        self, text, font, wrap, font_size, width, height, color, background
     ):
+        from PIL import Image, ImageDraw, ImageFont
         import textwrap
 
-        from PIL import Image, ImageDraw, ImageFont
-
-        font_path = self.fonts[font]
-
-        # Handle word wrapping
-        if wrap:
-            lines = textwrap.wrap(text, width=wrap)
-        else:
-            lines = [text]
-        font = ImageFont.truetype(font_path, font_size)
-        # font = ImageFont.truetype(font_path, font_size)
-        # if wrap == 0:
-        #     wrap = width / font_size
-
+        font = self.fonts[font]
+        font = cast(ImageFont.FreeTypeFont, ImageFont.truetype(font, font_size))
+        if wrap == 0:
+            wrap = width / font_size
+        lines = textwrap.wrap(text, width=wrap)
         log.debug(f"Lines: {lines}")
-        img = Image.new("RGBA", (width, height), background)
+        line_height = bbox_dim(font.getbbox("hg"))[1]
+        img_height = height  # line_height * len(lines)
+        img_width = width  # max(font.getsize(line)[0] for line in lines)
+
+        img = Image.new("RGBA", (img_width, img_height), background)
         draw = ImageDraw.Draw(img)
-
-        text_height = sum(font.getsize(line)[1] for line in lines)
-
-        # Vertical alignment
-        if v_align == "top":
-            y_text = 0
-        elif v_align == "center":
-            y_text = (height - text_height) // 2
-        else:  # bottom
-            y_text = height - text_height
-
-        # Draw each line of text
+        y_text = 0
+        # - bbox is [left, upper, right, lower]
         for line in lines:
-            line_width, line_height = font.getsize(line)
-
-            # Horizontal alignment
-            if h_align == "left":
-                x_text = 0
-            elif h_align == "center":
-                x_text = (width - line_width) // 2
-            else:  # right
-                x_text = width - line_width
-
-            draw.text((x_text, y_text), line, color, font=font)
-            y_text += line_height
+            width, height = bbox_dim(font.getbbox(line))
+            draw.text((0, y_text), line, color, font=font)
+            y_text += height
 
         # img.save(os.path.join(folder_paths.base_path, f'{str(uuid.uuid4())}.png'))
         return (pil2tensor(img),)

nodes/graph_utils.py

@@ -1,4 +1,7 @@
-import io, json, urllib.parse, urllib.request
+import io
+import json
+import urllib.parse
+import urllib.request
 
 import numpy as np
 import torch

nodes/image_utils.py

@@ -38,39 +38,4 @@ class StackImages:
         return (stacked_tensor,)
 
 
-class PickFromBatch:
-    """Pick a specific number of images from a batch, either from the start or end."""
-
-    @classmethod
-    def INPUT_TYPES(cls):
-        return {
-            "required": {
-                "image": ("IMAGE",),
-                "from_direction": (["end", "start"], {"default": "start"}),
-                "count": ("INT", {"default": 1}),
-            }
-        }
-
-    RETURN_TYPES = ("IMAGE",)
-    FUNCTION = "pick_from_batch"
-    CATEGORY = "mtb/image utils"
-
-    def pick_from_batch(self, image, from_direction, count):
-        batch_size = image.size(0)
-
-        # Limit count to the available number of images in the batch
-        count = min(count, batch_size)
-        if count < batch_size:
-            log.warning(
-                f"Requested {count} images, but only {batch_size} are available."
-            )
-
-        if from_direction == "end":
-            selected_tensors = image[-count:]
-        else:
-            selected_tensors = image[:count]
-
-        return (selected_tensors,)
-
-
-__nodes__ = [StackImages, PickFromBatch]
+__nodes__ = [StackImages]

nodes/io.py

@@ -1,4 +1,6 @@
-import json, subprocess, uuid
+import json
+import subprocess
+import uuid
 from pathlib import Path
 from typing import List, Optional
 
@@ -6,102 +8,42 @@ import comfy.model_management as model_management
 import folder_paths
 import numpy as np
 import torch
+from comfy.model_management import get_torch_device
 from PIL import Image
 
 from ..log import log
-from ..utils import PIL_FILTER_MAP, output_dir, session_id, tensor2np
+from ..utils import PIL_FILTER_MAP, audioInputDir, tensor2np
+
+try:
+    import librosa
+except ImportError:
+    log.warning("librosa not installed. I/O Audio features will not be available.")
 
 
-def get_playlist_path(playlist_name: str, persistant_playlist=False):
-    if persistant_playlist:
-        return output_dir / "playlists" / f"{playlist_name}.json"
-
-    return output_dir / "playlists" / session_id / f"{playlist_name}.json"
-
-
-class ReadPlaylist:
-    """Read a playlist"""
+class LoadAudio_:
+    """Load an audio file from the input folder (supports upload)"""
 
     @classmethod
     def INPUT_TYPES(cls):
         return {
             "required": {
-                "enable": ("BOOLEAN", {"default": True}),
-                "persistant_playlist": ("BOOLEAN", {"default": False}),
-                "playlist_name": ("STRING", {"default": "playlist_{index:04d}"}),
-                "index": ("INT", {"default": 0, "min": 0}),
+                "audio": ("AUDIO_UPLOAD",),
+                "sample_rate": ("INT", {"default": 44100}),
             }
         }
 
-    RETURN_TYPES = ("PLAYLIST",)
-    FUNCTION = "read_playlist"
-    CATEGORY = "mtb/IO"
+    RETURN_TYPES = ("AUDIO",)
+    RETURN_NAMES = ("audio",)
+    FUNCTION = "load_audio"
+    CATEGORY = "mtb/audio"
 
-    def read_playlist(
-        self, enable: bool, persistant_playlist: bool, playlist_name: str, index: int
-    ):
-        playlist_name = playlist_name.format(index=index)
-        playlist_path = get_playlist_path(playlist_name, persistant_playlist)
-        if not enable:
-            return (None,)
-
-        if not playlist_path.exists():
-            log.warning(f"Playlist {playlist_path} does not exist, skipping")
-            return (None,)
-
-        log.debug(f"Reading playlist {playlist_path}")
-        return (json.loads(playlist_path.read_text(encoding="utf-8")),)
-
-
-class AddToPlaylist:
-    """Add a video to the playlist"""
-
-    @classmethod
-    def INPUT_TYPES(cls):
-        return {
-            "required": {
-                "relative_paths": ("BOOLEAN", {"default": False}),
-                "persistant_playlist": ("BOOLEAN", {"default": False}),
-                "playlist_name": ("STRING", {"default": "playlist_{index:04d}"}),
-                "index": ("INT", {"default": 0, "min": 0}),
-            }
-        }
-
-    RETURN_TYPES = ()
-    OUTPUT_NODE = True
-    FUNCTION = "add_to_playlist"
-    CATEGORY = "mtb/IO"
-
-    def add_to_playlist(
-        self,
-        relative_paths: bool,
-        persistant_playlist: bool,
-        playlist_name: str,
-        index: int,
-        **kwargs,
-    ):
-        playlist_name = playlist_name.format(index=index)
-        playlist_path = get_playlist_path(playlist_name, persistant_playlist)
-
-        if not playlist_path.parent.exists():
-            playlist_path.parent.mkdir(parents=True, exist_ok=True)
-
-        playlist = []
-        if not playlist_path.exists():
-            playlist_path.write_text("[]")
-        else:
-            playlist = json.loads(playlist_path.read_text())
-        log.debug(f"Playlist {playlist_path} has {len(playlist)} items")
-        for video in kwargs.values():
-            if relative_paths:
-                video = Path(video).relative_to(output_dir).as_posix()
-
-            log.debug(f"Adding {video} to playlist")
-            playlist.append(video)
-
-        log.debug(f"Writing playlist {playlist_path}")
-        playlist_path.write_text(json.dumps(playlist), encoding="utf-8")
-        return ()
+    def load_audio(self, audio: str, sample_rate: int):
+        log.debug(f"Audio file: {audio}")
+        audio_file_path = audioInputDir / audio
+        log.debug(f"Loading audio file: {audio_file_path}")
+        audio_data, _ = librosa.load(audio_file_path.as_posix(), sr=sample_rate)
+        audio_tensor = torch.from_numpy(audio_data).to(get_torch_device())
+        return (audio_tensor.unsqueeze(0).float(),)
 
 
 class ExportWithFfmpeg:
@@ -110,11 +52,8 @@ class ExportWithFfmpeg:
     @classmethod
     def INPUT_TYPES(cls):
         return {
-            "optional": {
-                "images": ("IMAGE",),
-                "playlist": ("PLAYLIST",),
-            },
             "required": {
+                "images": ("IMAGE",),
                 # "frames": ("FRAMES",),
                 "fps": ("FLOAT", {"default": 24, "min": 1}),
                 "prefix": ("STRING", {"default": "export"}),
@@ -124,6 +63,7 @@ class ExportWithFfmpeg:
                     {"default": "prores_ks"},
                 ),
             },
+            "hidden": {"prompt": "PROMPT", "extra_pnginfo": "EXTRA_PNGINFO"},
         }
 
     RETURN_TYPES = ("VIDEO",)
@@ -133,61 +73,32 @@ class ExportWithFfmpeg:
 
     def export_prores(
         self,
+        images: torch.Tensor,
         fps: float,
         prefix: str,
         format: str,
         codec: str,
-        images: Optional[torch.Tensor] = None,
-        playlist: Optional[List[str]] = None,
+        prompt=None,
+        extra_pnginfo=None,
     ):
+        metadata = {}
+        if images.size(0) == 0:
+            return ("",)
+
+        if extra_pnginfo is not None:
+            metadata["extra"] = {}
+            for x in extra_pnginfo:
+                metadata["extra"][x] = json.dumps(extra_pnginfo[x])
+
+        if prompt is not None:
+            metadata["prompt"] = json.dumps(prompt)
+
+        output_dir = Path(folder_paths.get_output_directory())
         pix_fmt = "rgb48le" if codec == "prores_ks" else "yuv420p"
         file_ext = format
         file_id = f"{prefix}_{uuid.uuid4()}.{file_ext}"
 
-        if playlist is not None and images is not None:
-            log.info(f"Exporting to {output_dir / file_id}")
-
-        if playlist is not None:
-            if len(playlist) == 0:
-                log.debug("Playlist is empty, skipping")
-                return ("",)
-
-            temp_playlist_path = output_dir / f"temp_playlist_{uuid.uuid4()}.txt"
-            log.debug(
-                f"Create a temporary file to list the videos for concatenation to {temp_playlist_path}"
-            )
-
-            with open(temp_playlist_path, "w") as f:
-                for video_path in playlist:
-                    f.write(f"file '{video_path}'\n")
-
-            out_path = (output_dir / file_id).as_posix()
-
-            # Prepare the FFmpeg command for concatenating videos from the playlist
-            command = [
-                "ffmpeg",
-                "-f",
-                "concat",
-                "-safe",
-                "0",
-                "-i",
-                temp_playlist_path.as_posix(),
-                "-c",
-                "copy",
-                "-y",
-                out_path,
-            ]
-            log.debug(f"Executing {command}")
-            subprocess.run(command)
-
-            temp_playlist_path.unlink()
-
-            return (out_path,)
-
-        if (
-            images is None or images.size(0) == 0
-        ):  # the is None check is just for the type checker
-            return ("",)
+        log.debug(f"Exporting to {output_dir / file_id}")
 
         frames = tensor2np(images)
         log.debug(f"Frames type {type(frames[0])}")
@@ -199,6 +110,15 @@ class ExportWithFfmpeg:
 
         out_path = (output_dir / file_id).as_posix()
 
+        metadata_cmd = []
+
+        if metadata:
+            for k, v in metadata.items():
+                metadata_cmd += [
+                    "-metadata:s:v",
+                    f"{k}='{v if isinstance(v,str) else json.dumps(v)}'",
+                ]
+
         # Prepare the FFmpeg command
         command = [
             "ffmpeg",
@@ -217,6 +137,7 @@ class ExportWithFfmpeg:
             "-",
             "-c:v",
             codec,
+            *metadata_cmd,
             "-r",
             str(fps),
             "-y",
@@ -329,4 +250,4 @@ class SaveGif:
         return {"ui": {"gif": results}}
 
 
-__nodes__ = [SaveGif, ExportWithFfmpeg, AddToPlaylist, ReadPlaylist]
+__nodes__ = [SaveGif, ExportWithFfmpeg, LoadAudio_]

requirements.txt

@@ -5,4 +5,6 @@ requirements-parser
 rembg
 imageio_ffmpeg
 rich
-rich_argparse
+rich_argparse
+librosa
+torchaudio

utils.py

@@ -1,4 +1,13 @@
-import contextlib, functools, math, os, shlex, shutil, socket, subprocess, sys, uuid
+import contextlib
+import functools
+import math
+import os
+import shlex
+import shutil
+import socket
+import subprocess
+import sys
+import uuid
 from pathlib import Path
 from typing import List, Optional, Union
 
@@ -229,9 +238,9 @@ here = Path(__file__).parent.absolute()
 # - Construct the absolute path to the ComfyUI directory
 comfy_dir = Path(folder_paths.base_path)
 models_dir = Path(folder_paths.models_dir)
-output_dir = Path(folder_paths.output_directory)
 styles_dir = comfy_dir / "styles"
-session_id = str(uuid.uuid4())
+audioInputDir = comfy_dir / "input" / "audio"
+
 # - Construct the path to the font file
 font_path = here / "font.ttf"
 

web/README.md

@@ -13,10 +13,6 @@ data otherwise:
 ![debug](https://github.com/melMass/comfy_mtb/assets/7041726/1f4393e4-1c3d-4807-9501-fe8888bfae25)
 
 
-**note +**
-A basic HTML note mainly to add better looking notes/instructions for workflow makers:
-![image](https://github.com/melMass/comfy_mtb/assets/7041726/2ba1f832-0044-4bad-974c-e6387981af57)
-
 
 ## Standalone
 These scripts can be taken and placed independently of `comfy_mtb` or any other files, mimicking what pythongosss did for their 

web/comfy_shared.js

@@ -121,9 +121,6 @@ export const dynamic_connection = (
   connectionType = 'PSDLAYER',
   nameArray = []
 ) => {
-  if (!node.inputs[index].name.startsWith(connectionPrefix)) {
-    return
-  }
   // remove all non connected inputs
   if (!connected && node.inputs.length > 1) {
     log(`Removing input ${index} (${node.inputs[index].name})`)
@@ -334,43 +331,6 @@ function getBrightness(rgbObj) {
 }
 
 //- HTML / CSS UTILS
-export const loadScript = (
-  FILE_URL,
-  async = true,
-  type = 'text/javascript'
-) => {
-  return new Promise((resolve, reject) => {
-    try {
-      // Check if the script already exists
-      const existingScript = document.querySelector(`script[src="${FILE_URL}"]`)
-      if (existingScript) {
-        resolve({ status: true, message: 'Script already loaded' })
-        return
-      }
-
-      const scriptEle = document.createElement('script')
-      scriptEle.type = type
-      scriptEle.async = async
-      scriptEle.src = FILE_URL
-
-      scriptEle.addEventListener('load', (ev) => {
-        resolve({ status: true })
-      })
-
-      scriptEle.addEventListener('error', (ev) => {
-        reject({
-          status: false,
-          message: `Failed to load the script ＄{FILE_URL}`,
-        })
-      })
-
-      document.body.appendChild(scriptEle)
-    } catch (error) {
-      reject(error)
-    }
-  })
-}
-
 export function defineClass(className, classStyles) {
   const styleSheets = document.styleSheets
 

web/mtb_widgets.js

@@ -7,8 +7,6 @@
  *
  */
 
-// TODO: Use the builtin addDOMWidget everywhere appropriate
-
 import { app } from '../../scripts/app.js'
 import { api } from '../../scripts/api.js'
 
@@ -16,7 +14,7 @@ import parseCss from './extern/parse-css.js'
 import * as shared from './comfy_shared.js'
 import { log } from './comfy_shared.js'
 
-const newTypes = [, /*'BOOL'*/ 'COLOR', 'BBOX']
+const newTypes = [, /*'BOOL'*/ 'COLOR', 'BBOX', 'AUDIO_UPLOAD']
 
 const withFont = (ctx, font, cb) => {
   const oldFont = ctx.font
@@ -47,6 +45,54 @@ const calculateTextDimensions = (ctx, value, width, fontSize = 16) => {
   )
   return { textHeight, maxLineWidth }
 }
+function addPlaybackWidget(node, name, url) {
+  let isTick = true
+  const audio = new Audio(url)
+  const slider = node.addWidget(
+    'slider',
+    'loading',
+    0,
+    (v) => {
+      if (!isTick) {
+        audio.currentTime = v
+      }
+      isTick = false
+    },
+    {
+      min: 0,
+      max: 0,
+    }
+  )
+
+  const button = node.addWidget('button', `Play ${name}`, 'play', () => {
+    try {
+      if (audio.paused) {
+        audio.play()
+        button.name = `Pause ${name}`
+      } else {
+        audio.pause()
+        button.name = `Play ${name}`
+      }
+    } catch (error) {
+      alert(error)
+    }
+    app.canvas.setDirty(true)
+  })
+  audio.addEventListener('timeupdate', () => {
+    isTick = true
+    slider.value = audio.currentTime
+    app.canvas.setDirty(true)
+  })
+  audio.addEventListener('ended', () => {
+    button.name = `Play ${name}`
+    app.canvas.setDirty(true)
+  })
+  audio.addEventListener('loadedmetadata', () => {
+    slider.options.max = audio.duration
+    slider.name = `(${audio.duration})`
+    app.canvas.setDirty(true)
+  })
+}
 
 export const MtbWidgets = {
   BBOX: (key, val) => {
@@ -394,6 +440,119 @@ export const MtbWidgets = {
 
     return w
   },
+
+  AUDIO_UPLOAD: function (name, val) {
+    const w = {
+      name,
+      type: 'audio_upload',
+      value: val,
+      draw: function (ctx, node, widgetWidth, widgetY, height) {
+        const [cw, ch] = this.computeSize(widgetWidth)
+        shared.offsetDOMWidget(this, ctx, node, widgetWidth, widgetY, ch)
+      },
+      computeSize: function (width) {
+        if (width) {
+          return [width, 64]
+        }
+        return [128, 128]
+      },
+      onRemoved: function () {
+        if (this.inputEl) {
+          this.inputEl.remove()
+        }
+      },
+    }
+
+    const uploadFile = async (file, node) => {
+      try {
+        const body = new FormData()
+        body.append('name', 'loadAudio')
+        body.append('args', file)
+        const loadAudio = await api.fetchApi('/mtb/actions', {
+          method: 'POST',
+          body,
+        })
+
+        if (loadAudio.status === 200) {
+          const { result } = await loadAudio.json()
+          console.log('received from server', result)
+          console.log(
+            `Getting file /mtb/audio?filename=${encodeURIComponent(
+              result.name
+            )}`
+          )
+
+          w.value = result.name
+          addPlaybackWidget(
+            node,
+            result.name,
+            `/mtb/audio?filename=${encodeURIComponent(result.name)}`
+          )
+        } else {
+          alert(loadAudio.status + ' -' + loadAudio.statusText)
+        }
+        // if (resp.status === 200) {
+        //   const { name } = await resp.json()
+        //   pathWidget.value = name
+        //   addPlaybackWidget(
+        //     node,
+        //     name,
+        //     `/samplediffusion/audio?filename=${encodeURIComponent(name)}`
+        //   )
+        // } else {
+        //   alert(resp.status + ' - ' + resp.statusText)
+        // }
+      } catch (error) {
+        alert(error)
+        throw error
+      }
+    }
+
+    w.inputEl = document.createElement('div')
+    const hidden_input = document.createElement('input')
+    const label = document.createElement('label')
+
+    const uniqueId = 'input_' + Date.now()
+    Object.assign(hidden_input, {
+      type: 'file',
+      accept: 'audio/mpeg,audio/wav,audio/x-wav',
+      id: uniqueId,
+      style: `
+      width: 0.1px;
+      height: 0.1px;
+      opacity: 0;
+      overflow: hidden;
+      position: absolute;
+      z-index: -1;
+
+      `,
+      onchange: async () => {
+        if (hidden_input.files.length) {
+          console.log(hidden_input.files[0])
+          await uploadFile(hidden_input.files[0], this)
+        }
+      },
+    })
+
+    Object.assign(label, {
+      htmlFor: uniqueId,
+    })
+    label.textContent = 'Upload Audio File'
+    label.style = `
+    font-size: 1.25em;
+    font-weight: 700;
+    font-family: monospace;
+    padding:0.5em;
+    border-radius: 5px;
+    color: white;
+    background-color: #1e1e1e;
+    display: inline-block;
+    `
+    document.body.appendChild(w.inputEl)
+    w.inputEl.appendChild(hidden_input)
+    w.inputEl.appendChild(label)
+    return w
+  },
 }
 
 /**
@@ -477,6 +636,20 @@ const mtb_widgets = {
           minHeight: 30,
         }
       },
+      AUDIO_UPLOAD: (node, inputName, inputData, app) => {
+        console.debug('Registering audio')
+        return {
+          widget: node.addCustomWidget(
+            MtbWidgets.AUDIO_UPLOAD.bind(node)(
+              inputName,
+              inputData[1]?.default || ''
+            )
+          ),
+          minWidth: 150,
+          minHeight: 30,
+        }
+      },
+
       // BBOX: (node, inputName, inputData, app) => {
       //     console.debug("Registering bbox")
       //     return {
@@ -496,6 +669,10 @@ const mtb_widgets = {
   async beforeRegisterNodeDef(nodeType, nodeData, app) {
     // const rinputs = nodeData.input?.required
 
+    if (!nodeData.name.endsWith('(mtb)')) {
+      return
+    }
+
     let has_custom = false
     if (nodeData.input && nodeData.input.required) {
       for (const i of Object.keys(nodeData.input.required)) {
@@ -563,10 +740,6 @@ const mtb_widgets = {
       }
     }
 
-    if (!nodeData.name.endsWith('(mtb)')) {
-      return
-    }
-
     //- Extending Python Nodes
     switch (nodeData.name) {
       case 'Psd Save (mtb)': {
@@ -883,17 +1056,14 @@ const mtb_widgets = {
 
         break
       }
-      case 'Add To Playlist (mtb)': {
-        shared.setupDynamicConnections(nodeType, 'video', 'VIDEO')
-        break
-      }
       case 'Stack Images (mtb)':
       case 'Concat Images (mtb)': {
         shared.setupDynamicConnections(nodeType, 'image', 'IMAGE')
 
         break
       }
-      case 'Batch Float Assemble (mtb)': {
+      case 'Batch Float Assemble (mtb)':
+      case 'Plot Batch Float (mtb)': {
         shared.setupDynamicConnections(nodeType, 'floats', 'FLOATS')
         break
       }
@@ -902,7 +1072,6 @@ const mtb_widgets = {
 
         break
       }
-      // TODO: remove this, recommend pythongoss's version that is much better
       case 'Math Expression (mtb)': {
         const onNodeCreated = nodeType.prototype.onNodeCreated
         nodeType.prototype.onNodeCreated = function () {

web/note_plus.js

@@ -1,181 +0,0 @@
-import { app } from '../../scripts/app.js'
-import { api } from '../../scripts/api.js'
-import * as shared from './comfy_shared.js'
-
-class NotePlus extends LiteGraph.LGraphNode {
-  title = 'Note+ (mtb)'
-  category = 'mtb/utils'
-
-  constructor() {
-    super()
-
-    this.isVirtualNode = true
-    this.serialize_widgets = true
-
-    this.editing = false
-    this.live = true
-    this.rawVal = "<p style='color:red;font-family:monospace'\n>    Note+\n</p>"
-
-    this.calculated_height = 36
-
-    const inner = document.createElement('div')
-    inner.style.margin = '0'
-    inner.style.padding = '0'
-    this.html_widget = this.addDOMWidget('HTML', 'html', inner, {
-      setValue: (v) => {
-        // update our widget preview
-        this.html_widget.element.innerHTML = v
-        // calculate height
-        this.calculated_height = this.html_widget.element.scrollHeight + 36
-      },
-      getValue: () => this.rawVal,
-      getMinHeight: () => this.calculated_height, // (the edit button),
-    })
-
-    // console.log(`Value of HTML: ${this.html_widget.value}`)
-    this.html_widget.element.innerHTML = this.html_widget.value
-
-    //- ace based editor
-    this.addWidget('button', 'Edit', 'Edit', () => {
-      const container = document.createElement('div')
-      Object.assign(container.style, {
-        display: 'flex',
-        gap: '10px',
-      })
-
-      dialog.show('')
-      dialog.textElement.append(container)
-
-      const value = document.createElement('div')
-      value.id = 'noteplus-editor'
-      Object.assign(value.style, {
-        width: '300px',
-        height: '200px',
-        backgroundColor: 'rgb(30,30,30)',
-        color: 'whitesmoke',
-      })
-
-      container.append(value)
-
-      const live_edit = document.createElement('input')
-      live_edit.type = 'checkbox'
-      live_edit.checked = this.live
-      live_edit.onchange = () => {
-        this.live = live_edit.checked
-      }
-
-      const live_edit_label = document.createElement('label')
-      live_edit_label.textContent = 'Live Edit'
-      live_edit_label.append(live_edit)
-
-      value.after(live_edit_label)
-
-      this.setupEditor()
-      this.editor.setValue(this.html_widget.element.innerHTML)
-    })
-
-    const dialog = new app.ui.dialog.constructor()
-    dialog.element.classList.add('comfy-settings')
-
-    const closeButton = dialog.element.querySelector('button')
-    closeButton.textContent = 'CANCEL'
-    const saveButton = document.createElement('button')
-    saveButton.textContent = 'SAVE'
-    saveButton.onclick = () => {
-      this.updateHTML(this.editor.getValue())
-
-      this.editor.destroy()
-      this.editor.container.remove()
-
-      dialog.close()
-    }
-
-    closeButton.before(saveButton)
-
-    shared
-      .loadScript(
-        'https://cdn.jsdelivr.net/npm/ace-builds@1.16.0/src-min-noconflict/ace.min.js'
-      )
-      .catch((e) => {
-        console.error(e)
-      })
-  }
-
-  setupEditor() {
-    this.editor = ace.edit('noteplus-editor')
-    this.editor.setTheme('ace/theme/dracula')
-    this.editor.session.setMode('ace/mode/html')
-
-    this.editor.setShowPrintMargin(false)
-    this.editor.session.setUseWrapMode(true)
-    this.editor.renderer.setShowGutter(false)
-    this.editor.session.setTabSize(4)
-    this.editor.session.setUseSoftTabs(true)
-    this.editor.setFontSize(14)
-    this.editor.setReadOnly(false)
-    this.editor.setHighlightActiveLine(false)
-    this.editor.setShowFoldWidgets(true)
-
-    this.editor.session.on('change', (delta) => {
-      // delta.start, delta.end, delta.lines, delta.action
-      if (this.live) {
-        this.updateHTML(this.editor.getValue())
-      }
-    })
-  }
-
-  updateHTML(val) {
-    // if (CONTAINER_HTML.includes('${html}')) {
-    //   console.log('found template')
-    //   val = CONTAINER_HTML.replace('${html}', val)
-    // }
-
-    this.html_widget.value = val
-    this.rawVal = val
-
-    this.calculated_height = this.html_widget.element.scrollHeight
-
-    this.setSize(this.computeSize())
-  }
-
-  // //   onRemoved() {
-  // //     console.log('Removing', this)
-  // //     for (const w of this.widgets) {
-  // //       console.log('Removing', w)
-  // //       w.onRemove?.()
-  // //       w.onRemoved?.()
-  // //     }
-  // //   }
-}
-
-app.registerExtension({
-  name: 'mtb.noteplus',
-
-  setup() {
-    // app.ui.settings.addSetting({
-    //     id: "mtb.noteplus.Container",
-    //     name: "📦 HTML container",
-    //     type: "text",
-    //     defaultValue: "<div>${html}</div>",
-    //     tooltip:
-    //         "This defines the wrapper for the noteplus html content, use '${html}' to define the location of the placeholder",
-    //     attrs: {
-    //         style: {
-    //             fontFamily: "monospace",
-    //         },
-    //     },
-    //     onChange(value) {
-    //         if (!value) {
-    //             CONTAINER_HTML = null;
-    //             return;
-    //         }
-    //         console.log(`NOTEPLUS| value changed: ${value}`)
-    //         CONTAINER_HTML = value
-    //     },
-    // });
-  },
-
-  registerCustomNodes() {
-    LiteGraph.registerNodeType('Note Plus (mtb)', NotePlus)
-  },
-})