From 4652a2d52a9a7a1ec8c42ca0c9744594a98eb569 Mon Sep 17 00:00:00 2001 From: Sirshendu Ganguly Date: Wed, 22 Oct 2025 14:02:40 -0400 Subject: [PATCH] Add Runware Media Upload node with VIDEO and AUDIO support - Created new Runware Media Upload custom node - Supports VIDEO and AUDIO inputs from ComfyUI load nodes - Handles ComfyUI VideoFromFile objects properly - Implements real-time mediaUUID display in node UI - Added proper error handling and debug logging - Applied consistent Runware node styling - Includes example workflow --- __init__.py | 2 + clientlibs/main.js | 3 +- clientlibs/types.js | 5 + clientlibs/utils.js | 19 ++ modules/mediaUpload.py | 336 ++++++++++++++++++++++++++++ modules/utils/runwareUtils.py | 11 + workflows/Runware_Media_Upload.json | 1 + 7 files changed, 376 insertions(+), 1 deletion(-) create mode 100644 modules/mediaUpload.py create mode 100644 workflows/Runware_Media_Upload.json diff --git a/__init__.py b/__init__.py index 91f22c1..509bb5b 100644 --- a/__init__.py +++ b/__init__.py @@ -39,6 +39,7 @@ from .modules.elevenlabsProviderSettings import RunwareElevenLabsProviderSetting from .modules.elevenlabsProviderSettingsSections import RunwareElevenLabsProviderSettingsSections from .modules.acceleratorOptions import acceleratorOptions from .modules.viduProviderSettings import RunwareViduProviderSettings +from .modules.mediaUpload import runwareMediaUpload RUNWARE_COMFYUI_VERSION = "1.4.0 Beta" @@ -92,6 +93,7 @@ NODE_CLASS_MAPPINGS = { "Runware ElevenLabs Provider Settings Sections": RunwareElevenLabsProviderSettingsSections, "Runware Accelerator Options": acceleratorOptions, "Runware Vidu Provider Settings": RunwareViduProviderSettings, + "Runware Media Upload": runwareMediaUpload, } NODE_DISPLAY_NAME_MAPPINGS = { diff --git a/clientlibs/main.js b/clientlibs/main.js index 19702a6..4fe1045 100644 --- a/clientlibs/main.js +++ b/clientlibs/main.js @@ -1,6 +1,6 @@ import { app } from "../../scripts/app.js"; import { api } from "../../scripts/api.js"; -import { promptEnhanceHandler, syncDimensionsNodeHandler, searchNodeHandler, APIKeyHandler, captionNodeHandler, handleCustomErrors } from "./utils.js"; +import { promptEnhanceHandler, syncDimensionsNodeHandler, searchNodeHandler, APIKeyHandler, captionNodeHandler, mediaUUIDHandler, handleCustomErrors } from "./utils.js"; import { RUNWARE_NODE_TYPES, RUNWARE_NODE_PROPS, SEARCH_TERMS } from "./types.js"; const nodeInitList = []; @@ -9,6 +9,7 @@ app.registerExtension({ async setup() { api.addEventListener('runwareError', handleCustomErrors); api.addEventListener('runwareImageCaption', captionNodeHandler); + api.addEventListener('runwareMediaUUID', mediaUUIDHandler); }, async nodeCreated(node) { diff --git a/clientlibs/types.js b/clientlibs/types.js index 9e40e8a..5d24174 100644 --- a/clientlibs/types.js +++ b/clientlibs/types.js @@ -88,6 +88,7 @@ const RUNWARE_NODE_TYPES = { ELEVENLABSPROVIDERSETTINGS: "Runware ElevenLabs Provider Settings", ELEVENLABSPROVIDERSETTINGSSECTIONS: "Runware ElevenLabs Provider Settings Sections", VIDUPROVIDERSETTINGS: "Runware Vidu Provider Settings", + MEDIAUPLOAD: "Runware Media Upload", }; const RUNWARE_NODE_PROPS = { @@ -254,6 +255,10 @@ const RUNWARE_NODE_PROPS = { bgColor: DEFAULT_BGCOLOR, colorModeOnly: true, }, + [RUNWARE_NODE_TYPES.MEDIAUPLOAD]: { + bgColor: DEFAULT_BGCOLOR, + colorModeOnly: true, + }, }; export { diff --git a/clientlibs/utils.js b/clientlibs/utils.js index db579f4..a1e63a5 100644 --- a/clientlibs/utils.js +++ b/clientlibs/utils.js @@ -40,6 +40,24 @@ const runwareLocalAPI = { queryLocalAPI('modelSearch', { modelQuery, modelArch, modelType, modelCat, condtioning }) }; +function mediaUUIDHandler(msgEvent) { + const mediaData = msgEvent.detail; + const mediaUUID = mediaData.mediaUUID; + const mediaNodeID = parseInt(mediaData.nodeID); + + if(mediaData.success) { + const mediaNode = app.graph.getNodeById(mediaNodeID); + if(mediaNode !== null && mediaNode !== undefined) { + // Find the mediaUUID widget specifically + const mediaUUIDWidget = mediaNode.widgets.find(widget => widget.name === "mediaUUID"); + if(mediaUUIDWidget && mediaUUIDWidget.inputEl) { + mediaUUIDWidget.inputEl.value = mediaUUID; + } + } + } + return false; +} + function captionNodeHandler(msgEvent) { const captionData = msgEvent.detail; const captionText = captionData.captionText; @@ -574,6 +592,7 @@ export { promptEnhanceHandler, syncDimensionsNodeHandler, searchNodeHandler, + mediaUUIDHandler, captionNodeHandler, handleCustomErrors, APIKeyHandler, diff --git a/modules/mediaUpload.py b/modules/mediaUpload.py new file mode 100644 index 0000000..c974a4e --- /dev/null +++ b/modules/mediaUpload.py @@ -0,0 +1,336 @@ +import base64 +import os +import uuid +import time +import requests +import torch +import torchaudio +import numpy as np +from io import BytesIO +from .utils.runwareUtils import ( + RUNWARE_API_KEY, + RUNWARE_API_BASE_URL, + generalRequestWrapper, + genRandUUID, + genRandSeed, + sendMediaUUID, +) + +class RunwareMediaUpload: + def __init__(self): + pass + + @classmethod + def INPUT_TYPES(cls): + return { + "required": {}, + "optional": { + "media": ("VIDEO,AUDIO", { + "tooltip": "Media tensor from ComfyUI load video or audio node", + }), + "mediaUUID": ("STRING", { + "multiline": True, + "placeholder": "Generated media UUID will appear here automatically.", + "tooltip": "This field will be automatically populated with the generated media UUID after upload.", + }), + }, + "hidden": { "node_id": "UNIQUE_ID" } + } + + RETURN_TYPES = ("STRING",) + RETURN_NAMES = ("mediaUUID",) + FUNCTION = "uploadMedia" + CATEGORY = "Runware" + DESCRIPTION = "Upload audio media to Runware's media storage and get mediaUUID" + OUTPUT_NODE = True + + @classmethod + def IS_CHANGED(s, **kwargs): + return float("NAN") + + def uploadMedia(self, media=None, mediaUUID=None, **kwargs): + print(f"[Debug] uploadMedia called with media: {type(media)}, mediaUUID: {mediaUUID}") + if media is None: + raise ValueError("Media input is required") + + try: + # Convert media tensor to base64 + print(f"[Debug] Converting media to base64...") + media_base64 = self._convert_media_to_base64(media) + print(f"[Debug] Media converted to base64, length: {len(media_base64)}") + + # Upload to Runware + print(f"[Debug] Starting upload to Runware...") + media_uuid = self._upload_to_runware(media_base64) + print(f"[Debug] Upload completed! MediaUUID: {media_uuid}") + print(f"[Debug] ===== RESULT: {media_uuid} =====") + + # Send the mediaUUID back to the UI + sendMediaUUID(media_uuid, kwargs.get("node_id")) + + return (media_uuid,) + + except Exception as e: + print(f"[Debug] Media upload failed: {str(e)}") + raise e + + def _convert_media_to_base64(self, media_tensor): + """Convert media tensor to base64 data URI""" + try: + # Handle ComfyUI VIDEO objects (VideoFromFile) + print(f"[Debug] Media type: {type(media_tensor)}") + print(f"[Debug] Media attributes: {dir(media_tensor)}") + + # Check if this is a ComfyUI VideoFromFile object + if hasattr(media_tensor, 'video_path') or str(type(media_tensor)).find('VideoFromFile') != -1: + # This is a ComfyUI VideoFromFile object + print(f"[Debug] VideoFromFile object detected") + + # Try to get the file path from the private attribute + video_file = getattr(media_tensor, '_VideoFromFile__file', None) + if video_file is None: + # Try other possible attributes + for attr in ['video_path', 'path', 'file_path', 'filename', '_file']: + if hasattr(media_tensor, attr): + video_file = getattr(media_tensor, attr) + print(f"[Debug] Found video file via {attr}: {video_file}") + break + + if video_file is None: + raise ValueError("Could not find video file in VideoFromFile object") + + # If it's a file object, get the path + if hasattr(video_file, 'name'): + video_path = video_file.name + elif isinstance(video_file, str): + video_path = video_file + else: + raise ValueError(f"Unexpected video file type: {type(video_file)}") + + print(f"[Debug] Processing video file: {video_path}") + + # Read the video file and convert to base64 + with open(video_path, 'rb') as f: + video_bytes = f.read() + + # Detect MIME type based on file extension + if video_path.lower().endswith('.mp4'): + mime_type = 'video/mp4' + elif video_path.lower().endswith('.avi'): + mime_type = 'video/avi' + elif video_path.lower().endswith('.mov'): + mime_type = 'video/quicktime' + elif video_path.lower().endswith('.webm'): + mime_type = 'video/webm' + else: + mime_type = 'video/mp4' # Default + + # Encode to base64 + video_base64 = base64.b64encode(video_bytes).decode('utf-8') + + # Return as data URI + return f"data:{mime_type};base64,{video_base64}" + + # ComfyUI AUDIO objects are dictionaries with 'waveform' and 'sample_rate' + elif isinstance(media_tensor, dict): + if 'waveform' in media_tensor: + waveform = media_tensor['waveform'] + sample_rate = media_tensor.get('sample_rate', 22050) + + # Convert tensor to proper format for torchaudio + if isinstance(waveform, torch.Tensor): + # Ensure waveform is in the correct format [channels, samples] + if waveform.dim() == 3: + waveform = waveform.squeeze(0) # Remove batch dimension + elif waveform.dim() == 1: + waveform = waveform.unsqueeze(0) # Add channel dimension + else: + waveform = torch.tensor(waveform) + if waveform.dim() == 1: + waveform = waveform.unsqueeze(0) + + # Create WAV file in memory + buffer = BytesIO() + torchaudio.save(buffer, waveform, sample_rate, format="wav") + buffer.seek(0) + + # Get the WAV file bytes + wav_bytes = buffer.getvalue() + + # Encode to base64 + wav_base64 = base64.b64encode(wav_bytes).decode('utf-8') + + # Return as data URI + return f"data:audio/wav;base64,{wav_base64}" + else: + raise ValueError("Invalid audio object: missing 'waveform' key") + else: + # Fallback for direct tensor input + if isinstance(media_tensor, torch.Tensor): + waveform = media_tensor + sample_rate = 22050 # Default sample rate + + # Ensure waveform is in the correct format [channels, samples] + if waveform.dim() == 3: + waveform = waveform.squeeze(0) # Remove batch dimension + elif waveform.dim() == 1: + waveform = waveform.unsqueeze(0) # Add channel dimension + else: + waveform = torch.tensor(media_tensor) + sample_rate = 22050 + if waveform.dim() == 1: + waveform = waveform.unsqueeze(0) + + # Create WAV file in memory + buffer = BytesIO() + torchaudio.save(buffer, waveform, sample_rate, format="wav") + buffer.seek(0) + + # Get the WAV file bytes + wav_bytes = buffer.getvalue() + + # Encode to base64 + wav_base64 = base64.b64encode(wav_bytes).decode('utf-8') + + # Return as data URI + return f"data:audio/wav;base64,{wav_base64}" + + except Exception as e: + print(f"[Debug] Error converting media to base64: {str(e)}") + raise e + + def _upload_to_runware(self, media_data_uri): + """Upload media to Runware and return mediaUUID""" + global RUNWARE_API_KEY, RUNWARE_API_BASE_URL + + task_uuid = genRandUUID() + print(f"[Debug] Generated task UUID: {task_uuid}") + + # Strip data URI prefix for media storage API + if media_data_uri.startswith("data:"): + media_data = media_data_uri.split(",", 1)[1] + print(f"[Debug] Stripped data URI prefix, media data length: {len(media_data)}") + else: + media_data = media_data_uri + + upload_config = [ + { + "taskType": "mediaStorage", + "taskUUID": task_uuid, + "operation": "upload", + "media": media_data, + } + ] + + print(f"[Debug] Upload config: {upload_config[0]}") + print(f"[Debug] API URL: {RUNWARE_API_BASE_URL}") + + headers = { + "Authorization": f"Bearer {RUNWARE_API_KEY}", + "Content-Type": "application/json", + "Accept-Encoding": "gzip, deflate, br, zstd", + } + + try: + def recaller(): + print(f"[Debug] Making POST request to Runware API...") + return requests.post( + RUNWARE_API_BASE_URL, + headers=headers, + json=upload_config, + timeout=30, + allow_redirects=False, + stream=True, + ) + + upload_result = generalRequestWrapper(recaller) + upload_result = upload_result.json() + print(f"[Debug] Upload response: {upload_result}") + + if "errors" in upload_result: + print(f"[Debug] Upload error: {upload_result}") + raise Exception(f"Upload failed: {upload_result}") + + # Check if we got immediate result or need to poll + if upload_result.get("data") and len(upload_result["data"]) > 0: + data = upload_result["data"][0] + print(f"[Debug] Upload data: {data}") + if "mediaUUID" in data and data["mediaUUID"] != "1": + print(f"[Debug] Got immediate mediaUUID: {data['mediaUUID']}") + return data["mediaUUID"] + + # Poll for result + print(f"[Debug] Starting polling for result...") + return self._poll_for_result(task_uuid) + + except Exception as e: + print(f"[Debug] Upload request failed: {str(e)}") + raise e + + def _poll_for_result(self, task_uuid): + """Poll for media upload result""" + global RUNWARE_API_KEY, RUNWARE_API_BASE_URL + + poll_config = [ + { + "taskType": "getResponse", + "taskUUID": task_uuid, + } + ] + + headers = { + "Authorization": f"Bearer {RUNWARE_API_KEY}", + "Content-Type": "application/json", + "Accept-Encoding": "gzip, deflate, br, zstd", + } + + max_attempts = 100 + for attempt in range(max_attempts): + try: + print(f"[Debug] Poll attempt {attempt + 1}/{max_attempts}") + def recaller(): + return requests.post( + RUNWARE_API_BASE_URL, + headers=headers, + json=poll_config, + timeout=30, + allow_redirects=False, + stream=True, + ) + + poll_result = generalRequestWrapper(recaller) + poll_result = poll_result.json() + print(f"[Debug] Poll response: {poll_result}") + + if "errors" in poll_result: + print(f"[Debug] Poll error: {poll_result}") + continue + + if "data" in poll_result and len(poll_result["data"]) > 0: + data = poll_result["data"][0] + print(f"[Debug] Poll data: {data}") + + # Check if mediaUUID is still "1" (still processing) + if "mediaUUID" in data and data["mediaUUID"] == "1": + print(f"[Debug] Still processing... attempt {attempt + 1}") + time.sleep(2) + continue + + # Check for completion - look for mediaUUID + if "mediaUUID" in data and data["mediaUUID"] != "1": + print(f"[Debug] Upload completed! MediaUUID: {data['mediaUUID']}") + return data["mediaUUID"] + + print(f"[Debug] Attempt {attempt + 1}: No data in response, continuing...") + time.sleep(2) + + except Exception as e: + print(f"[Debug] Poll attempt {attempt + 1} failed: {str(e)}") + time.sleep(2) + continue + + raise Exception("Polling timeout - upload did not complete") + + +# Export the class directly +runwareMediaUpload = RunwareMediaUpload diff --git a/modules/utils/runwareUtils.py b/modules/utils/runwareUtils.py index ebf3150..07492f3 100644 --- a/modules/utils/runwareUtils.py +++ b/modules/utils/runwareUtils.py @@ -233,6 +233,17 @@ def checkAPIKey(apiKey): except Exception as e: return False +def sendMediaUUID(mediaUUID, nodeID): + PromptServer.instance.send_sync( + "runwareMediaUUID", + { + "success": True, + "mediaUUID": mediaUUID, + "nodeID": nodeID, + "widgetName": "mediaUUID", # Target the mediaUUID widget + }, + ) + def sendImageCaption(captionText, nodeID): PromptServer.instance.send_sync( "runwareImageCaption", diff --git a/workflows/Runware_Media_Upload.json b/workflows/Runware_Media_Upload.json new file mode 100644 index 0000000..be4c2b7 --- /dev/null +++ b/workflows/Runware_Media_Upload.json @@ -0,0 +1 @@ +{"id":"7493d64d-0c0c-40ca-a1df-816552f0526b","revision":0,"last_node_id":6,"last_link_id":3,"nodes":[{"id":3,"type":"LoadVideo","pos":[252.06771850585938,646.0390625],"size":[274.080078125,279.56005859375],"flags":{},"order":0,"mode":0,"inputs":[{"localized_name":"file","name":"file","type":"COMBO","widget":{"name":"file"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"VIDEO","name":"VIDEO","type":"VIDEO","links":[3]}],"properties":{"cnr_id":"comfy-core","ver":"0.3.46","Node name for S&R":"LoadVideo"},"widgets_values":["test_video.mp4","image"]},{"id":6,"type":"Runware Media Upload","pos":[558.8807373046875,651.7319946289062],"size":[400,200],"flags":{},"order":1,"mode":0,"inputs":[{"localized_name":"media","name":"media","shape":7,"type":"VIDEO,AUDIO","link":3},{"localized_name":"mediaUUID","name":"mediaUUID","shape":7,"type":"STRING","widget":{"name":"mediaUUID"},"link":null}],"outputs":[{"localized_name":"mediaUUID","name":"mediaUUID","type":"STRING","links":null}],"properties":{"cnr_id":"ComfyUI-Runware","ver":"16016793b1e66f9b7bf364643bdf36447a35d5c9","Node name for S&R":"Runware Media Upload"},"widgets_values":["254f37d6-98dc-4333-a530-232ea5847cce"],"bgcolor":"#5345bf"}],"links":[[3,3,0,6,0,"VIDEO,AUDIO"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.8264462809917354,"offset":[439.5536367488321,-167.09876328463446]}},"version":0.4} \ No newline at end of file