From 30dcf89ec6549aedbf3db6073ba5f88e70fe9690 Mon Sep 17 00:00:00 2001 From: Sirshendu Ganguly Date: Mon, 27 Oct 2025 14:41:58 -0400 Subject: [PATCH] Add video transcription, fix model names --- .DS_Store | Bin 8196 -> 8196 bytes __init__.py | 2 + clientlibs/main.js | 3 +- clientlibs/utils.js | 19 +++++ modules/frameImages.py | 10 +-- modules/utils/runwareUtils.py | 11 +++ modules/videoInference.py | 75 +------------------ modules/videoModelSearch.py | 4 +- modules/videoTranscription.py | 132 ++++++++++++++++++++++++++++++++++ 9 files changed, 172 insertions(+), 84 deletions(-) create mode 100644 modules/videoTranscription.py diff --git a/.DS_Store b/.DS_Store index efed26ea8e05f69186f2b85f6a227d1e46336079..14caf09b5a7cb6581314300201d42d69e78755f8 100644 GIT binary patch delta 197 zcmZp1XmQw}EXc$YF}YHtS|(mVptLwOC^aXsB(p5lC9x#6I3hDSKQB15Ds}T7!9|Q* zoXQRc3}7(XUrdy7_2fF?*^J?nr9@(xc)SUlAs}js#f((3M#fj0pNomH31hPuzb08m E06|AOoB#j- delta 197 zcmZp1XmQw}EXWkjGr3ZvS|%bhIX^Ervnn-SK%lfZH7GSFu_UuB)g`ecwRrO$!9|Q* z$_@q$V8A)qUrdy7;p95u*^Kd%r9@(xyofMEK-3hA8L488jGs0?7ZYO>#%3{oO|pyt DDAGCS diff --git a/__init__.py b/__init__.py index 764c498..9c08db2 100644 --- a/__init__.py +++ b/__init__.py @@ -16,6 +16,7 @@ from .modules.imageMasking import imageMasking from .modules.controlNetPreprocessor import controlNetPreprocessor from .modules.apiManager import apiManager from .modules.imageCaptioning import imageCaptioning +from .modules.videoTranscription import videoTranscription from .modules.controlNetCombine import controlNetCombine from .modules.embeddingSearch import embeddingSearch from .modules.embeddingsCombine import embeddingsCombine @@ -81,6 +82,7 @@ NODE_CLASS_MAPPINGS = { "Runware ControlNet PreProcessor": controlNetPreprocessor, "Runware API Manager": apiManager, "Runware Image Caption": imageCaptioning, + "Runware Video Transcription": videoTranscription, "Runware ControlNet Combine": controlNetCombine, "Runware IPAdapter": ipAdapter, "Runware IPAdapters Combine": ipAdapterCombine, diff --git a/clientlibs/main.js b/clientlibs/main.js index 6fe9ecb..650cc46 100644 --- a/clientlibs/main.js +++ b/clientlibs/main.js @@ -1,6 +1,6 @@ import { app } from "../../scripts/app.js"; import { api } from "../../scripts/api.js"; -import { promptEnhanceHandler, syncDimensionsNodeHandler, searchNodeHandler, APIKeyHandler, captionNodeHandler, mediaUUIDHandler, handleCustomErrors } from "./utils.js"; +import { promptEnhanceHandler, syncDimensionsNodeHandler, searchNodeHandler, APIKeyHandler, captionNodeHandler, mediaUUIDHandler, videoTranscriptionHandler, handleCustomErrors } from "./utils.js"; import { RUNWARE_NODE_TYPES, RUNWARE_NODE_PROPS, SEARCH_TERMS } from "./types.js"; const nodeInitList = []; @@ -10,6 +10,7 @@ app.registerExtension({ api.addEventListener('runwareError', handleCustomErrors); api.addEventListener('runwareImageCaption', captionNodeHandler); api.addEventListener('runwareMediaUUID', mediaUUIDHandler); + api.addEventListener('runwareVideoTranscription', videoTranscriptionHandler); }, async nodeCreated(node) { diff --git a/clientlibs/utils.js b/clientlibs/utils.js index dd05aa6..06b6811 100644 --- a/clientlibs/utils.js +++ b/clientlibs/utils.js @@ -88,6 +88,24 @@ function captionNodeHandler(msgEvent) { return false; } +function videoTranscriptionHandler(msgEvent) { + const transcriptionData = msgEvent.detail; + const transcriptionText = transcriptionData.transcriptionText; + const transcriptionNodeID = parseInt(transcriptionData.nodeID); + + if(transcriptionData.success) { + const transcriptionNode = app.graph.getNodeById(transcriptionNodeID); + if(transcriptionNode !== null && transcriptionNode !== undefined) { + // Find the prompt widget (where transcription is displayed) + const promptWidget = transcriptionNode.widgets.find(widget => widget.name === "prompt"); + if(promptWidget && promptWidget.inputEl) { + promptWidget.inputEl.value = transcriptionText; + } + } + } + return false; +} + function notifyUser(message, type="info", title = "Runware", life = 4.5) { app.extensionManager.toast.add({ severity: type, // 'info', 'success', 'warn', 'error' \\ @@ -606,6 +624,7 @@ export { searchNodeHandler, mediaUUIDHandler, captionNodeHandler, + videoTranscriptionHandler, handleCustomErrors, APIKeyHandler, }; \ No newline at end of file diff --git a/modules/frameImages.py b/modules/frameImages.py index 1ec0fd0..23427d1 100644 --- a/modules/frameImages.py +++ b/modules/frameImages.py @@ -56,17 +56,13 @@ class RunwareFrameImages: position = kwargs.get(position_key, "auto") if image is not None: - # Upload image and get UUID - image_uuid = rwUtils.convertTensor2IMGForVideo(image) - image_url = f"https://im.runware.ai/image/ii/{image_uuid}.webp" + # Convert image to data URI or UUID + image_data = rwUtils.convertTensor2IMG(image) - frame_data = { - "inputImage": image_url - } + frame_data = {"inputImage": image_data} # Only add frame position if not auto if position != "auto": - # Convert string numbers to integers for API if position.isdigit(): frame_data["frame"] = int(position) else: diff --git a/modules/utils/runwareUtils.py b/modules/utils/runwareUtils.py index 07492f3..75da673 100644 --- a/modules/utils/runwareUtils.py +++ b/modules/utils/runwareUtils.py @@ -255,6 +255,17 @@ def sendImageCaption(captionText, nodeID): }, ) +def sendVideoTranscription(transcriptionText, nodeID): + PromptServer.instance.send_sync( + "runwareVideoTranscription", + { + "success": True, + "transcriptionText": transcriptionText, + "nodeID": nodeID, + "widgetName": "prompt", # Target the prompt widget (matching imageCaptioning pattern) + }, + ) + def inferenecRequest(genConfig): global RUNWARE_API_KEY, RUNWARE_API_BASE_URL, SESSION_TIMEOUT RUNWARE_API_KEY = os.getenv("RUNWARE_API_KEY") diff --git a/modules/videoInference.py b/modules/videoInference.py index 1ff0804..b0219d4 100644 --- a/modules/videoInference.py +++ b/modules/videoInference.py @@ -1,81 +1,10 @@ from .utils import runwareUtils as rwUtils from .videoModelSearch import videoModelSearch +from .frameImages import RunwareFrameImages import json import comfy.model_management -class RunwareFrameImages: - @classmethod - def INPUT_TYPES(cls): - return { - "required": {}, - "optional": { - "image1": ("IMAGE", { - "tooltip": "First frame image. When only one image is provided, it becomes the first frame automatically.", - }), - "frame1_position": (["auto", "first", "last"], { - "default": "auto", - "tooltip": "Position for the first image. 'auto' uses automatic distribution rules.", - }), - "image2": ("IMAGE", { - "tooltip": "Second frame image. With two images, they become first and last frames automatically.", - }), - "frame2_position": (["auto", "first", "last"], { - "default": "auto", - "tooltip": "Position for the second image. 'auto' uses automatic distribution rules.", - }), - "image3": ("IMAGE", { - "tooltip": "Third frame image. Will be distributed between first and last frames.", - }), - "frame3_position": (["auto", "first", "last"], { - "default": "auto", - "tooltip": "Position for the third image. 'auto' uses automatic distribution rules.", - }), - "image4": ("IMAGE", { - "tooltip": "Fourth frame image. Will be distributed between first and last frames.", - }), - "frame4_position": (["auto", "first", "last"], { - "default": "auto", - "tooltip": "Position for the fourth image. 'auto' uses automatic distribution rules.", - }), - } - } - - DESCRIPTION = "Configure frame images for video generation with precise positioning control. Supports automatic distribution or manual positioning." - FUNCTION = "create_frame_images" - RETURN_TYPES = ("RUNWAREFRAMEIMAGES",) - RETURN_NAMES = ("Frame Images",) - CATEGORY = "Runware" - - def create_frame_images(self, **kwargs): - frame_images = [] - - # Process each image input - for i in range(1, 5): # Support up to 4 images - image_key = f"image{i}" - position_key = f"frame{i}_position" - - image = kwargs.get(image_key) - position = kwargs.get(position_key, "auto") - - if image is not None: - # Upload image and get UUID - image_uuid = rwUtils.convertTensor2IMGForVideo(image) - image_url = f"https://im.runware.ai/image/ii/{image_uuid}.webp" - - frame_data = { - "inputImage": image_url - } - - # Only add frame position if not auto - if position != "auto": - frame_data["frame"] = position - - frame_images.append(frame_data) - - return (frame_images,) - - class txt2vid: @classmethod def INPUT_TYPES(cls): @@ -439,11 +368,9 @@ class txt2vid: # Node class mappings for ComfyUI registration NODE_CLASS_MAPPINGS = { - "RunwareFrameImages": RunwareFrameImages, "txt2vid": txt2vid, } NODE_DISPLAY_NAME_MAPPINGS = { - "RunwareFrameImages": "Runware Frame Images", "txt2vid": "Runware Video Inference", } \ No newline at end of file diff --git a/modules/videoModelSearch.py b/modules/videoModelSearch.py index 37f6641..75c2898 100644 --- a/modules/videoModelSearch.py +++ b/modules/videoModelSearch.py @@ -18,7 +18,7 @@ class videoModelSearch: "google:3@0 (Veo 3.0)", "google:3@1 (Veo 3.0 Fast)", "google:3@2 (Veo 3.1)", - "google:3@3 (Veo 3.1 Pro)", + "google:3@3 (Veo 3.1 Fast)", ], "Seedance": [ "bytedance:2@1 (Seedance 1.0 Pro)", @@ -77,7 +77,7 @@ class videoModelSearch: "google:3@0": {"width": 1280, "height": 720}, # Veo 3.0 "google:3@1": {"width": 1280, "height": 720}, # Veo 3.0 Fast "google:3@2": {"width": 1280, "height": 720}, # Veo 3.1 - "google:3@3": {"width": 1280, "height": 720}, # Veo 3.1 Pro + "google:3@3": {"width": 1280, "height": 720}, # Veo 3.1 Fast # Seedance Models "bytedance:2@1": {"width": 864, "height": 480}, # Seedance 1.0 Pro diff --git a/modules/videoTranscription.py b/modules/videoTranscription.py new file mode 100644 index 0000000..88e39df --- /dev/null +++ b/modules/videoTranscription.py @@ -0,0 +1,132 @@ +from .utils import runwareUtils as rwUtils +import json +import comfy.model_management + + +class videoTranscription: + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "Always Retranscribe": ("BOOLEAN", { + "default": False, + "tooltip": "Enable this option to always retranscribe the video each time you run the workflow.", + "label_on": "Enabled", + "label_off": "Disabled", + }), + "prompt": ("STRING", { + "multiline": True, + "default": "", + "placeholder": "Video transcription will appear here automatically.", + "tooltip": "The video transcription will be loaded here automatically." + }), + "video": ("STRING", { + "default": "", + "placeholder": "Enter video URL, file path, or Runware video UUID", + "tooltip": "The video to transcribe. Can be a URL, file path, or Runware video UUID." + }), + "model": ("STRING", { + "default": "memories:1@1", + "tooltip": "The model to use for video transcription. Default: memories:1@1" + }), + }, + "hidden": { "node_id": "UNIQUE_ID" } + } + + DESCRIPTION = "Transcribe spoken audio and visual context from videos into accurate text with speaker labeling and optional chapter summaries." + + FUNCTION = "transcribeVideo" + RETURN_TYPES = ("STRING",) + RETURN_NAMES = ("Video Transcription",) + CATEGORY = "Runware" + OUTPUT_NODE = True + + @classmethod + def IS_CHANGED(s, **kwargs): + alwaysRetranscribe = kwargs.get("Always Retranscribe") + if(alwaysRetranscribe): + return float("NAN") + return True + + def transcribeVideo(self, **kwargs): + video = kwargs.get("video", "") + model = kwargs.get("model", "memories:1@1") + + if not video or not video.strip(): + raise Exception("Video URL, file path, or UUID is required") + + # Create task parameters with correct structure + task_params = { + "taskType": "caption", + "taskUUID": rwUtils.genRandUUID(), + "model": model, + "inputs": { + "video": video + } + } + + # Send the task + genConfig = [task_params] + + # Debug: Print the request being sent + print(f"[DEBUG] Sending Video Transcription Request:") + print(f"[DEBUG] Request Payload: {json.dumps(genConfig, indent=2)}") + + try: + genResult = rwUtils.inferenecRequest(genConfig) + + # Debug: Print the response received + print(f"[DEBUG] Received Video Transcription Response:") + print(f"[DEBUG] Response: {json.dumps(genResult, indent=2)}") + except Exception as e: + raise e + + # Get task UUID for polling + taskUUID = task_params["taskUUID"] + + # Poll for result + while True: + # Check for interrupt before each poll + comfy.model_management.throw_exception_if_processing_interrupted() + + # Poll for result + pollResult = rwUtils.pollVideoResult(taskUUID) + print(f"[Debugging] Poll result: {pollResult}") + + # Check for errors first + if pollResult and "errors" in pollResult and len(pollResult["errors"]) > 0: + error_info = pollResult["errors"][0] + error_message = error_info.get("message", "Unknown error") + raise Exception(f"Video transcription failed: {error_message}") + + # Check if we have data + if pollResult and "data" in pollResult and len(pollResult["data"]) > 0: + data = pollResult["data"][0] + + # Check status + if "status" in data: + status = data["status"] + + if status == "success": + # Check if transcription is complete + if "text" in data: + transcriptionText = data["text"] + + # Send transcription result + rwUtils.sendVideoTranscription(transcriptionText, kwargs.get("node_id")) + return (transcriptionText,) + elif status == "processing": + # Continue polling + pass + else: + # Error or other status + raise Exception(f"Unexpected status: {status}") + + # Check for interrupt before waiting + comfy.model_management.throw_exception_if_processing_interrupted() + + # Wait before next poll + for _ in range(10): # 10 x 0.1 second = 1 second total + comfy.model_management.throw_exception_if_processing_interrupted() + rwUtils.time.sleep(0.1) +