nodes/Runware-ComfyUI-Runware
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add video transcription, fix model names

Browse Source
This commit is contained in:
Sirshendu Ganguly
2025-10-27 14:41:58 -04:00
parent 056c15a1b7
commit 30dcf89ec6
9 changed files with 172 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.DS_Store
Vendored
BIN
View File
Binary file not shown.
__init__.py
+2
View File
@@ -16,6 +16,7 @@ from .modules.imageMasking import imageMasking
from .modules.controlNetPreprocessor import controlNetPreprocessor
from .modules.apiManager import apiManager
from .modules.imageCaptioning import imageCaptioning
from .modules.videoTranscription import videoTranscription
from .modules.controlNetCombine import controlNetCombine
from .modules.embeddingSearch import embeddingSearch
from .modules.embeddingsCombine import embeddingsCombine
@@ -81,6 +82,7 @@ NODE_CLASS_MAPPINGS = {
"Runware ControlNet PreProcessor": controlNetPreprocessor,
"Runware API Manager": apiManager,
"Runware Image Caption": imageCaptioning,
"Runware Video Transcription": videoTranscription,
"Runware ControlNet Combine": controlNetCombine,
"Runware IPAdapter": ipAdapter,
"Runware IPAdapters Combine": ipAdapterCombine,
clientlibs/main.js
+2 -1
View File
@@ -1,6 +1,6 @@
import { app } from "../../scripts/app.js";
import { api } from "../../scripts/api.js";
import { promptEnhanceHandler, syncDimensionsNodeHandler, searchNodeHandler, APIKeyHandler, captionNodeHandler, mediaUUIDHandler, handleCustomErrors } from "./utils.js";
import { promptEnhanceHandler, syncDimensionsNodeHandler, searchNodeHandler, APIKeyHandler, captionNodeHandler, mediaUUIDHandler, videoTranscriptionHandler, handleCustomErrors } from "./utils.js";
import { RUNWARE_NODE_TYPES, RUNWARE_NODE_PROPS, SEARCH_TERMS } from "./types.js";
const nodeInitList = [];
@@ -10,6 +10,7 @@ app.registerExtension({
api.addEventListener('runwareError', handleCustomErrors);
api.addEventListener('runwareImageCaption', captionNodeHandler);
api.addEventListener('runwareMediaUUID', mediaUUIDHandler);
api.addEventListener('runwareVideoTranscription', videoTranscriptionHandler);
},
async nodeCreated(node) {
clientlibs/utils.js
+19
View File
@@ -88,6 +88,24 @@ function captionNodeHandler(msgEvent) {
return false;
}
function videoTranscriptionHandler(msgEvent) {
const transcriptionData = msgEvent.detail;
const transcriptionText = transcriptionData.transcriptionText;
const transcriptionNodeID = parseInt(transcriptionData.nodeID);
if(transcriptionData.success) {
const transcriptionNode = app.graph.getNodeById(transcriptionNodeID);
if(transcriptionNode !== null && transcriptionNode !== undefined) {
// Find the prompt widget (where transcription is displayed)
const promptWidget = transcriptionNode.widgets.find(widget => widget.name === "prompt");
if(promptWidget && promptWidget.inputEl) {
promptWidget.inputEl.value = transcriptionText;
}
}
}
return false;
}
function notifyUser(message, type="info", title = "Runware", life = 4.5) {
app.extensionManager.toast.add({
severity: type, // 'info', 'success', 'warn', 'error' \\
@@ -606,6 +624,7 @@ export {
searchNodeHandler,
mediaUUIDHandler,
captionNodeHandler,
videoTranscriptionHandler,
handleCustomErrors,
APIKeyHandler,
};
modules/frameImages.py
+3 -7
View File
@@ -56,17 +56,13 @@ class RunwareFrameImages:
position = kwargs.get(position_key, "auto")
if image is not None:
# Upload image and get UUID
image_uuid = rwUtils.convertTensor2IMGForVideo(image)
image_url = f"https://im.runware.ai/image/ii/{image_uuid}.webp"
# Convert image to data URI or UUID
image_data = rwUtils.convertTensor2IMG(image)
frame_data = {
"inputImage": image_url
}
frame_data = {"inputImage": image_data}
# Only add frame position if not auto
if position != "auto":
# Convert string numbers to integers for API
if position.isdigit():
frame_data["frame"] = int(position)
else:
modules/utils/runwareUtils.py
+11
View File
@@ -255,6 +255,17 @@ def sendImageCaption(captionText, nodeID):
},
)
def sendVideoTranscription(transcriptionText, nodeID):
PromptServer.instance.send_sync(
"runwareVideoTranscription",
{
"success": True,
"transcriptionText": transcriptionText,
"nodeID": nodeID,
"widgetName": "prompt", # Target the prompt widget (matching imageCaptioning pattern)
},
)
def inferenecRequest(genConfig):
global RUNWARE_API_KEY, RUNWARE_API_BASE_URL, SESSION_TIMEOUT
RUNWARE_API_KEY = os.getenv("RUNWARE_API_KEY")
modules/videoInference.py
+1 -74
View File
@@ -1,81 +1,10 @@
from .utils import runwareUtils as rwUtils
from .videoModelSearch import videoModelSearch
from .frameImages import RunwareFrameImages
import json
import comfy.model_management
class RunwareFrameImages:
@classmethod
def INPUT_TYPES(cls):
return {
"required": {},
"optional": {
"image1": ("IMAGE", {
"tooltip": "First frame image. When only one image is provided, it becomes the first frame automatically.",
}),
"frame1_position": (["auto", "first", "last"], {
"default": "auto",
"tooltip": "Position for the first image. 'auto' uses automatic distribution rules.",
}),
"image2": ("IMAGE", {
"tooltip": "Second frame image. With two images, they become first and last frames automatically.",
}),
"frame2_position": (["auto", "first", "last"], {
"default": "auto",
"tooltip": "Position for the second image. 'auto' uses automatic distribution rules.",
}),
"image3": ("IMAGE", {
"tooltip": "Third frame image. Will be distributed between first and last frames.",
}),
"frame3_position": (["auto", "first", "last"], {
"default": "auto",
"tooltip": "Position for the third image. 'auto' uses automatic distribution rules.",
}),
"image4": ("IMAGE", {
"tooltip": "Fourth frame image. Will be distributed between first and last frames.",
}),
"frame4_position": (["auto", "first", "last"], {
"default": "auto",
"tooltip": "Position for the fourth image. 'auto' uses automatic distribution rules.",
}),
}
}
DESCRIPTION = "Configure frame images for video generation with precise positioning control. Supports automatic distribution or manual positioning."
FUNCTION = "create_frame_images"
RETURN_TYPES = ("RUNWAREFRAMEIMAGES",)
RETURN_NAMES = ("Frame Images",)
CATEGORY = "Runware"
def create_frame_images(self, **kwargs):
frame_images = []
# Process each image input
for i in range(1, 5): # Support up to 4 images
image_key = f"image{i}"
position_key = f"frame{i}_position"
image = kwargs.get(image_key)
position = kwargs.get(position_key, "auto")
if image is not None:
# Upload image and get UUID
image_uuid = rwUtils.convertTensor2IMGForVideo(image)
image_url = f"https://im.runware.ai/image/ii/{image_uuid}.webp"
frame_data = {
"inputImage": image_url
}
# Only add frame position if not auto
if position != "auto":
frame_data["frame"] = position
frame_images.append(frame_data)
return (frame_images,)
class txt2vid:
@classmethod
def INPUT_TYPES(cls):
@@ -439,11 +368,9 @@ class txt2vid:
# Node class mappings for ComfyUI registration
NODE_CLASS_MAPPINGS = {
"RunwareFrameImages": RunwareFrameImages,
"txt2vid": txt2vid,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"RunwareFrameImages": "Runware Frame Images",
"txt2vid": "Runware Video Inference",
}
modules/videoModelSearch.py
+2 -2
View File
@@ -18,7 +18,7 @@ class videoModelSearch:
"google:3@0 (Veo 3.0)",
"google:3@1 (Veo 3.0 Fast)",
"google:3@2 (Veo 3.1)",
"google:3@3 (Veo 3.1 Pro)",
"google:3@3 (Veo 3.1 Fast)",
],
"Seedance": [
"bytedance:2@1 (Seedance 1.0 Pro)",
@@ -77,7 +77,7 @@ class videoModelSearch:
"google:3@0": {"width": 1280, "height": 720}, # Veo 3.0
"google:3@1": {"width": 1280, "height": 720}, # Veo 3.0 Fast
"google:3@2": {"width": 1280, "height": 720}, # Veo 3.1
"google:3@3": {"width": 1280, "height": 720}, # Veo 3.1 Pro
"google:3@3": {"width": 1280, "height": 720}, # Veo 3.1 Fast
# Seedance Models
"bytedance:2@1": {"width": 864, "height": 480}, # Seedance 1.0 Pro
modules/videoTranscription.py
+132
View File
@@ -0,0 +1,132 @@
from .utils import runwareUtils as rwUtils
import json
import comfy.model_management
class videoTranscription:
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"Always Retranscribe": ("BOOLEAN", {
"default": False,
"tooltip": "Enable this option to always retranscribe the video each time you run the workflow.",
"label_on": "Enabled",
"label_off": "Disabled",
}),
"prompt": ("STRING", {
"multiline": True,
"default": "",
"placeholder": "Video transcription will appear here automatically.",
"tooltip": "The video transcription will be loaded here automatically."
}),
"video": ("STRING", {
"default": "",
"placeholder": "Enter video URL, file path, or Runware video UUID",
"tooltip": "The video to transcribe. Can be a URL, file path, or Runware video UUID."
}),
"model": ("STRING", {
"default": "memories:1@1",
"tooltip": "The model to use for video transcription. Default: memories:1@1"
}),
},
"hidden": { "node_id": "UNIQUE_ID" }
}
DESCRIPTION = "Transcribe spoken audio and visual context from videos into accurate text with speaker labeling and optional chapter summaries."
FUNCTION = "transcribeVideo"
RETURN_TYPES = ("STRING",)
RETURN_NAMES = ("Video Transcription",)
CATEGORY = "Runware"
OUTPUT_NODE = True
@classmethod
def IS_CHANGED(s, **kwargs):
alwaysRetranscribe = kwargs.get("Always Retranscribe")
if(alwaysRetranscribe):
return float("NAN")
return True
def transcribeVideo(self, **kwargs):
video = kwargs.get("video", "")
model = kwargs.get("model", "memories:1@1")
if not video or not video.strip():
raise Exception("Video URL, file path, or UUID is required")
# Create task parameters with correct structure
task_params = {
"taskType": "caption",
"taskUUID": rwUtils.genRandUUID(),
"model": model,
"inputs": {
"video": video
}
}
# Send the task
genConfig = [task_params]
# Debug: Print the request being sent
print(f"[DEBUG] Sending Video Transcription Request:")
print(f"[DEBUG] Request Payload: {json.dumps(genConfig, indent=2)}")
try:
genResult = rwUtils.inferenecRequest(genConfig)
# Debug: Print the response received
print(f"[DEBUG] Received Video Transcription Response:")
print(f"[DEBUG] Response: {json.dumps(genResult, indent=2)}")
except Exception as e:
raise e
# Get task UUID for polling
taskUUID = task_params["taskUUID"]
# Poll for result
while True:
# Check for interrupt before each poll
comfy.model_management.throw_exception_if_processing_interrupted()
# Poll for result
pollResult = rwUtils.pollVideoResult(taskUUID)
print(f"[Debugging] Poll result: {pollResult}")
# Check for errors first
if pollResult and "errors" in pollResult and len(pollResult["errors"]) > 0:
error_info = pollResult["errors"][0]
error_message = error_info.get("message", "Unknown error")
raise Exception(f"Video transcription failed: {error_message}")
# Check if we have data
if pollResult and "data" in pollResult and len(pollResult["data"]) > 0:
data = pollResult["data"][0]
# Check status
if "status" in data:
status = data["status"]
if status == "success":
# Check if transcription is complete
if "text" in data:
transcriptionText = data["text"]
# Send transcription result
rwUtils.sendVideoTranscription(transcriptionText, kwargs.get("node_id"))
return (transcriptionText,)
elif status == "processing":
# Continue polling
pass
else:
# Error or other status
raise Exception(f"Unexpected status: {status}")
# Check for interrupt before waiting
comfy.model_management.throw_exception_if_processing_interrupted()
# Wait before next poll
for _ in range(10): # 10 x 0.1 second = 1 second total
comfy.model_management.throw_exception_if_processing_interrupted()
rwUtils.time.sleep(0.1)