nodes/Runware-ComfyUI-Runware
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add video transcription, fix model names

Browse Source
This commit is contained in:
Sirshendu Ganguly
2025-10-27 14:41:58 -04:00
parent 056c15a1b7
commit 30dcf89ec6
9 changed files with 172 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/frameImages.py
+3 -7
View File
@@ -56,17 +56,13 @@ class RunwareFrameImages:
position = kwargs.get(position_key, "auto")
if image is not None:
# Upload image and get UUID
image_uuid = rwUtils.convertTensor2IMGForVideo(image)
image_url = f"https://im.runware.ai/image/ii/{image_uuid}.webp"
# Convert image to data URI or UUID
image_data = rwUtils.convertTensor2IMG(image)
frame_data = {
"inputImage": image_url
}
frame_data = {"inputImage": image_data}
# Only add frame position if not auto
if position != "auto":
# Convert string numbers to integers for API
if position.isdigit():
frame_data["frame"] = int(position)
else:
modules/utils/runwareUtils.py
+11
View File
@@ -255,6 +255,17 @@ def sendImageCaption(captionText, nodeID):
},
)
def sendVideoTranscription(transcriptionText, nodeID):
PromptServer.instance.send_sync(
"runwareVideoTranscription",
{
"success": True,
"transcriptionText": transcriptionText,
"nodeID": nodeID,
"widgetName": "prompt", # Target the prompt widget (matching imageCaptioning pattern)
},
)
def inferenecRequest(genConfig):
global RUNWARE_API_KEY, RUNWARE_API_BASE_URL, SESSION_TIMEOUT
RUNWARE_API_KEY = os.getenv("RUNWARE_API_KEY")
modules/videoInference.py
+1 -74
View File
@@ -1,81 +1,10 @@
from .utils import runwareUtils as rwUtils
from .videoModelSearch import videoModelSearch
from .frameImages import RunwareFrameImages
import json
import comfy.model_management
class RunwareFrameImages:
@classmethod
def INPUT_TYPES(cls):
return {
"required": {},
"optional": {
"image1": ("IMAGE", {
"tooltip": "First frame image. When only one image is provided, it becomes the first frame automatically.",
}),
"frame1_position": (["auto", "first", "last"], {
"default": "auto",
"tooltip": "Position for the first image. 'auto' uses automatic distribution rules.",
}),
"image2": ("IMAGE", {
"tooltip": "Second frame image. With two images, they become first and last frames automatically.",
}),
"frame2_position": (["auto", "first", "last"], {
"default": "auto",
"tooltip": "Position for the second image. 'auto' uses automatic distribution rules.",
}),
"image3": ("IMAGE", {
"tooltip": "Third frame image. Will be distributed between first and last frames.",
}),
"frame3_position": (["auto", "first", "last"], {
"default": "auto",
"tooltip": "Position for the third image. 'auto' uses automatic distribution rules.",
}),
"image4": ("IMAGE", {
"tooltip": "Fourth frame image. Will be distributed between first and last frames.",
}),
"frame4_position": (["auto", "first", "last"], {
"default": "auto",
"tooltip": "Position for the fourth image. 'auto' uses automatic distribution rules.",
}),
}
}
DESCRIPTION = "Configure frame images for video generation with precise positioning control. Supports automatic distribution or manual positioning."
FUNCTION = "create_frame_images"
RETURN_TYPES = ("RUNWAREFRAMEIMAGES",)
RETURN_NAMES = ("Frame Images",)
CATEGORY = "Runware"
def create_frame_images(self, **kwargs):
frame_images = []
# Process each image input
for i in range(1, 5): # Support up to 4 images
image_key = f"image{i}"
position_key = f"frame{i}_position"
image = kwargs.get(image_key)
position = kwargs.get(position_key, "auto")
if image is not None:
# Upload image and get UUID
image_uuid = rwUtils.convertTensor2IMGForVideo(image)
image_url = f"https://im.runware.ai/image/ii/{image_uuid}.webp"
frame_data = {
"inputImage": image_url
}
# Only add frame position if not auto
if position != "auto":
frame_data["frame"] = position
frame_images.append(frame_data)
return (frame_images,)
class txt2vid:
@classmethod
def INPUT_TYPES(cls):
@@ -439,11 +368,9 @@ class txt2vid:
# Node class mappings for ComfyUI registration
NODE_CLASS_MAPPINGS = {
"RunwareFrameImages": RunwareFrameImages,
"txt2vid": txt2vid,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"RunwareFrameImages": "Runware Frame Images",
"txt2vid": "Runware Video Inference",
}
modules/videoModelSearch.py
+2 -2
View File
@@ -18,7 +18,7 @@ class videoModelSearch:
"google:3@0 (Veo 3.0)",
"google:3@1 (Veo 3.0 Fast)",
"google:3@2 (Veo 3.1)",
"google:3@3 (Veo 3.1 Pro)",
"google:3@3 (Veo 3.1 Fast)",
],
"Seedance": [
"bytedance:2@1 (Seedance 1.0 Pro)",
@@ -77,7 +77,7 @@ class videoModelSearch:
"google:3@0": {"width": 1280, "height": 720}, # Veo 3.0
"google:3@1": {"width": 1280, "height": 720}, # Veo 3.0 Fast
"google:3@2": {"width": 1280, "height": 720}, # Veo 3.1
"google:3@3": {"width": 1280, "height": 720}, # Veo 3.1 Pro
"google:3@3": {"width": 1280, "height": 720}, # Veo 3.1 Fast
# Seedance Models
"bytedance:2@1": {"width": 864, "height": 480}, # Seedance 1.0 Pro
modules/videoTranscription.py
+132
View File
@@ -0,0 +1,132 @@
from .utils import runwareUtils as rwUtils
import json
import comfy.model_management
class videoTranscription:
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"Always Retranscribe": ("BOOLEAN", {
"default": False,
"tooltip": "Enable this option to always retranscribe the video each time you run the workflow.",
"label_on": "Enabled",
"label_off": "Disabled",
}),
"prompt": ("STRING", {
"multiline": True,
"default": "",
"placeholder": "Video transcription will appear here automatically.",
"tooltip": "The video transcription will be loaded here automatically."
}),
"video": ("STRING", {
"default": "",
"placeholder": "Enter video URL, file path, or Runware video UUID",
"tooltip": "The video to transcribe. Can be a URL, file path, or Runware video UUID."
}),
"model": ("STRING", {
"default": "memories:1@1",
"tooltip": "The model to use for video transcription. Default: memories:1@1"
}),
},
"hidden": { "node_id": "UNIQUE_ID" }
}
DESCRIPTION = "Transcribe spoken audio and visual context from videos into accurate text with speaker labeling and optional chapter summaries."
FUNCTION = "transcribeVideo"
RETURN_TYPES = ("STRING",)
RETURN_NAMES = ("Video Transcription",)
CATEGORY = "Runware"
OUTPUT_NODE = True
@classmethod
def IS_CHANGED(s, **kwargs):
alwaysRetranscribe = kwargs.get("Always Retranscribe")
if(alwaysRetranscribe):
return float("NAN")
return True
def transcribeVideo(self, **kwargs):
video = kwargs.get("video", "")
model = kwargs.get("model", "memories:1@1")
if not video or not video.strip():
raise Exception("Video URL, file path, or UUID is required")
# Create task parameters with correct structure
task_params = {
"taskType": "caption",
"taskUUID": rwUtils.genRandUUID(),
"model": model,
"inputs": {
"video": video
}
}
# Send the task
genConfig = [task_params]
# Debug: Print the request being sent
print(f"[DEBUG] Sending Video Transcription Request:")
print(f"[DEBUG] Request Payload: {json.dumps(genConfig, indent=2)}")
try:
genResult = rwUtils.inferenecRequest(genConfig)
# Debug: Print the response received
print(f"[DEBUG] Received Video Transcription Response:")
print(f"[DEBUG] Response: {json.dumps(genResult, indent=2)}")
except Exception as e:
raise e
# Get task UUID for polling
taskUUID = task_params["taskUUID"]
# Poll for result
while True:
# Check for interrupt before each poll
comfy.model_management.throw_exception_if_processing_interrupted()
# Poll for result
pollResult = rwUtils.pollVideoResult(taskUUID)
print(f"[Debugging] Poll result: {pollResult}")
# Check for errors first
if pollResult and "errors" in pollResult and len(pollResult["errors"]) > 0:
error_info = pollResult["errors"][0]
error_message = error_info.get("message", "Unknown error")
raise Exception(f"Video transcription failed: {error_message}")
# Check if we have data
if pollResult and "data" in pollResult and len(pollResult["data"]) > 0:
data = pollResult["data"][0]
# Check status
if "status" in data:
status = data["status"]
if status == "success":
# Check if transcription is complete
if "text" in data:
transcriptionText = data["text"]
# Send transcription result
rwUtils.sendVideoTranscription(transcriptionText, kwargs.get("node_id"))
return (transcriptionText,)
elif status == "processing":
# Continue polling
pass
else:
# Error or other status
raise Exception(f"Unexpected status: {status}")
# Check for interrupt before waiting
comfy.model_management.throw_exception_if_processing_interrupted()
# Wait before next poll
for _ in range(10): # 10 x 0.1 second = 1 second total
comfy.model_management.throw_exception_if_processing_interrupted()
rwUtils.time.sleep(0.1)