nodes/Runware-ComfyUI-Runware
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add Sync provider settings, audio/speech inputs, and useBatchSize parameter

Browse Source
This commit is contained in:
Sirshendu Ganguly
2025-12-14 17:51:33 -05:00
parent cb36affb8b
commit 2bc36e1b11
15 changed files with 766 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/audioInput.py
+59
View File
@@ -0,0 +1,59 @@
"""
Runware Audio Input Node
Creates an audio input configuration with id and source
"""
from typing import Optional, Dict, Any
class RunwareAudioInput:
"""Runware Audio Input Node"""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"id": ("STRING", {
"tooltip": "Reference ID used to map audio input. Required when using segments array.",
"default": "",
}),
"source": ("STRING", {
"tooltip": "URL, base64, data URI, or UUID for the audio file. Audio formats: WAV, MP3, OGG, M4A, M3A, AAC, WMA, FLAC, MP4. Recommended: WAV/MP3 at 44.1kHz or 48kHz sampling rate.",
"default": "",
}),
},
"optional": {}
}
RETURN_TYPES = ("RUNWAREAUDIOINPUT",)
RETURN_NAMES = ("Audio Input",)
FUNCTION = "createAudioInput"
CATEGORY = "Runware"
DESCRIPTION = "Create an audio input configuration with reference ID and source for video generation."
def createAudioInput(self, **kwargs) -> tuple[Dict[str, Any]]:
"""Create audio input configuration"""
audio_id = kwargs.get("id", "").strip()
source = kwargs.get("source", "").strip()
if not audio_id or not source:
return ({},)
audioInput: Dict[str, Any] = {
"id": audio_id,
"source": source,
}
return (audioInput,)
# Node class mappings
NODE_CLASS_MAPPINGS = {
"RunwareAudioInput": RunwareAudioInput,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"RunwareAudioInput": "Runware Video Audio Input",
}
modules/audioInputCombine.py
+60
View File
@@ -0,0 +1,60 @@
"""
Runware Audio Input Combine Node
Combines multiple audio inputs into a list
"""
from typing import List, Dict, Any
class RunwareAudioInputCombine:
"""Runware Audio Input Combine Node"""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"Audio Input 1": ("RUNWAREAUDIOINPUT", {
"tooltip": "Connect a Runware Audio Input node.",
}),
},
"optional": {
"Audio Input 2": ("RUNWAREAUDIOINPUT", {
"tooltip": "Connect a Runware Audio Input node.",
}),
"Audio Input 3": ("RUNWAREAUDIOINPUT", {
"tooltip": "Connect a Runware Audio Input node.",
}),
"Audio Input 4": ("RUNWAREAUDIOINPUT", {
"tooltip": "Connect a Runware Audio Input node.",
}),
},
}
DESCRIPTION = "Combine multiple audio inputs to connect with Runware Video Inference Inputs."
FUNCTION = "audioInputCombine"
RETURN_TYPES = ("RUNWAREAUDIOINPUT",)
RETURN_NAMES = ("Audio Inputs",)
CATEGORY = "Runware"
def audioInputCombine(self, **kwargs):
"""Combine audio inputs into a list"""
audioInputs = []
for i in range(1, 5):
audioKey = f"Audio Input {i}"
audioInput = kwargs.get(audioKey)
if audioInput is not None and isinstance(audioInput, dict) and len(audioInput) > 0:
audioInputs.append(audioInput)
return (audioInputs,)
# Node class mappings
NODE_CLASS_MAPPINGS = {
"RunwareAudioInputCombine": RunwareAudioInputCombine,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"RunwareAudioInputCombine": "Runware Audio Inputs Combine",
}
modules/providerSettings/syncProviderSettings.py
+157
View File
@@ -0,0 +1,157 @@
"""
Runware Sync Provider Settings Node
Provides Sync-specific settings for video generation with lip-sync capabilities
"""
from typing import Optional, Dict, Any, List
class RunwareSyncProviderSettings:
"""Runware Sync Provider Settings Node"""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"syncMode": (["bounce", "loop", "cut_off", "silence", "remap"], {
"tooltip": "Synchronization strategy. Enum: bounce, loop, cut_off, silence, remap",
"default": "bounce",
}),
},
"optional": {
"useEditRegion": ("BOOLEAN", {
"tooltip": "Enable to include editRegion parameter",
"default": False,
}),
"editRegion": (["lips", "face", "head"], {
"tooltip": "Edit region for the model. Only works with react-1. Defaults to face, which affects lipsync + emotions in the face region. When head is selected, model generates natural talking head movements along with emotions + lipsync. Only used when 'Use Edit Region' is enabled.",
"default": "face",
}),
"useEmotionPrompt": ("BOOLEAN", {
"tooltip": "Enable to include emotionPrompt parameter",
"default": False,
}),
"emotionPrompt": (["happy", "sad", "angry", "disgusted", "surprised", "neutral"], {
"tooltip": "Emotion prompt for the generation. Only works for react-1 model. Only single word emotions are supported. Only used when 'Use Emotion Prompt' is enabled.",
"default": "happy",
}),
"useTemperature": ("BOOLEAN", {
"tooltip": "Enable to include temperature parameter",
"default": False,
}),
"temperature": ("FLOAT", {
"tooltip": "Sampling temperature. 0 -> least expressive, 1 -> most expressive. Only used when 'Use Temperature' is enabled.",
"default": 0.5,
"min": 0.0,
"max": 1.0,
"step": 0.1,
}),
"useActiveSpeakerDetection": ("BOOLEAN", {
"tooltip": "Enable to include activeSpeakerDetection parameter",
"default": False,
}),
"activeSpeakerDetection": ("BOOLEAN", {
"tooltip": "Enable active speaker detection. Only used when 'Use Active Speaker Detection' is enabled.",
"default": True,
"label_on": "Enabled",
"label_off": "Disabled",
}),
"useOcclusionDetectionEnabled": ("BOOLEAN", {
"tooltip": "Enable to include occlusionDetectionEnabled parameter",
"default": False,
}),
"occlusionDetectionEnabled": ("BOOLEAN", {
"tooltip": "Enable occlusion detection. Only used when 'Use Occlusion Detection Enabled' is enabled.",
"default": False,
"label_on": "Enabled",
"label_off": "Disabled",
}),
"Sync Segment 1": ("RUNWARESYNCSEGMENT", {
"tooltip": "Connect a Runware Sync Segment node (optional)",
}),
"Sync Segment 2": ("RUNWARESYNCSEGMENT", {
"tooltip": "Connect a Runware Sync Segment node (optional)",
}),
"Sync Segment 3": ("RUNWARESYNCSEGMENT", {
"tooltip": "Connect a Runware Sync Segment node (optional)",
}),
"Sync Segment 4": ("RUNWARESYNCSEGMENT", {
"tooltip": "Connect a Runware Sync Segment node (optional)",
}),
}
}
RETURN_TYPES = ("RUNWAREPROVIDERSETTINGS",)
RETURN_NAMES = ("Provider Settings",)
FUNCTION = "createProviderSettings"
CATEGORY = "Runware/Provider Settings"
DESCRIPTION = "Configure Sync-specific provider settings for video generation including lip-sync, emotion prompts, and segment configurations."
def createProviderSettings(self, **kwargs) -> tuple[Dict[str, Any]]:
"""Create Sync provider settings"""
# Get required parameters
syncMode = kwargs.get("syncMode", "bounce")
# Get control parameters
useEditRegion = kwargs.get("useEditRegion", False)
useEmotionPrompt = kwargs.get("useEmotionPrompt", False)
useTemperature = kwargs.get("useTemperature", False)
useActiveSpeakerDetection = kwargs.get("useActiveSpeakerDetection", False)
useOcclusionDetectionEnabled = kwargs.get("useOcclusionDetectionEnabled", False)
# Get value parameters
editRegion = kwargs.get("editRegion", "face")
emotionPrompt = kwargs.get("emotionPrompt", "happy")
temperature = kwargs.get("temperature", 0.5)
activeSpeakerDetection = kwargs.get("activeSpeakerDetection", True)
occlusionDetectionEnabled = kwargs.get("occlusionDetectionEnabled", False)
# Get sync segments (up to 4)
segments: List[Dict[str, Any]] = []
for i in range(1, 5):
segmentKey = f"Sync Segment {i}"
segment = kwargs.get(segmentKey)
if segment is not None and isinstance(segment, dict):
segments.append(segment)
# Build settings dictionary
syncSettings: Dict[str, Any] = {
"syncMode": syncMode,
}
# Add optional parameters only if enabled
if useEditRegion:
syncSettings["editRegion"] = editRegion
if useEmotionPrompt:
syncSettings["emotionPrompt"] = emotionPrompt
if useTemperature:
syncSettings["temperature"] = float(temperature)
if useActiveSpeakerDetection:
syncSettings["activeSpeakerDetection"] = activeSpeakerDetection
if useOcclusionDetectionEnabled:
syncSettings["occlusionDetectionEnabled"] = occlusionDetectionEnabled
# Add segments if any are provided
if segments:
syncSettings["segments"] = segments
# Clean up None values
syncSettings = {k: v for k, v in syncSettings.items() if v is not None}
# Return settings wrapped with provider key "sync"
if syncSettings:
return ({"sync": syncSettings},)
else:
return ({},)
# Node class mappings
NODE_CLASS_MAPPINGS = {
"RunwareSyncProviderSettings": RunwareSyncProviderSettings,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"RunwareSyncProviderSettings": "Runware Sync Provider Settings",
}
modules/providerSettings/syncSegment.py
+104
View File
@@ -0,0 +1,104 @@
"""
Runware Sync Segment Node
Creates a sync segment configuration for sync provider settings
"""
from typing import Optional, Dict, Any
class RunwareSyncSegment:
"""Runware Sync Segment Node"""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"startTime": ("FLOAT", {
"tooltip": "Start time of the segment in seconds",
"default": 0.0,
"min": 0.0,
"step": 0.1,
}),
"endTime": ("FLOAT", {
"tooltip": "End time of the segment in seconds",
"default": 1.0,
"min": 0.0,
"step": 0.1,
}),
"ref": ("STRING", {
"tooltip": "Reference identifier for the audio track (e.g., 'audio-track-1', 'speech-track-1')",
"default": "audio-track-1",
}),
},
"optional": {
"useAudioStartTime": ("BOOLEAN", {
"tooltip": "Enable to include audioStartTime parameter",
"default": False,
}),
"audioStartTime": ("FLOAT", {
"tooltip": "Start time in the audio track in seconds. Only used when 'Use Audio Start Time' is enabled.",
"default": 0.0,
"min": 0.0,
"step": 0.1,
}),
"useAudioEndTime": ("BOOLEAN", {
"tooltip": "Enable to include audioEndTime parameter",
"default": False,
}),
"audioEndTime": ("FLOAT", {
"tooltip": "End time in the audio track in seconds. Only used when 'Use Audio End Time' is enabled.",
"default": 1.0,
"min": 0.0,
"step": 0.1,
}),
}
}
RETURN_TYPES = ("RUNWARESYNCSEGMENT",)
RETURN_NAMES = ("Sync Segment",)
FUNCTION = "createSyncSegment"
CATEGORY = "Runware/Provider Settings"
DESCRIPTION = "Create a sync segment configuration for sync provider settings."
def createSyncSegment(self, **kwargs) -> tuple[Dict[str, Any]]:
"""Create sync segment configuration"""
# Get required parameters
startTime = kwargs.get("startTime", 0.0)
endTime = kwargs.get("endTime", 1.0)
ref = kwargs.get("ref", "audio-track-1")
# Get optional parameters
useAudioStartTime = kwargs.get("useAudioStartTime", False)
audioStartTime = kwargs.get("audioStartTime", 0.0)
useAudioEndTime = kwargs.get("useAudioEndTime", False)
audioEndTime = kwargs.get("audioEndTime", 1.0)
# Build segment dictionary
segment: Dict[str, Any] = {
"startTime": float(startTime),
"endTime": float(endTime),
"ref": str(ref),
}
# Add optional audio timing parameters
if useAudioStartTime:
segment["audioStartTime"] = float(audioStartTime)
if useAudioEndTime:
segment["audioEndTime"] = float(audioEndTime)
# Clean up None values
segment = {k: v for k, v in segment.items() if v is not None}
return (segment,)
# Node class mappings
NODE_CLASS_MAPPINGS = {
"RunwareSyncSegment": RunwareSyncSegment,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"RunwareSyncSegment": "Runware Sync Segment",
}
modules/speechInput.py
+72
View File
@@ -0,0 +1,72 @@
"""
Runware Speech Input Node
Creates a speech input configuration with TTS provider settings
"""
from typing import Optional, Dict, Any
class RunwareSpeechInput:
"""Runware Speech Input Node"""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"id": ("STRING", {
"tooltip": "Reference ID used to map speech input. Required when using segments array.",
"default": "",
}),
"provider": ("STRING", {
"tooltip": "TTS provider name. Currently supports: elevenlabs",
"default": "elevenlabs",
}),
"voiceId": ("STRING", {
"tooltip": "ElevenLabs voice ID for text-to-speech generation",
"default": "",
}),
"text": ("STRING", {
"multiline": True,
"tooltip": "The text script to be spoken",
"default": "",
}),
},
"optional": {}
}
RETURN_TYPES = ("RUNWARESPEECHINPUT",)
RETURN_NAMES = ("Speech Input",)
FUNCTION = "createSpeechInput"
CATEGORY = "Runware"
DESCRIPTION = "Create a speech input configuration with TTS provider settings for video generation."
def createSpeechInput(self, **kwargs) -> tuple[Dict[str, Any]]:
"""Create speech input configuration"""
speech_id = kwargs.get("id", "").strip()
provider = kwargs.get("provider", "elevenlabs").strip()
voiceId = kwargs.get("voiceId", "").strip()
text = kwargs.get("text", "").strip()
if not speech_id or not provider or not voiceId or not text:
return ({},)
speechInput: Dict[str, Any] = {
"id": speech_id,
"provider": provider,
"voiceId": voiceId,
"text": text,
}
return (speechInput,)
# Node class mappings
NODE_CLASS_MAPPINGS = {
"RunwareSpeechInput": RunwareSpeechInput,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"RunwareSpeechInput": "Runware Video Speech Input",
}
modules/speechInputCombine.py
+60
View File
@@ -0,0 +1,60 @@
"""
Runware Speech Input Combine Node
Combines multiple speech inputs into a list
"""
from typing import List, Dict, Any
class RunwareSpeechInputCombine:
"""Runware Speech Input Combine Node"""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"Speech Input 1": ("RUNWARESPEECHINPUT", {
"tooltip": "Connect a Runware Speech Input node.",
}),
},
"optional": {
"Speech Input 2": ("RUNWARESPEECHINPUT", {
"tooltip": "Connect a Runware Speech Input node.",
}),
"Speech Input 3": ("RUNWARESPEECHINPUT", {
"tooltip": "Connect a Runware Speech Input node.",
}),
"Speech Input 4": ("RUNWARESPEECHINPUT", {
"tooltip": "Connect a Runware Speech Input node.",
}),
},
}
DESCRIPTION = "Combine multiple speech inputs to connect with Runware Video Inference Inputs."
FUNCTION = "speechInputCombine"
RETURN_TYPES = ("RUNWARESPEECHINPUT",)
RETURN_NAMES = ("Speech Inputs",)
CATEGORY = "Runware"
def speechInputCombine(self, **kwargs):
"""Combine speech inputs into a list"""
speechInputs = []
for i in range(1, 5):
speechKey = f"Speech Input {i}"
speechInput = kwargs.get(speechKey)
if speechInput is not None and isinstance(speechInput, dict) and len(speechInput) > 0:
speechInputs.append(speechInput)
return (speechInputs,)
# Node class mappings
NODE_CLASS_MAPPINGS = {
"RunwareSpeechInputCombine": RunwareSpeechInputCombine,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"RunwareSpeechInputCombine": "Runware Speech Inputs Combine",
}
modules/videoInference.py
+11 -3
View File
@@ -65,8 +65,12 @@ class txt2vid:
"min": 1,
"max": 100,
}),
"useBatchSize": ("BOOLEAN", {
"tooltip": "Enable to include batchSize parameter in API request. Disable if your model doesn't support batch size.",
"default": False,
}),
"batchSize": ("INT", {
"tooltip": "The number of videos to generate in a single request.",
"tooltip": "The number of videos to generate in a single request. Only used when 'Use Batch Size' is enabled.",
"default": 1,
"min": 1,
"max": 4,
@@ -163,11 +167,12 @@ class txt2vid:
fps = kwargs.get("fps", 24)
useFps = kwargs.get("useFps", True)
outputFormat = kwargs.get("outputFormat", "mp4")
batchSize = kwargs.get("batchSize", 1)
seed = kwargs.get("seed", 1)
useSeed = kwargs.get("useSeed", True)
steps = kwargs.get("steps", 20)
useSteps = kwargs.get("useSteps", False)
useBatchSize = kwargs.get("useBatchSize", False)
batchSize = kwargs.get("batchSize", 1)
acceleration = kwargs.get("acceleration", "none")
# Handle model input - could be dict or string
@@ -190,11 +195,14 @@ class txt2vid:
"taskUUID": rwUtils.genRandUUID(),
"model": model,
"outputFormat": outputFormat,
"numberResults": batchSize,
"includeCost": True,
}
]
# Add batchSize (numberResults) only if enabled
if useBatchSize:
genConfig[0]["numberResults"] = batchSize
# Add resolution if enabled
if useResolution and resolution:
genConfig[0]["resolution"] = resolution
modules/videoInferenceInputs.py
+27 -2
View File
@@ -17,7 +17,13 @@ class videoInferenceInputs:
"tooltip": "Connect the Runware Video Inputs Frame node to provide timeline-constrained frame images."
}),
"Audio": ("STRING", {
"tooltip": "Connect the mediaUUID output from Runware Media Upload node with audio file. This audio will be synchronized with the generated video."
"tooltip": "Connect the mediaUUID output from Runware Media Upload node with audio file. This audio will be synchronized with the generated video. (Legacy: single audio string)"
}),
"Audio Inputs": ("RUNWAREAUDIOINPUT", {
"tooltip": "Connect Runware Video Audio Input node or Runware Audio Inputs Combine node to provide audio inputs with reference IDs."
}),
"Speech Inputs": ("RUNWARESPEECHINPUT", {
"tooltip": "Connect Runware Video Speech Input node or Runware Speech Inputs Combine node to provide TTS speech inputs with reference IDs."
}),
"Video": ("STRING", {
"tooltip": "Connect the mediaUUID output from Runware Media Upload node with a reference/input video."
@@ -47,6 +53,8 @@ class videoInferenceInputs:
"""Create video inference inputs from provided parameters"""
image = kwargs.get("Image", None)
audio = kwargs.get("Audio", None)
audioInputs = kwargs.get("Audio Inputs", None)
speechInputs = kwargs.get("Speech Inputs", None)
video = kwargs.get("Video", None)
mask = kwargs.get("Mask", None)
frame = kwargs.get("Frame", None)
@@ -59,8 +67,25 @@ class videoInferenceInputs:
if image is not None:
inputs["image"] = rwUtils.convertTensor2IMG(image)
if audio is not None and audio.strip() != "":
# Handle audio: can be either a string (legacy), a single dict, or a list of audio inputs
if audioInputs is not None:
if isinstance(audioInputs, list) and len(audioInputs) > 0:
# List from combine node
inputs["audio"] = audioInputs
elif isinstance(audioInputs, dict) and len(audioInputs) > 0:
# Single audio input dict
inputs["audio"] = [audioInputs]
elif audio is not None and audio.strip() != "":
inputs["audio"] = audio.strip()
# Handle speech inputs: can be a single dict or a list of speech inputs
if speechInputs is not None:
if isinstance(speechInputs, list) and len(speechInputs) > 0:
# List from combine node
inputs["speech"] = speechInputs
elif isinstance(speechInputs, dict) and len(speechInputs) > 0:
# Single speech input dict
inputs["speech"] = [speechInputs]
if video is not None and video.strip() != "":
inputs["video"] = video.strip()
modules/videoModelSearch.py
+17 -1
View File
@@ -79,6 +79,11 @@ class videoModelSearch:
"lumaai:2@1 (Luma Ray 2)",
"lumaai:2@2 (Luma Ray 2 Flash)",
],
"Sync": [
"sync:lipsync-2@1 (Sync LipSync 2)",
"sync:lipsync-2-pro@1 (Sync LipSync 2 Pro)",
"sync:react-1@1 (Sync React-1)",
],
}
# Model dimensions mapping
@@ -158,6 +163,11 @@ class videoModelSearch:
"lumaai:1@1": {"width": 1080, "height": 720},
"lumaai:2@1": {"width": 1080, "height": 720},
"lumaai:2@2": {"width": 1080, "height": 720},
# Sync Models
"sync:lipsync-2@1": {"width": 0, "height": 0},
"sync:lipsync-2-pro@1": {"width": 0, "height": 0},
"sync:react-1@1": {"width": 0, "height": 0},
}
# Model resolutions mapping
@@ -237,6 +247,11 @@ class videoModelSearch:
"lumaai:1@1": "720p",
"lumaai:2@1": "720p",
"lumaai:2@2": "720p",
# Sync Models
"sync:lipsync-2@1": "720p",
"sync:lipsync-2-pro@1": "720p",
"sync:react-1@1": "720p",
}
MODEL_ARCHITECTURES = [
@@ -252,7 +267,8 @@ class videoModelSearch:
"Lightricks",
"Ovi",
"Runway",
"Luma"
"Luma",
"Sync"
]
DEFAULT_DIMENSIONS = {"width": 1024, "height": 576}