nodes/Runware-ComfyUI-Runware
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #59 from Runware/feature-syncModels

Browse Source 
Add Sync provider settings, audio/speech inputs, and useBatchSize parameter
This commit is contained in:
Sirshendu Ganguly
2025-12-14 18:07:51 -05:00
committed by GitHub
parent cb36affb8b 587d405c35
commit d65e37d40a
15 changed files with 756 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
__init__.py
+12
View File
@@ -42,6 +42,10 @@ from .modules.acceleratorOptions import acceleratorOptions
from .modules.providerSettings.viduProviderSettings import RunwareViduProviderSettings
from .modules.mediaUpload import runwareMediaUpload
from .modules.inputAudios import inputAudios
from .modules.audioInput import RunwareAudioInput
from .modules.audioInputCombine import RunwareAudioInputCombine
from .modules.speechInput import RunwareSpeechInput
from .modules.speechInputCombine import RunwareSpeechInputCombine
from .modules.referenceVideos import referenceVideos
from .modules.videoInferenceInputs import videoInferenceInputs
from .modules.providerSettings.lightricksProviderSettings import RunwareLightricksProviderSettings
@@ -51,6 +55,8 @@ from .modules.providerSettings.runwayProviderSettings import RunwareRunwayProvid
from .modules.providerSettings.lumaProviderSettings import RunwareLumaProviderSettings
from .modules.providerSettings.midjourneyProviderSettings import RunwareMidjourneyProviderSettings
from .modules.providerSettings.alibabaProviderSettings import RunwareAlibabaProviderSettings
from .modules.providerSettings.syncSegment import RunwareSyncSegment
from .modules.providerSettings.syncProviderSettings import RunwareSyncProviderSettings
from .modules.vectorize import vectorize
from .modules.videoBgRemoval import videoBgRemoval
from .modules.videoUpscaler import videoUpscaler
@@ -114,6 +120,10 @@ NODE_CLASS_MAPPINGS = {
"Runware Vidu Provider Settings": RunwareViduProviderSettings,
"Runware Media Upload": runwareMediaUpload,
"Runware Input Audios": inputAudios,
"Runware Video Audio Input": RunwareAudioInput,
"Runware Audio Inputs Combine": RunwareAudioInputCombine,
"Runware Video Speech Input": RunwareSpeechInput,
"Runware Speech Inputs Combine": RunwareSpeechInputCombine,
"Runware Reference Videos": referenceVideos,
"Runware Video Inference Inputs": videoInferenceInputs,
"Runware Lightricks Provider Settings": RunwareLightricksProviderSettings,
@@ -123,6 +133,8 @@ NODE_CLASS_MAPPINGS = {
"Runware Runway Provider Settings": RunwareRunwayProviderSettings,
"Runware Midjourney Provider Settings": RunwareMidjourneyProviderSettings,
"Runware Alibaba Provider Settings": RunwareAlibabaProviderSettings,
"Runware Sync Segment": RunwareSyncSegment,
"Runware Sync Provider Settings": RunwareSyncProviderSettings,
"Runware Vectorize": vectorize,
"Runware Video Background Removal": videoBgRemoval,
"Runware Video Upscaler": videoUpscaler,
clientlibs/main.js
+5 -1
View File
@@ -1,6 +1,6 @@
import { app } from "../../scripts/app.js";
import { api } from "../../scripts/api.js";
import { promptEnhanceHandler, syncDimensionsNodeHandler, searchNodeHandler, APIKeyHandler, captionNodeHandler, mediaUUIDHandler, videoTranscriptionHandler, handleCustomErrors, videoInferenceDimensionsHandler, videoModelSearchFilterHandler, audioModelSearchFilterHandler, useParameterToggleHandler, imageInferenceToggleHandler, upscalerToggleHandler, videoUpscalerToggleHandler, audioInferenceToggleHandler, acceleratorOptionsToggleHandler, bytedanceProviderSettingsToggleHandler, openaiProviderSettingsToggleHandler, lightricksProviderSettingsToggleHandler, klingProviderSettingsToggleHandler, lumaProviderSettingsToggleHandler, briaProviderSettingsToggleHandler, pixverseProviderSettingsToggleHandler, alibabaProviderSettingsToggleHandler } from "./utils.js";
import { promptEnhanceHandler, syncDimensionsNodeHandler, searchNodeHandler, APIKeyHandler, captionNodeHandler, mediaUUIDHandler, videoTranscriptionHandler, handleCustomErrors, videoInferenceDimensionsHandler, videoModelSearchFilterHandler, audioModelSearchFilterHandler, useParameterToggleHandler, imageInferenceToggleHandler, upscalerToggleHandler, videoUpscalerToggleHandler, audioInferenceToggleHandler, acceleratorOptionsToggleHandler, bytedanceProviderSettingsToggleHandler, openaiProviderSettingsToggleHandler, lightricksProviderSettingsToggleHandler, klingProviderSettingsToggleHandler, lumaProviderSettingsToggleHandler, briaProviderSettingsToggleHandler, pixverseProviderSettingsToggleHandler, alibabaProviderSettingsToggleHandler, syncProviderSettingsToggleHandler, syncSegmentToggleHandler } from "./utils.js";
import { RUNWARE_NODE_TYPES, RUNWARE_NODE_PROPS, SEARCH_TERMS } from "./types.js";
const nodeInitList = [];
@@ -73,6 +73,10 @@ app.registerExtension({
pixverseProviderSettingsToggleHandler(node);
} else if(nodeClass === RUNWARE_NODE_TYPES.ALIBABAPROVIDERSETTINGS) {
alibabaProviderSettingsToggleHandler(node);
} else if(nodeClass === RUNWARE_NODE_TYPES.SYNCPROVIDERSETTINGS) {
syncProviderSettingsToggleHandler(node);
} else if(nodeClass === RUNWARE_NODE_TYPES.SYNCSEGMENT) {
syncSegmentToggleHandler(node);
}
if(crNodeProps.colorModeOnly === true) return;
clientlibs/types.js
+20
View File
@@ -92,6 +92,8 @@ const RUNWARE_NODE_TYPES = {
LORAMIX: "Runware Lora Mix",
MEDIAUPLOAD: "Runware Media Upload",
INPUTAUDIOS: "Runware Input Audios",
AUDIOINPUT: "Runware Video Audio Input",
SPEECHINPUT: "Runware Video Speech Input",
REFERENCEVIDEOS: "Runware Reference Videos",
VIDEOINFERENCEINPUTS: "Runware Video Inference Inputs",
LIGHTRICKSPROVIDERSETTINGS: "Runware Lightricks Provider Settings",
@@ -100,6 +102,8 @@ const RUNWARE_NODE_TYPES = {
LUMAPROVIDERSETTINGS: "Runware Luma Provider Settings",
MIDJOURNEYPROVIDERSETTINGS: "Runware Midjourney Provider Settings",
ALIBABAPROVIDERSETTINGS: "Runware Alibaba Provider Settings",
SYNCSEGMENT: "Runware Sync Segment",
SYNCPROVIDERSETTINGS: "Runware Sync Provider Settings",
VECTORIZE: "Runware Vectorize",
VIDEOBGREMOVAL: "Runware Video Background Removal",
VIDEOUPSCALER: "Runware Video Upscaler",
@@ -292,6 +296,14 @@ const RUNWARE_NODE_PROPS = {
bgColor: DEFAULT_BGCOLOR,
colorModeOnly: true,
},
[RUNWARE_NODE_TYPES.AUDIOINPUT]: {
bgColor: DEFAULT_BGCOLOR,
colorModeOnly: true,
},
[RUNWARE_NODE_TYPES.SPEECHINPUT]: {
bgColor: DEFAULT_BGCOLOR,
colorModeOnly: true,
},
[RUNWARE_NODE_TYPES.REFERENCEVIDEOS]: {
bgColor: DEFAULT_BGCOLOR,
colorModeOnly: true,
@@ -324,6 +336,14 @@ const RUNWARE_NODE_PROPS = {
bgColor: DEFAULT_BGCOLOR,
colorModeOnly: true,
},
[RUNWARE_NODE_TYPES.SYNCSEGMENT]: {
bgColor: DEFAULT_BGCOLOR,
colorModeOnly: true,
},
[RUNWARE_NODE_TYPES.SYNCPROVIDERSETTINGS]: {
bgColor: DEFAULT_BGCOLOR,
colorModeOnly: true,
},
[RUNWARE_NODE_TYPES.VECTORIZE]: {
bgColor: DEFAULT_BGCOLOR,
},
clientlibs/utils.js
+151 -1
View File
@@ -1591,6 +1591,8 @@ function videoInferenceDimensionsHandler(videoInferenceNode) {
const seedWidget = videoInferenceNode.widgets.find(w => w.name === "seed");
const useStepsWidget = videoInferenceNode.widgets.find(w => w.name === "useSteps");
const stepsWidget = videoInferenceNode.widgets.find(w => w.name === "steps");
const useBatchSizeWidget = videoInferenceNode.widgets.find(w => w.name === "useBatchSize");
const batchSizeWidget = videoInferenceNode.widgets.find(w => w.name === "batchSize");
// Helper function to toggle widget enabled state
function toggleWidgetState(useWidget, paramWidget, paramName) {
@@ -1654,6 +1656,10 @@ function videoInferenceDimensionsHandler(videoInferenceNode) {
if (useStepsWidget && stepsWidget) {
toggleWidgetState(useStepsWidget, stepsWidget, "steps");
}
if (useBatchSizeWidget && batchSizeWidget) {
toggleWidgetState(useBatchSizeWidget, batchSizeWidget, "batchSize");
}
}
function videoModelSearchFilterHandler(videoModelSearchNode) {
@@ -1723,6 +1729,11 @@ function videoModelSearchFilterHandler(videoModelSearchNode) {
"lumaai:2@1 (Luma Ray 2)",
"lumaai:2@2 (Luma Ray 2 Flash)",
],
"Sync": [
"sync:lipsync-2@1 (Sync LipSync 2)",
"sync:lipsync-2-pro@1 (Sync LipSync 2 Pro)",
"sync:react-1@1 (Sync React-1)",
],
};
const MODEL_DIMENSIONS = {
@@ -1778,6 +1789,9 @@ function videoModelSearchFilterHandler(videoModelSearchNode) {
"lumaai:1@1": {"width": 1080, "height": 720},
"lumaai:2@1": {"width": 1080, "height": 720},
"lumaai:2@2": {"width": 1080, "height": 720},
"sync:lipsync-2@1": {"width": 0, "height": 0},
"sync:lipsync-2-pro@1": {"width": 0, "height": 0},
"sync:react-1@1": {"width": 0, "height": 0},
};
const MODEL_RESOLUTIONS = {
@@ -1833,6 +1847,9 @@ function videoModelSearchFilterHandler(videoModelSearchNode) {
"lumaai:1@1": "720p",
"lumaai:2@1": "720p",
"lumaai:2@2": "720p",
"sync:lipsync-2@1": "720p",
"sync:lipsync-2-pro@1": "720p",
"sync:react-1@1": "720p",
};
const DEFAULT_DIMENSIONS = {"width": 1024, "height": 576};
@@ -2449,4 +2466,137 @@ export {
briaProviderSettingsToggleHandler,
pixverseProviderSettingsToggleHandler,
alibabaProviderSettingsToggleHandler,
};
syncProviderSettingsToggleHandler,
syncSegmentToggleHandler,
};
function syncProviderSettingsToggleHandler(syncNode) {
// Find all "use" parameter widgets for Sync Provider Settings
const useEditRegionWidget = syncNode.widgets.find(w => w.name === "useEditRegion");
const editRegionWidget = syncNode.widgets.find(w => w.name === "editRegion");
const useEmotionPromptWidget = syncNode.widgets.find(w => w.name === "useEmotionPrompt");
const emotionPromptWidget = syncNode.widgets.find(w => w.name === "emotionPrompt");
const useTemperatureWidget = syncNode.widgets.find(w => w.name === "useTemperature");
const temperatureWidget = syncNode.widgets.find(w => w.name === "temperature");
const useActiveSpeakerDetectionWidget = syncNode.widgets.find(w => w.name === "useActiveSpeakerDetection");
const activeSpeakerDetectionWidget = syncNode.widgets.find(w => w.name === "activeSpeakerDetection");
const useOcclusionDetectionEnabledWidget = syncNode.widgets.find(w => w.name === "useOcclusionDetectionEnabled");
const occlusionDetectionEnabledWidget = syncNode.widgets.find(w => w.name === "occlusionDetectionEnabled");
// Helper function to toggle widget enabled state
function toggleWidgetState(useWidget, paramWidget, paramName) {
if (!useWidget || !paramWidget) return;
function toggleEnabled() {
const enabled = useWidget.value === true;
if (paramWidget.inputEl) {
paramWidget.inputEl.disabled = !enabled;
paramWidget.inputEl.style.opacity = enabled ? "1" : "0.5";
paramWidget.inputEl.style.cursor = enabled ? "text" : "not-allowed";
paramWidget.inputEl.readOnly = !enabled;
}
paramWidget.disabled = !enabled;
if (!paramWidget.inputEl) {
const nodeElement = syncNode.htmlElements?.widgetsContainer || syncNode.htmlElements;
if (nodeElement) {
const input = nodeElement.querySelector(`input[name="${paramName}"], textarea[name="${paramName}"], select[name="${paramName}"]`);
if (input) {
input.disabled = !enabled;
input.style.opacity = enabled ? "1" : "0.5";
input.style.cursor = enabled ? "text" : "not-allowed";
input.readOnly = !enabled;
if (input.tagName === "SELECT") {
input.style.pointerEvents = enabled ? "auto" : "none";
}
}
}
}
syncNode.setDirtyCanvas(true);
}
appendWidgetCB(useWidget, () => {
setTimeout(toggleEnabled, 50);
});
setTimeout(toggleEnabled, 100);
}
// Set up toggle handlers
if (useEditRegionWidget && editRegionWidget) {
toggleWidgetState(useEditRegionWidget, editRegionWidget, "editRegion");
}
if (useEmotionPromptWidget && emotionPromptWidget) {
toggleWidgetState(useEmotionPromptWidget, emotionPromptWidget, "emotionPrompt");
}
if (useTemperatureWidget && temperatureWidget) {
toggleWidgetState(useTemperatureWidget, temperatureWidget, "temperature");
}
if (useActiveSpeakerDetectionWidget && activeSpeakerDetectionWidget) {
toggleWidgetState(useActiveSpeakerDetectionWidget, activeSpeakerDetectionWidget, "activeSpeakerDetection");
}
if (useOcclusionDetectionEnabledWidget && occlusionDetectionEnabledWidget) {
toggleWidgetState(useOcclusionDetectionEnabledWidget, occlusionDetectionEnabledWidget, "occlusionDetectionEnabled");
}
}
function syncSegmentToggleHandler(syncSegmentNode) {
// Find all "use" parameter widgets for Sync Segment
const useAudioStartTimeWidget = syncSegmentNode.widgets.find(w => w.name === "useAudioStartTime");
const audioStartTimeWidget = syncSegmentNode.widgets.find(w => w.name === "audioStartTime");
const useAudioEndTimeWidget = syncSegmentNode.widgets.find(w => w.name === "useAudioEndTime");
const audioEndTimeWidget = syncSegmentNode.widgets.find(w => w.name === "audioEndTime");
// Helper function to toggle widget enabled state
function toggleWidgetState(useWidget, paramWidget, paramName) {
if (!useWidget || !paramWidget) return;
function toggleEnabled() {
const enabled = useWidget.value === true;
if (paramWidget.inputEl) {
paramWidget.inputEl.disabled = !enabled;
paramWidget.inputEl.style.opacity = enabled ? "1" : "0.5";
paramWidget.inputEl.style.cursor = enabled ? "text" : "not-allowed";
paramWidget.inputEl.readOnly = !enabled;
}
paramWidget.disabled = !enabled;
if (!paramWidget.inputEl) {
const nodeElement = syncSegmentNode.htmlElements?.widgetsContainer || syncSegmentNode.htmlElements;
if (nodeElement) {
const input = nodeElement.querySelector(`input[name="${paramName}"], textarea[name="${paramName}"], select[name="${paramName}"]`);
if (input) {
input.disabled = !enabled;
input.style.opacity = enabled ? "1" : "0.5";
input.style.cursor = enabled ? "text" : "not-allowed";
input.readOnly = !enabled;
if (input.tagName === "SELECT") {
input.style.pointerEvents = enabled ? "auto" : "none";
}
}
}
}
syncSegmentNode.setDirtyCanvas(true);
}
appendWidgetCB(useWidget, () => {
setTimeout(toggleEnabled, 50);
});
setTimeout(toggleEnabled, 100);
}
// Set up toggle handlers
if (useAudioStartTimeWidget && audioStartTimeWidget) {
toggleWidgetState(useAudioStartTimeWidget, audioStartTimeWidget, "audioStartTime");
}
if (useAudioEndTimeWidget && audioEndTimeWidget) {
toggleWidgetState(useAudioEndTimeWidget, audioEndTimeWidget, "audioEndTime");
}
}
modules/audioInput.py
+59
View File
@@ -0,0 +1,59 @@
"""
Runware Audio Input Node
Creates an audio input configuration with id and source
"""
from typing import Optional, Dict, Any
class RunwareAudioInput:
"""Runware Audio Input Node"""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"id": ("STRING", {
"tooltip": "Reference ID used to map audio input. Required when using segments array.",
"default": "",
}),
"source": ("STRING", {
"tooltip": "URL, base64, data URI, or UUID for the audio file. Audio formats: WAV, MP3, OGG, M4A, M3A, AAC, WMA, FLAC, MP4. Recommended: WAV/MP3 at 44.1kHz or 48kHz sampling rate.",
"default": "",
}),
},
"optional": {}
}
RETURN_TYPES = ("RUNWAREAUDIOINPUT",)
RETURN_NAMES = ("Audio Input",)
FUNCTION = "createAudioInput"
CATEGORY = "Runware"
DESCRIPTION = "Create an audio input configuration with reference ID and source for video generation."
def createAudioInput(self, **kwargs) -> tuple[Dict[str, Any]]:
"""Create audio input configuration"""
audio_id = kwargs.get("id", "").strip()
source = kwargs.get("source", "").strip()
if not audio_id or not source:
return ({},)
audioInput: Dict[str, Any] = {
"id": audio_id,
"source": source,
}
return (audioInput,)
# Node class mappings
NODE_CLASS_MAPPINGS = {
"RunwareAudioInput": RunwareAudioInput,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"RunwareAudioInput": "Runware Video Audio Input",
}
modules/audioInputCombine.py
+60
View File
@@ -0,0 +1,60 @@
"""
Runware Audio Input Combine Node
Combines multiple audio inputs into a list
"""
from typing import List, Dict, Any
class RunwareAudioInputCombine:
"""Runware Audio Input Combine Node"""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"Audio Input 1": ("RUNWAREAUDIOINPUT", {
"tooltip": "Connect a Runware Audio Input node.",
}),
},
"optional": {
"Audio Input 2": ("RUNWAREAUDIOINPUT", {
"tooltip": "Connect a Runware Audio Input node.",
}),
"Audio Input 3": ("RUNWAREAUDIOINPUT", {
"tooltip": "Connect a Runware Audio Input node.",
}),
"Audio Input 4": ("RUNWAREAUDIOINPUT", {
"tooltip": "Connect a Runware Audio Input node.",
}),
},
}
DESCRIPTION = "Combine multiple audio inputs to connect with Runware Video Inference Inputs."
FUNCTION = "audioInputCombine"
RETURN_TYPES = ("RUNWAREAUDIOINPUT",)
RETURN_NAMES = ("Audio Inputs",)
CATEGORY = "Runware"
def audioInputCombine(self, **kwargs):
"""Combine audio inputs into a list"""
audioInputs = []
for i in range(1, 5):
audioKey = f"Audio Input {i}"
audioInput = kwargs.get(audioKey)
if audioInput is not None and isinstance(audioInput, dict) and len(audioInput) > 0:
audioInputs.append(audioInput)
return (audioInputs,)
# Node class mappings
NODE_CLASS_MAPPINGS = {
"RunwareAudioInputCombine": RunwareAudioInputCombine,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"RunwareAudioInputCombine": "Runware Audio Inputs Combine",
}
modules/providerSettings/syncProviderSettings.py
+157
View File
@@ -0,0 +1,157 @@
"""
Runware Sync Provider Settings Node
Provides Sync-specific settings for video generation with lip-sync capabilities
"""
from typing import Optional, Dict, Any, List
class RunwareSyncProviderSettings:
"""Runware Sync Provider Settings Node"""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"syncMode": (["bounce", "loop", "cut_off", "silence", "remap"], {
"tooltip": "Synchronization strategy. Enum: bounce, loop, cut_off, silence, remap",
"default": "bounce",
}),
},
"optional": {
"useEditRegion": ("BOOLEAN", {
"tooltip": "Enable to include editRegion parameter",
"default": False,
}),
"editRegion": (["lips", "face", "head"], {
"tooltip": "Edit region for the model. Only works with react-1. Defaults to face, which affects lipsync + emotions in the face region. When head is selected, model generates natural talking head movements along with emotions + lipsync. Only used when 'Use Edit Region' is enabled.",
"default": "face",
}),
"useEmotionPrompt": ("BOOLEAN", {
"tooltip": "Enable to include emotionPrompt parameter",
"default": False,
}),
"emotionPrompt": (["happy", "sad", "angry", "disgusted", "surprised", "neutral"], {
"tooltip": "Emotion prompt for the generation. Only works for react-1 model. Only single word emotions are supported. Only used when 'Use Emotion Prompt' is enabled.",
"default": "happy",
}),
"useTemperature": ("BOOLEAN", {
"tooltip": "Enable to include temperature parameter",
"default": False,
}),
"temperature": ("FLOAT", {
"tooltip": "Sampling temperature. 0 -> least expressive, 1 -> most expressive. Only used when 'Use Temperature' is enabled.",
"default": 0.5,
"min": 0.0,
"max": 1.0,
"step": 0.1,
}),
"useActiveSpeakerDetection": ("BOOLEAN", {
"tooltip": "Enable to include activeSpeakerDetection parameter",
"default": False,
}),
"activeSpeakerDetection": ("BOOLEAN", {
"tooltip": "Enable active speaker detection. Only used when 'Use Active Speaker Detection' is enabled.",
"default": True,
"label_on": "Enabled",
"label_off": "Disabled",
}),
"useOcclusionDetectionEnabled": ("BOOLEAN", {
"tooltip": "Enable to include occlusionDetectionEnabled parameter",
"default": False,
}),
"occlusionDetectionEnabled": ("BOOLEAN", {
"tooltip": "Enable occlusion detection. Only used when 'Use Occlusion Detection Enabled' is enabled.",
"default": False,
"label_on": "Enabled",
"label_off": "Disabled",
}),
"Sync Segment 1": ("RUNWARESYNCSEGMENT", {
"tooltip": "Connect a Runware Sync Segment node (optional)",
}),
"Sync Segment 2": ("RUNWARESYNCSEGMENT", {
"tooltip": "Connect a Runware Sync Segment node (optional)",
}),
"Sync Segment 3": ("RUNWARESYNCSEGMENT", {
"tooltip": "Connect a Runware Sync Segment node (optional)",
}),
"Sync Segment 4": ("RUNWARESYNCSEGMENT", {
"tooltip": "Connect a Runware Sync Segment node (optional)",
}),
}
}
RETURN_TYPES = ("RUNWAREPROVIDERSETTINGS",)
RETURN_NAMES = ("Provider Settings",)
FUNCTION = "createProviderSettings"
CATEGORY = "Runware/Provider Settings"
DESCRIPTION = "Configure Sync-specific provider settings for video generation including lip-sync, emotion prompts, and segment configurations."
def createProviderSettings(self, **kwargs) -> tuple[Dict[str, Any]]:
"""Create Sync provider settings"""
# Get required parameters
syncMode = kwargs.get("syncMode", "bounce")
# Get control parameters
useEditRegion = kwargs.get("useEditRegion", False)
useEmotionPrompt = kwargs.get("useEmotionPrompt", False)
useTemperature = kwargs.get("useTemperature", False)
useActiveSpeakerDetection = kwargs.get("useActiveSpeakerDetection", False)
useOcclusionDetectionEnabled = kwargs.get("useOcclusionDetectionEnabled", False)
# Get value parameters
editRegion = kwargs.get("editRegion", "face")
emotionPrompt = kwargs.get("emotionPrompt", "happy")
temperature = kwargs.get("temperature", 0.5)
activeSpeakerDetection = kwargs.get("activeSpeakerDetection", True)
occlusionDetectionEnabled = kwargs.get("occlusionDetectionEnabled", False)
# Get sync segments (up to 4)
segments: List[Dict[str, Any]] = []
for i in range(1, 5):
segmentKey = f"Sync Segment {i}"
segment = kwargs.get(segmentKey)
if segment is not None and isinstance(segment, dict):
segments.append(segment)
# Build settings dictionary
syncSettings: Dict[str, Any] = {
"syncMode": syncMode,
}
# Add optional parameters only if enabled
if useEditRegion:
syncSettings["editRegion"] = editRegion
if useEmotionPrompt:
syncSettings["emotionPrompt"] = emotionPrompt
if useTemperature:
syncSettings["temperature"] = float(temperature)
if useActiveSpeakerDetection:
syncSettings["activeSpeakerDetection"] = activeSpeakerDetection
if useOcclusionDetectionEnabled:
syncSettings["occlusionDetectionEnabled"] = occlusionDetectionEnabled
# Add segments if any are provided
if segments:
syncSettings["segments"] = segments
# Clean up None values
syncSettings = {k: v for k, v in syncSettings.items() if v is not None}
# Return settings wrapped with provider key "sync"
if syncSettings:
return ({"sync": syncSettings},)
else:
return ({},)
# Node class mappings
NODE_CLASS_MAPPINGS = {
"RunwareSyncProviderSettings": RunwareSyncProviderSettings,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"RunwareSyncProviderSettings": "Runware Sync Provider Settings",
}
modules/providerSettings/syncSegment.py
+104
View File
@@ -0,0 +1,104 @@
"""
Runware Sync Segment Node
Creates a sync segment configuration for sync provider settings
"""
from typing import Optional, Dict, Any
class RunwareSyncSegment:
"""Runware Sync Segment Node"""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"startTime": ("FLOAT", {
"tooltip": "Start time of the segment in seconds",
"default": 0.0,
"min": 0.0,
"step": 0.1,
}),
"endTime": ("FLOAT", {
"tooltip": "End time of the segment in seconds",
"default": 1.0,
"min": 0.0,
"step": 0.1,
}),
"ref": ("STRING", {
"tooltip": "Reference identifier for the audio track (e.g., 'audio-track-1', 'speech-track-1')",
"default": "audio-track-1",
}),
},
"optional": {
"useAudioStartTime": ("BOOLEAN", {
"tooltip": "Enable to include audioStartTime parameter",
"default": False,
}),
"audioStartTime": ("FLOAT", {
"tooltip": "Start time in the audio track in seconds. Only used when 'Use Audio Start Time' is enabled.",
"default": 0.0,
"min": 0.0,
"step": 0.1,
}),
"useAudioEndTime": ("BOOLEAN", {
"tooltip": "Enable to include audioEndTime parameter",
"default": False,
}),
"audioEndTime": ("FLOAT", {
"tooltip": "End time in the audio track in seconds. Only used when 'Use Audio End Time' is enabled.",
"default": 1.0,
"min": 0.0,
"step": 0.1,
}),
}
}
RETURN_TYPES = ("RUNWARESYNCSEGMENT",)
RETURN_NAMES = ("Sync Segment",)
FUNCTION = "createSyncSegment"
CATEGORY = "Runware/Provider Settings"
DESCRIPTION = "Create a sync segment configuration for sync provider settings."
def createSyncSegment(self, **kwargs) -> tuple[Dict[str, Any]]:
"""Create sync segment configuration"""
# Get required parameters
startTime = kwargs.get("startTime", 0.0)
endTime = kwargs.get("endTime", 1.0)
ref = kwargs.get("ref", "audio-track-1")
# Get optional parameters
useAudioStartTime = kwargs.get("useAudioStartTime", False)
audioStartTime = kwargs.get("audioStartTime", 0.0)
useAudioEndTime = kwargs.get("useAudioEndTime", False)
audioEndTime = kwargs.get("audioEndTime", 1.0)
# Build segment dictionary
segment: Dict[str, Any] = {
"startTime": float(startTime),
"endTime": float(endTime),
"ref": str(ref),
}
# Add optional audio timing parameters
if useAudioStartTime:
segment["audioStartTime"] = float(audioStartTime)
if useAudioEndTime:
segment["audioEndTime"] = float(audioEndTime)
# Clean up None values
segment = {k: v for k, v in segment.items() if v is not None}
return (segment,)
# Node class mappings
NODE_CLASS_MAPPINGS = {
"RunwareSyncSegment": RunwareSyncSegment,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"RunwareSyncSegment": "Runware Sync Segment",
}
modules/speechInput.py
+72
View File
@@ -0,0 +1,72 @@
"""
Runware Speech Input Node
Creates a speech input configuration with TTS provider settings
"""
from typing import Optional, Dict, Any
class RunwareSpeechInput:
"""Runware Speech Input Node"""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"id": ("STRING", {
"tooltip": "Reference ID used to map speech input. Required when using segments array.",
"default": "",
}),
"provider": ("STRING", {
"tooltip": "TTS provider name. Currently supports: elevenlabs",
"default": "elevenlabs",
}),
"voiceId": ("STRING", {
"tooltip": "ElevenLabs voice ID for text-to-speech generation",
"default": "",
}),
"text": ("STRING", {
"multiline": True,
"tooltip": "The text script to be spoken",
"default": "",
}),
},
"optional": {}
}
RETURN_TYPES = ("RUNWARESPEECHINPUT",)
RETURN_NAMES = ("Speech Input",)
FUNCTION = "createSpeechInput"
CATEGORY = "Runware"
DESCRIPTION = "Create a speech input configuration with TTS provider settings for video generation."
def createSpeechInput(self, **kwargs) -> tuple[Dict[str, Any]]:
"""Create speech input configuration"""
speech_id = kwargs.get("id", "").strip()
provider = kwargs.get("provider", "elevenlabs").strip()
voiceId = kwargs.get("voiceId", "").strip()
text = kwargs.get("text", "").strip()
if not speech_id or not provider or not voiceId or not text:
return ({},)
speechInput: Dict[str, Any] = {
"id": speech_id,
"provider": provider,
"voiceId": voiceId,
"text": text,
}
return (speechInput,)
# Node class mappings
NODE_CLASS_MAPPINGS = {
"RunwareSpeechInput": RunwareSpeechInput,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"RunwareSpeechInput": "Runware Video Speech Input",
}
modules/speechInputCombine.py
+60
View File
@@ -0,0 +1,60 @@
"""
Runware Speech Input Combine Node
Combines multiple speech inputs into a list
"""
from typing import List, Dict, Any
class RunwareSpeechInputCombine:
"""Runware Speech Input Combine Node"""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"Speech Input 1": ("RUNWARESPEECHINPUT", {
"tooltip": "Connect a Runware Speech Input node.",
}),
},
"optional": {
"Speech Input 2": ("RUNWARESPEECHINPUT", {
"tooltip": "Connect a Runware Speech Input node.",
}),
"Speech Input 3": ("RUNWARESPEECHINPUT", {
"tooltip": "Connect a Runware Speech Input node.",
}),
"Speech Input 4": ("RUNWARESPEECHINPUT", {
"tooltip": "Connect a Runware Speech Input node.",
}),
},
}
DESCRIPTION = "Combine multiple speech inputs to connect with Runware Video Inference Inputs."
FUNCTION = "speechInputCombine"
RETURN_TYPES = ("RUNWARESPEECHINPUT",)
RETURN_NAMES = ("Speech Inputs",)
CATEGORY = "Runware"
def speechInputCombine(self, **kwargs):
"""Combine speech inputs into a list"""
speechInputs = []
for i in range(1, 5):
speechKey = f"Speech Input {i}"
speechInput = kwargs.get(speechKey)
if speechInput is not None and isinstance(speechInput, dict) and len(speechInput) > 0:
speechInputs.append(speechInput)
return (speechInputs,)
# Node class mappings
NODE_CLASS_MAPPINGS = {
"RunwareSpeechInputCombine": RunwareSpeechInputCombine,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"RunwareSpeechInputCombine": "Runware Speech Inputs Combine",
}
modules/videoInference.py
+11 -3
View File
@@ -65,8 +65,12 @@ class txt2vid:
"min": 1,
"max": 100,
}),
"useBatchSize": ("BOOLEAN", {
"tooltip": "Enable to include batchSize parameter in API request. Disable if your model doesn't support batch size.",
"default": False,
}),
"batchSize": ("INT", {
"tooltip": "The number of videos to generate in a single request.",
"tooltip": "The number of videos to generate in a single request. Only used when 'Use Batch Size' is enabled.",
"default": 1,
"min": 1,
"max": 4,
@@ -163,11 +167,12 @@ class txt2vid:
fps = kwargs.get("fps", 24)
useFps = kwargs.get("useFps", True)
outputFormat = kwargs.get("outputFormat", "mp4")
batchSize = kwargs.get("batchSize", 1)
seed = kwargs.get("seed", 1)
useSeed = kwargs.get("useSeed", True)
steps = kwargs.get("steps", 20)
useSteps = kwargs.get("useSteps", False)
useBatchSize = kwargs.get("useBatchSize", False)
batchSize = kwargs.get("batchSize", 1)
acceleration = kwargs.get("acceleration", "none")
# Handle model input - could be dict or string
@@ -190,11 +195,14 @@ class txt2vid:
"taskUUID": rwUtils.genRandUUID(),
"model": model,
"outputFormat": outputFormat,
"numberResults": batchSize,
"includeCost": True,
}
]
# Add batchSize (numberResults) only if enabled
if useBatchSize:
genConfig[0]["numberResults"] = batchSize
# Add resolution if enabled
if useResolution and resolution:
genConfig[0]["resolution"] = resolution
modules/videoInferenceInputs.py
+27 -2
View File
@@ -17,7 +17,13 @@ class videoInferenceInputs:
"tooltip": "Connect the Runware Video Inputs Frame node to provide timeline-constrained frame images."
}),
"Audio": ("STRING", {
"tooltip": "Connect the mediaUUID output from Runware Media Upload node with audio file. This audio will be synchronized with the generated video."
"tooltip": "Connect the mediaUUID output from Runware Media Upload node with audio file. This audio will be synchronized with the generated video. (Legacy: single audio string)"
}),
"Audio Inputs": ("RUNWAREAUDIOINPUT", {
"tooltip": "Connect Runware Video Audio Input node or Runware Audio Inputs Combine node to provide audio inputs with reference IDs."
}),
"Speech Inputs": ("RUNWARESPEECHINPUT", {
"tooltip": "Connect Runware Video Speech Input node or Runware Speech Inputs Combine node to provide TTS speech inputs with reference IDs."
}),
"Video": ("STRING", {
"tooltip": "Connect the mediaUUID output from Runware Media Upload node with a reference/input video."
@@ -47,6 +53,8 @@ class videoInferenceInputs:
"""Create video inference inputs from provided parameters"""
image = kwargs.get("Image", None)
audio = kwargs.get("Audio", None)
audioInputs = kwargs.get("Audio Inputs", None)
speechInputs = kwargs.get("Speech Inputs", None)
video = kwargs.get("Video", None)
mask = kwargs.get("Mask", None)
frame = kwargs.get("Frame", None)
@@ -59,8 +67,25 @@ class videoInferenceInputs:
if image is not None:
inputs["image"] = rwUtils.convertTensor2IMG(image)
if audio is not None and audio.strip() != "":
# Handle audio: can be either a string (legacy), a single dict, or a list of audio inputs
if audioInputs is not None:
if isinstance(audioInputs, list) and len(audioInputs) > 0:
# List from combine node
inputs["audio"] = audioInputs
elif isinstance(audioInputs, dict) and len(audioInputs) > 0:
# Single audio input dict
inputs["audio"] = [audioInputs]
elif audio is not None and audio.strip() != "":
inputs["audio"] = audio.strip()
# Handle speech inputs: can be a single dict or a list of speech inputs
if speechInputs is not None:
if isinstance(speechInputs, list) and len(speechInputs) > 0:
# List from combine node
inputs["speech"] = speechInputs
elif isinstance(speechInputs, dict) and len(speechInputs) > 0:
# Single speech input dict
inputs["speech"] = [speechInputs]
if video is not None and video.strip() != "":
inputs["video"] = video.strip()
modules/videoModelSearch.py
+17 -1
View File
@@ -79,6 +79,11 @@ class videoModelSearch:
"lumaai:2@1 (Luma Ray 2)",
"lumaai:2@2 (Luma Ray 2 Flash)",
],
"Sync": [
"sync:lipsync-2@1 (Sync LipSync 2)",
"sync:lipsync-2-pro@1 (Sync LipSync 2 Pro)",
"sync:react-1@1 (Sync React-1)",
],
}
# Model dimensions mapping
@@ -158,6 +163,11 @@ class videoModelSearch:
"lumaai:1@1": {"width": 1080, "height": 720},
"lumaai:2@1": {"width": 1080, "height": 720},
"lumaai:2@2": {"width": 1080, "height": 720},
# Sync Models
"sync:lipsync-2@1": {"width": 0, "height": 0},
"sync:lipsync-2-pro@1": {"width": 0, "height": 0},
"sync:react-1@1": {"width": 0, "height": 0},
}
# Model resolutions mapping
@@ -237,6 +247,11 @@ class videoModelSearch:
"lumaai:1@1": "720p",
"lumaai:2@1": "720p",
"lumaai:2@2": "720p",
# Sync Models
"sync:lipsync-2@1": "720p",
"sync:lipsync-2-pro@1": "720p",
"sync:react-1@1": "720p",
}
MODEL_ARCHITECTURES = [
@@ -252,7 +267,8 @@ class videoModelSearch:
"Lightricks",
"Ovi",
"Runway",
"Luma"
"Luma",
"Sync"
]
DEFAULT_DIMENSIONS = {"width": 1024, "height": 576}
workflows/Runware_Basic_Image_Inference_Wan_2_5_Preview.json
-1
View File
File diff suppressed because one or more lines are too long
workflows/Runware_Video_Inference_Sync_LipSync_2.json
+1
View File
File diff suppressed because one or more lines are too long