From 2bc36e1b111643e710da5bd2ab13167dfc0faf2a Mon Sep 17 00:00:00 2001 From: Sirshendu Ganguly Date: Sun, 14 Dec 2025 17:51:33 -0500 Subject: [PATCH] Add Sync provider settings, audio/speech inputs, and useBatchSize parameter --- __init__.py | 12 ++ clientlibs/main.js | 6 +- clientlibs/types.js | 30 ++++ clientlibs/utils.js | 152 ++++++++++++++++- modules/audioInput.py | 59 +++++++ modules/audioInputCombine.py | 60 +++++++ .../providerSettings/syncProviderSettings.py | 157 ++++++++++++++++++ modules/providerSettings/syncSegment.py | 104 ++++++++++++ modules/speechInput.py | 72 ++++++++ modules/speechInputCombine.py | 60 +++++++ modules/videoInference.py | 14 +- modules/videoInferenceInputs.py | 29 +++- modules/videoModelSearch.py | 18 +- ...Basic_Image_Inference_Wan_2_5_Preview.json | 1 - ...unware_Video_Inference_Sync_LipSync_2.json | 1 + 15 files changed, 766 insertions(+), 9 deletions(-) create mode 100644 modules/audioInput.py create mode 100644 modules/audioInputCombine.py create mode 100644 modules/providerSettings/syncProviderSettings.py create mode 100644 modules/providerSettings/syncSegment.py create mode 100644 modules/speechInput.py create mode 100644 modules/speechInputCombine.py delete mode 100644 workflows/Runware_Basic_Image_Inference_Wan_2_5_Preview.json create mode 100644 workflows/Runware_Video_Inference_Sync_LipSync_2.json diff --git a/__init__.py b/__init__.py index 12219fc..edacd1b 100644 --- a/__init__.py +++ b/__init__.py @@ -42,6 +42,10 @@ from .modules.acceleratorOptions import acceleratorOptions from .modules.providerSettings.viduProviderSettings import RunwareViduProviderSettings from .modules.mediaUpload import runwareMediaUpload from .modules.inputAudios import inputAudios +from .modules.audioInput import RunwareAudioInput +from .modules.audioInputCombine import RunwareAudioInputCombine +from .modules.speechInput import RunwareSpeechInput +from .modules.speechInputCombine import RunwareSpeechInputCombine from .modules.referenceVideos import referenceVideos from .modules.videoInferenceInputs import videoInferenceInputs from .modules.providerSettings.lightricksProviderSettings import RunwareLightricksProviderSettings @@ -51,6 +55,8 @@ from .modules.providerSettings.runwayProviderSettings import RunwareRunwayProvid from .modules.providerSettings.lumaProviderSettings import RunwareLumaProviderSettings from .modules.providerSettings.midjourneyProviderSettings import RunwareMidjourneyProviderSettings from .modules.providerSettings.alibabaProviderSettings import RunwareAlibabaProviderSettings +from .modules.providerSettings.syncSegment import RunwareSyncSegment +from .modules.providerSettings.syncProviderSettings import RunwareSyncProviderSettings from .modules.vectorize import vectorize from .modules.videoBgRemoval import videoBgRemoval from .modules.videoUpscaler import videoUpscaler @@ -114,6 +120,10 @@ NODE_CLASS_MAPPINGS = { "Runware Vidu Provider Settings": RunwareViduProviderSettings, "Runware Media Upload": runwareMediaUpload, "Runware Input Audios": inputAudios, + "Runware Video Audio Input": RunwareAudioInput, + "Runware Audio Inputs Combine": RunwareAudioInputCombine, + "Runware Video Speech Input": RunwareSpeechInput, + "Runware Speech Inputs Combine": RunwareSpeechInputCombine, "Runware Reference Videos": referenceVideos, "Runware Video Inference Inputs": videoInferenceInputs, "Runware Lightricks Provider Settings": RunwareLightricksProviderSettings, @@ -123,6 +133,8 @@ NODE_CLASS_MAPPINGS = { "Runware Runway Provider Settings": RunwareRunwayProviderSettings, "Runware Midjourney Provider Settings": RunwareMidjourneyProviderSettings, "Runware Alibaba Provider Settings": RunwareAlibabaProviderSettings, + "Runware Sync Segment": RunwareSyncSegment, + "Runware Sync Provider Settings": RunwareSyncProviderSettings, "Runware Vectorize": vectorize, "Runware Video Background Removal": videoBgRemoval, "Runware Video Upscaler": videoUpscaler, diff --git a/clientlibs/main.js b/clientlibs/main.js index e0aa300..7b12784 100644 --- a/clientlibs/main.js +++ b/clientlibs/main.js @@ -1,6 +1,6 @@ import { app } from "../../scripts/app.js"; import { api } from "../../scripts/api.js"; -import { promptEnhanceHandler, syncDimensionsNodeHandler, searchNodeHandler, APIKeyHandler, captionNodeHandler, mediaUUIDHandler, videoTranscriptionHandler, handleCustomErrors, videoInferenceDimensionsHandler, videoModelSearchFilterHandler, audioModelSearchFilterHandler, useParameterToggleHandler, imageInferenceToggleHandler, upscalerToggleHandler, videoUpscalerToggleHandler, audioInferenceToggleHandler, acceleratorOptionsToggleHandler, bytedanceProviderSettingsToggleHandler, openaiProviderSettingsToggleHandler, lightricksProviderSettingsToggleHandler, klingProviderSettingsToggleHandler, lumaProviderSettingsToggleHandler, briaProviderSettingsToggleHandler, pixverseProviderSettingsToggleHandler, alibabaProviderSettingsToggleHandler } from "./utils.js"; +import { promptEnhanceHandler, syncDimensionsNodeHandler, searchNodeHandler, APIKeyHandler, captionNodeHandler, mediaUUIDHandler, videoTranscriptionHandler, handleCustomErrors, videoInferenceDimensionsHandler, videoModelSearchFilterHandler, audioModelSearchFilterHandler, useParameterToggleHandler, imageInferenceToggleHandler, upscalerToggleHandler, videoUpscalerToggleHandler, audioInferenceToggleHandler, acceleratorOptionsToggleHandler, bytedanceProviderSettingsToggleHandler, openaiProviderSettingsToggleHandler, lightricksProviderSettingsToggleHandler, klingProviderSettingsToggleHandler, lumaProviderSettingsToggleHandler, briaProviderSettingsToggleHandler, pixverseProviderSettingsToggleHandler, alibabaProviderSettingsToggleHandler, syncProviderSettingsToggleHandler, syncSegmentToggleHandler } from "./utils.js"; import { RUNWARE_NODE_TYPES, RUNWARE_NODE_PROPS, SEARCH_TERMS } from "./types.js"; const nodeInitList = []; @@ -73,6 +73,10 @@ app.registerExtension({ pixverseProviderSettingsToggleHandler(node); } else if(nodeClass === RUNWARE_NODE_TYPES.ALIBABAPROVIDERSETTINGS) { alibabaProviderSettingsToggleHandler(node); + } else if(nodeClass === RUNWARE_NODE_TYPES.SYNCPROVIDERSETTINGS) { + syncProviderSettingsToggleHandler(node); + } else if(nodeClass === RUNWARE_NODE_TYPES.SYNCSEGMENT) { + syncSegmentToggleHandler(node); } if(crNodeProps.colorModeOnly === true) return; diff --git a/clientlibs/types.js b/clientlibs/types.js index 974b0de..2299a72 100644 --- a/clientlibs/types.js +++ b/clientlibs/types.js @@ -92,6 +92,10 @@ const RUNWARE_NODE_TYPES = { LORAMIX: "Runware Lora Mix", MEDIAUPLOAD: "Runware Media Upload", INPUTAUDIOS: "Runware Input Audios", + AUDIOINPUT: "Runware Video Audio Input", + AUDIOINPUTCOMBINE: "Runware Audio Inputs Combine", + SPEECHINPUT: "Runware Video Speech Input", + SPEECHINPUTCOMBINE: "Runware Speech Inputs Combine", REFERENCEVIDEOS: "Runware Reference Videos", VIDEOINFERENCEINPUTS: "Runware Video Inference Inputs", LIGHTRICKSPROVIDERSETTINGS: "Runware Lightricks Provider Settings", @@ -100,6 +104,8 @@ const RUNWARE_NODE_TYPES = { LUMAPROVIDERSETTINGS: "Runware Luma Provider Settings", MIDJOURNEYPROVIDERSETTINGS: "Runware Midjourney Provider Settings", ALIBABAPROVIDERSETTINGS: "Runware Alibaba Provider Settings", + SYNCSEGMENT: "Runware Sync Segment", + SYNCPROVIDERSETTINGS: "Runware Sync Provider Settings", VECTORIZE: "Runware Vectorize", VIDEOBGREMOVAL: "Runware Video Background Removal", VIDEOUPSCALER: "Runware Video Upscaler", @@ -292,6 +298,22 @@ const RUNWARE_NODE_PROPS = { bgColor: DEFAULT_BGCOLOR, colorModeOnly: true, }, + [RUNWARE_NODE_TYPES.AUDIOINPUT]: { + bgColor: DEFAULT_BGCOLOR, + colorModeOnly: true, + }, + [RUNWARE_NODE_TYPES.AUDIOINPUTCOMBINE]: { + bgColor: DEFAULT_BGCOLOR, + colorModeOnly: true, + }, + [RUNWARE_NODE_TYPES.SPEECHINPUT]: { + bgColor: DEFAULT_BGCOLOR, + colorModeOnly: true, + }, + [RUNWARE_NODE_TYPES.SPEECHINPUTCOMBINE]: { + bgColor: DEFAULT_BGCOLOR, + colorModeOnly: true, + }, [RUNWARE_NODE_TYPES.REFERENCEVIDEOS]: { bgColor: DEFAULT_BGCOLOR, colorModeOnly: true, @@ -324,6 +346,14 @@ const RUNWARE_NODE_PROPS = { bgColor: DEFAULT_BGCOLOR, colorModeOnly: true, }, + [RUNWARE_NODE_TYPES.SYNCSEGMENT]: { + bgColor: DEFAULT_BGCOLOR, + colorModeOnly: true, + }, + [RUNWARE_NODE_TYPES.SYNCPROVIDERSETTINGS]: { + bgColor: DEFAULT_BGCOLOR, + colorModeOnly: true, + }, [RUNWARE_NODE_TYPES.VECTORIZE]: { bgColor: DEFAULT_BGCOLOR, }, diff --git a/clientlibs/utils.js b/clientlibs/utils.js index 601f6ef..b7bff75 100644 --- a/clientlibs/utils.js +++ b/clientlibs/utils.js @@ -1591,6 +1591,8 @@ function videoInferenceDimensionsHandler(videoInferenceNode) { const seedWidget = videoInferenceNode.widgets.find(w => w.name === "seed"); const useStepsWidget = videoInferenceNode.widgets.find(w => w.name === "useSteps"); const stepsWidget = videoInferenceNode.widgets.find(w => w.name === "steps"); + const useBatchSizeWidget = videoInferenceNode.widgets.find(w => w.name === "useBatchSize"); + const batchSizeWidget = videoInferenceNode.widgets.find(w => w.name === "batchSize"); // Helper function to toggle widget enabled state function toggleWidgetState(useWidget, paramWidget, paramName) { @@ -1654,6 +1656,10 @@ function videoInferenceDimensionsHandler(videoInferenceNode) { if (useStepsWidget && stepsWidget) { toggleWidgetState(useStepsWidget, stepsWidget, "steps"); } + + if (useBatchSizeWidget && batchSizeWidget) { + toggleWidgetState(useBatchSizeWidget, batchSizeWidget, "batchSize"); + } } function videoModelSearchFilterHandler(videoModelSearchNode) { @@ -1723,6 +1729,11 @@ function videoModelSearchFilterHandler(videoModelSearchNode) { "lumaai:2@1 (Luma Ray 2)", "lumaai:2@2 (Luma Ray 2 Flash)", ], + "Sync": [ + "sync:lipsync-2@1 (Sync LipSync 2)", + "sync:lipsync-2-pro@1 (Sync LipSync 2 Pro)", + "sync:react-1@1 (Sync React-1)", + ], }; const MODEL_DIMENSIONS = { @@ -1778,6 +1789,9 @@ function videoModelSearchFilterHandler(videoModelSearchNode) { "lumaai:1@1": {"width": 1080, "height": 720}, "lumaai:2@1": {"width": 1080, "height": 720}, "lumaai:2@2": {"width": 1080, "height": 720}, + "sync:lipsync-2@1": {"width": 0, "height": 0}, + "sync:lipsync-2-pro@1": {"width": 0, "height": 0}, + "sync:react-1@1": {"width": 0, "height": 0}, }; const MODEL_RESOLUTIONS = { @@ -1833,6 +1847,9 @@ function videoModelSearchFilterHandler(videoModelSearchNode) { "lumaai:1@1": "720p", "lumaai:2@1": "720p", "lumaai:2@2": "720p", + "sync:lipsync-2@1": "720p", + "sync:lipsync-2-pro@1": "720p", + "sync:react-1@1": "720p", }; const DEFAULT_DIMENSIONS = {"width": 1024, "height": 576}; @@ -2449,4 +2466,137 @@ export { briaProviderSettingsToggleHandler, pixverseProviderSettingsToggleHandler, alibabaProviderSettingsToggleHandler, -}; \ No newline at end of file + syncProviderSettingsToggleHandler, + syncSegmentToggleHandler, +}; + +function syncProviderSettingsToggleHandler(syncNode) { + // Find all "use" parameter widgets for Sync Provider Settings + const useEditRegionWidget = syncNode.widgets.find(w => w.name === "useEditRegion"); + const editRegionWidget = syncNode.widgets.find(w => w.name === "editRegion"); + const useEmotionPromptWidget = syncNode.widgets.find(w => w.name === "useEmotionPrompt"); + const emotionPromptWidget = syncNode.widgets.find(w => w.name === "emotionPrompt"); + const useTemperatureWidget = syncNode.widgets.find(w => w.name === "useTemperature"); + const temperatureWidget = syncNode.widgets.find(w => w.name === "temperature"); + const useActiveSpeakerDetectionWidget = syncNode.widgets.find(w => w.name === "useActiveSpeakerDetection"); + const activeSpeakerDetectionWidget = syncNode.widgets.find(w => w.name === "activeSpeakerDetection"); + const useOcclusionDetectionEnabledWidget = syncNode.widgets.find(w => w.name === "useOcclusionDetectionEnabled"); + const occlusionDetectionEnabledWidget = syncNode.widgets.find(w => w.name === "occlusionDetectionEnabled"); + + // Helper function to toggle widget enabled state + function toggleWidgetState(useWidget, paramWidget, paramName) { + if (!useWidget || !paramWidget) return; + + function toggleEnabled() { + const enabled = useWidget.value === true; + + if (paramWidget.inputEl) { + paramWidget.inputEl.disabled = !enabled; + paramWidget.inputEl.style.opacity = enabled ? "1" : "0.5"; + paramWidget.inputEl.style.cursor = enabled ? "text" : "not-allowed"; + paramWidget.inputEl.readOnly = !enabled; + } + + paramWidget.disabled = !enabled; + + if (!paramWidget.inputEl) { + const nodeElement = syncNode.htmlElements?.widgetsContainer || syncNode.htmlElements; + if (nodeElement) { + const input = nodeElement.querySelector(`input[name="${paramName}"], textarea[name="${paramName}"], select[name="${paramName}"]`); + if (input) { + input.disabled = !enabled; + input.style.opacity = enabled ? "1" : "0.5"; + input.style.cursor = enabled ? "text" : "not-allowed"; + input.readOnly = !enabled; + if (input.tagName === "SELECT") { + input.style.pointerEvents = enabled ? "auto" : "none"; + } + } + } + } + + syncNode.setDirtyCanvas(true); + } + + appendWidgetCB(useWidget, () => { + setTimeout(toggleEnabled, 50); + }); + + setTimeout(toggleEnabled, 100); + } + + // Set up toggle handlers + if (useEditRegionWidget && editRegionWidget) { + toggleWidgetState(useEditRegionWidget, editRegionWidget, "editRegion"); + } + if (useEmotionPromptWidget && emotionPromptWidget) { + toggleWidgetState(useEmotionPromptWidget, emotionPromptWidget, "emotionPrompt"); + } + if (useTemperatureWidget && temperatureWidget) { + toggleWidgetState(useTemperatureWidget, temperatureWidget, "temperature"); + } + if (useActiveSpeakerDetectionWidget && activeSpeakerDetectionWidget) { + toggleWidgetState(useActiveSpeakerDetectionWidget, activeSpeakerDetectionWidget, "activeSpeakerDetection"); + } + if (useOcclusionDetectionEnabledWidget && occlusionDetectionEnabledWidget) { + toggleWidgetState(useOcclusionDetectionEnabledWidget, occlusionDetectionEnabledWidget, "occlusionDetectionEnabled"); + } +} + +function syncSegmentToggleHandler(syncSegmentNode) { + // Find all "use" parameter widgets for Sync Segment + const useAudioStartTimeWidget = syncSegmentNode.widgets.find(w => w.name === "useAudioStartTime"); + const audioStartTimeWidget = syncSegmentNode.widgets.find(w => w.name === "audioStartTime"); + const useAudioEndTimeWidget = syncSegmentNode.widgets.find(w => w.name === "useAudioEndTime"); + const audioEndTimeWidget = syncSegmentNode.widgets.find(w => w.name === "audioEndTime"); + + // Helper function to toggle widget enabled state + function toggleWidgetState(useWidget, paramWidget, paramName) { + if (!useWidget || !paramWidget) return; + + function toggleEnabled() { + const enabled = useWidget.value === true; + + if (paramWidget.inputEl) { + paramWidget.inputEl.disabled = !enabled; + paramWidget.inputEl.style.opacity = enabled ? "1" : "0.5"; + paramWidget.inputEl.style.cursor = enabled ? "text" : "not-allowed"; + paramWidget.inputEl.readOnly = !enabled; + } + + paramWidget.disabled = !enabled; + + if (!paramWidget.inputEl) { + const nodeElement = syncSegmentNode.htmlElements?.widgetsContainer || syncSegmentNode.htmlElements; + if (nodeElement) { + const input = nodeElement.querySelector(`input[name="${paramName}"], textarea[name="${paramName}"], select[name="${paramName}"]`); + if (input) { + input.disabled = !enabled; + input.style.opacity = enabled ? "1" : "0.5"; + input.style.cursor = enabled ? "text" : "not-allowed"; + input.readOnly = !enabled; + if (input.tagName === "SELECT") { + input.style.pointerEvents = enabled ? "auto" : "none"; + } + } + } + } + + syncSegmentNode.setDirtyCanvas(true); + } + + appendWidgetCB(useWidget, () => { + setTimeout(toggleEnabled, 50); + }); + + setTimeout(toggleEnabled, 100); + } + + // Set up toggle handlers + if (useAudioStartTimeWidget && audioStartTimeWidget) { + toggleWidgetState(useAudioStartTimeWidget, audioStartTimeWidget, "audioStartTime"); + } + if (useAudioEndTimeWidget && audioEndTimeWidget) { + toggleWidgetState(useAudioEndTimeWidget, audioEndTimeWidget, "audioEndTime"); + } +} \ No newline at end of file diff --git a/modules/audioInput.py b/modules/audioInput.py new file mode 100644 index 0000000..fdec4be --- /dev/null +++ b/modules/audioInput.py @@ -0,0 +1,59 @@ +""" +Runware Audio Input Node +Creates an audio input configuration with id and source +""" + +from typing import Optional, Dict, Any + + +class RunwareAudioInput: + """Runware Audio Input Node""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "id": ("STRING", { + "tooltip": "Reference ID used to map audio input. Required when using segments array.", + "default": "", + }), + "source": ("STRING", { + "tooltip": "URL, base64, data URI, or UUID for the audio file. Audio formats: WAV, MP3, OGG, M4A, M3A, AAC, WMA, FLAC, MP4. Recommended: WAV/MP3 at 44.1kHz or 48kHz sampling rate.", + "default": "", + }), + }, + "optional": {} + } + + RETURN_TYPES = ("RUNWAREAUDIOINPUT",) + RETURN_NAMES = ("Audio Input",) + FUNCTION = "createAudioInput" + CATEGORY = "Runware" + DESCRIPTION = "Create an audio input configuration with reference ID and source for video generation." + + def createAudioInput(self, **kwargs) -> tuple[Dict[str, Any]]: + """Create audio input configuration""" + + audio_id = kwargs.get("id", "").strip() + source = kwargs.get("source", "").strip() + + if not audio_id or not source: + return ({},) + + audioInput: Dict[str, Any] = { + "id": audio_id, + "source": source, + } + + return (audioInput,) + + +# Node class mappings +NODE_CLASS_MAPPINGS = { + "RunwareAudioInput": RunwareAudioInput, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "RunwareAudioInput": "Runware Video Audio Input", +} + diff --git a/modules/audioInputCombine.py b/modules/audioInputCombine.py new file mode 100644 index 0000000..560e07a --- /dev/null +++ b/modules/audioInputCombine.py @@ -0,0 +1,60 @@ +""" +Runware Audio Input Combine Node +Combines multiple audio inputs into a list +""" + +from typing import List, Dict, Any + + +class RunwareAudioInputCombine: + """Runware Audio Input Combine Node""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "Audio Input 1": ("RUNWAREAUDIOINPUT", { + "tooltip": "Connect a Runware Audio Input node.", + }), + }, + "optional": { + "Audio Input 2": ("RUNWAREAUDIOINPUT", { + "tooltip": "Connect a Runware Audio Input node.", + }), + "Audio Input 3": ("RUNWAREAUDIOINPUT", { + "tooltip": "Connect a Runware Audio Input node.", + }), + "Audio Input 4": ("RUNWAREAUDIOINPUT", { + "tooltip": "Connect a Runware Audio Input node.", + }), + }, + } + + DESCRIPTION = "Combine multiple audio inputs to connect with Runware Video Inference Inputs." + FUNCTION = "audioInputCombine" + RETURN_TYPES = ("RUNWAREAUDIOINPUT",) + RETURN_NAMES = ("Audio Inputs",) + CATEGORY = "Runware" + + def audioInputCombine(self, **kwargs): + """Combine audio inputs into a list""" + audioInputs = [] + + for i in range(1, 5): + audioKey = f"Audio Input {i}" + audioInput = kwargs.get(audioKey) + if audioInput is not None and isinstance(audioInput, dict) and len(audioInput) > 0: + audioInputs.append(audioInput) + + return (audioInputs,) + + +# Node class mappings +NODE_CLASS_MAPPINGS = { + "RunwareAudioInputCombine": RunwareAudioInputCombine, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "RunwareAudioInputCombine": "Runware Audio Inputs Combine", +} + diff --git a/modules/providerSettings/syncProviderSettings.py b/modules/providerSettings/syncProviderSettings.py new file mode 100644 index 0000000..f81cf80 --- /dev/null +++ b/modules/providerSettings/syncProviderSettings.py @@ -0,0 +1,157 @@ +""" +Runware Sync Provider Settings Node +Provides Sync-specific settings for video generation with lip-sync capabilities +""" + +from typing import Optional, Dict, Any, List + + +class RunwareSyncProviderSettings: + """Runware Sync Provider Settings Node""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "syncMode": (["bounce", "loop", "cut_off", "silence", "remap"], { + "tooltip": "Synchronization strategy. Enum: bounce, loop, cut_off, silence, remap", + "default": "bounce", + }), + }, + "optional": { + "useEditRegion": ("BOOLEAN", { + "tooltip": "Enable to include editRegion parameter", + "default": False, + }), + "editRegion": (["lips", "face", "head"], { + "tooltip": "Edit region for the model. Only works with react-1. Defaults to face, which affects lipsync + emotions in the face region. When head is selected, model generates natural talking head movements along with emotions + lipsync. Only used when 'Use Edit Region' is enabled.", + "default": "face", + }), + "useEmotionPrompt": ("BOOLEAN", { + "tooltip": "Enable to include emotionPrompt parameter", + "default": False, + }), + "emotionPrompt": (["happy", "sad", "angry", "disgusted", "surprised", "neutral"], { + "tooltip": "Emotion prompt for the generation. Only works for react-1 model. Only single word emotions are supported. Only used when 'Use Emotion Prompt' is enabled.", + "default": "happy", + }), + "useTemperature": ("BOOLEAN", { + "tooltip": "Enable to include temperature parameter", + "default": False, + }), + "temperature": ("FLOAT", { + "tooltip": "Sampling temperature. 0 -> least expressive, 1 -> most expressive. Only used when 'Use Temperature' is enabled.", + "default": 0.5, + "min": 0.0, + "max": 1.0, + "step": 0.1, + }), + "useActiveSpeakerDetection": ("BOOLEAN", { + "tooltip": "Enable to include activeSpeakerDetection parameter", + "default": False, + }), + "activeSpeakerDetection": ("BOOLEAN", { + "tooltip": "Enable active speaker detection. Only used when 'Use Active Speaker Detection' is enabled.", + "default": True, + "label_on": "Enabled", + "label_off": "Disabled", + }), + "useOcclusionDetectionEnabled": ("BOOLEAN", { + "tooltip": "Enable to include occlusionDetectionEnabled parameter", + "default": False, + }), + "occlusionDetectionEnabled": ("BOOLEAN", { + "tooltip": "Enable occlusion detection. Only used when 'Use Occlusion Detection Enabled' is enabled.", + "default": False, + "label_on": "Enabled", + "label_off": "Disabled", + }), + "Sync Segment 1": ("RUNWARESYNCSEGMENT", { + "tooltip": "Connect a Runware Sync Segment node (optional)", + }), + "Sync Segment 2": ("RUNWARESYNCSEGMENT", { + "tooltip": "Connect a Runware Sync Segment node (optional)", + }), + "Sync Segment 3": ("RUNWARESYNCSEGMENT", { + "tooltip": "Connect a Runware Sync Segment node (optional)", + }), + "Sync Segment 4": ("RUNWARESYNCSEGMENT", { + "tooltip": "Connect a Runware Sync Segment node (optional)", + }), + } + } + + RETURN_TYPES = ("RUNWAREPROVIDERSETTINGS",) + RETURN_NAMES = ("Provider Settings",) + FUNCTION = "createProviderSettings" + CATEGORY = "Runware/Provider Settings" + DESCRIPTION = "Configure Sync-specific provider settings for video generation including lip-sync, emotion prompts, and segment configurations." + + def createProviderSettings(self, **kwargs) -> tuple[Dict[str, Any]]: + """Create Sync provider settings""" + + # Get required parameters + syncMode = kwargs.get("syncMode", "bounce") + + # Get control parameters + useEditRegion = kwargs.get("useEditRegion", False) + useEmotionPrompt = kwargs.get("useEmotionPrompt", False) + useTemperature = kwargs.get("useTemperature", False) + useActiveSpeakerDetection = kwargs.get("useActiveSpeakerDetection", False) + useOcclusionDetectionEnabled = kwargs.get("useOcclusionDetectionEnabled", False) + + # Get value parameters + editRegion = kwargs.get("editRegion", "face") + emotionPrompt = kwargs.get("emotionPrompt", "happy") + temperature = kwargs.get("temperature", 0.5) + activeSpeakerDetection = kwargs.get("activeSpeakerDetection", True) + occlusionDetectionEnabled = kwargs.get("occlusionDetectionEnabled", False) + + # Get sync segments (up to 4) + segments: List[Dict[str, Any]] = [] + for i in range(1, 5): + segmentKey = f"Sync Segment {i}" + segment = kwargs.get(segmentKey) + if segment is not None and isinstance(segment, dict): + segments.append(segment) + + # Build settings dictionary + syncSettings: Dict[str, Any] = { + "syncMode": syncMode, + } + + # Add optional parameters only if enabled + if useEditRegion: + syncSettings["editRegion"] = editRegion + if useEmotionPrompt: + syncSettings["emotionPrompt"] = emotionPrompt + if useTemperature: + syncSettings["temperature"] = float(temperature) + if useActiveSpeakerDetection: + syncSettings["activeSpeakerDetection"] = activeSpeakerDetection + if useOcclusionDetectionEnabled: + syncSettings["occlusionDetectionEnabled"] = occlusionDetectionEnabled + + # Add segments if any are provided + if segments: + syncSettings["segments"] = segments + + # Clean up None values + syncSettings = {k: v for k, v in syncSettings.items() if v is not None} + + # Return settings wrapped with provider key "sync" + if syncSettings: + return ({"sync": syncSettings},) + else: + return ({},) + + +# Node class mappings +NODE_CLASS_MAPPINGS = { + "RunwareSyncProviderSettings": RunwareSyncProviderSettings, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "RunwareSyncProviderSettings": "Runware Sync Provider Settings", +} + diff --git a/modules/providerSettings/syncSegment.py b/modules/providerSettings/syncSegment.py new file mode 100644 index 0000000..460afdb --- /dev/null +++ b/modules/providerSettings/syncSegment.py @@ -0,0 +1,104 @@ +""" +Runware Sync Segment Node +Creates a sync segment configuration for sync provider settings +""" + +from typing import Optional, Dict, Any + + +class RunwareSyncSegment: + """Runware Sync Segment Node""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "startTime": ("FLOAT", { + "tooltip": "Start time of the segment in seconds", + "default": 0.0, + "min": 0.0, + "step": 0.1, + }), + "endTime": ("FLOAT", { + "tooltip": "End time of the segment in seconds", + "default": 1.0, + "min": 0.0, + "step": 0.1, + }), + "ref": ("STRING", { + "tooltip": "Reference identifier for the audio track (e.g., 'audio-track-1', 'speech-track-1')", + "default": "audio-track-1", + }), + }, + "optional": { + "useAudioStartTime": ("BOOLEAN", { + "tooltip": "Enable to include audioStartTime parameter", + "default": False, + }), + "audioStartTime": ("FLOAT", { + "tooltip": "Start time in the audio track in seconds. Only used when 'Use Audio Start Time' is enabled.", + "default": 0.0, + "min": 0.0, + "step": 0.1, + }), + "useAudioEndTime": ("BOOLEAN", { + "tooltip": "Enable to include audioEndTime parameter", + "default": False, + }), + "audioEndTime": ("FLOAT", { + "tooltip": "End time in the audio track in seconds. Only used when 'Use Audio End Time' is enabled.", + "default": 1.0, + "min": 0.0, + "step": 0.1, + }), + } + } + + RETURN_TYPES = ("RUNWARESYNCSEGMENT",) + RETURN_NAMES = ("Sync Segment",) + FUNCTION = "createSyncSegment" + CATEGORY = "Runware/Provider Settings" + DESCRIPTION = "Create a sync segment configuration for sync provider settings." + + def createSyncSegment(self, **kwargs) -> tuple[Dict[str, Any]]: + """Create sync segment configuration""" + + # Get required parameters + startTime = kwargs.get("startTime", 0.0) + endTime = kwargs.get("endTime", 1.0) + ref = kwargs.get("ref", "audio-track-1") + + # Get optional parameters + useAudioStartTime = kwargs.get("useAudioStartTime", False) + audioStartTime = kwargs.get("audioStartTime", 0.0) + useAudioEndTime = kwargs.get("useAudioEndTime", False) + audioEndTime = kwargs.get("audioEndTime", 1.0) + + # Build segment dictionary + segment: Dict[str, Any] = { + "startTime": float(startTime), + "endTime": float(endTime), + "ref": str(ref), + } + + # Add optional audio timing parameters + if useAudioStartTime: + segment["audioStartTime"] = float(audioStartTime) + if useAudioEndTime: + segment["audioEndTime"] = float(audioEndTime) + + # Clean up None values + segment = {k: v for k, v in segment.items() if v is not None} + + return (segment,) + + +# Node class mappings +NODE_CLASS_MAPPINGS = { + "RunwareSyncSegment": RunwareSyncSegment, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "RunwareSyncSegment": "Runware Sync Segment", +} + diff --git a/modules/speechInput.py b/modules/speechInput.py new file mode 100644 index 0000000..48b9e57 --- /dev/null +++ b/modules/speechInput.py @@ -0,0 +1,72 @@ +""" +Runware Speech Input Node +Creates a speech input configuration with TTS provider settings +""" + +from typing import Optional, Dict, Any + + +class RunwareSpeechInput: + """Runware Speech Input Node""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "id": ("STRING", { + "tooltip": "Reference ID used to map speech input. Required when using segments array.", + "default": "", + }), + "provider": ("STRING", { + "tooltip": "TTS provider name. Currently supports: elevenlabs", + "default": "elevenlabs", + }), + "voiceId": ("STRING", { + "tooltip": "ElevenLabs voice ID for text-to-speech generation", + "default": "", + }), + "text": ("STRING", { + "multiline": True, + "tooltip": "The text script to be spoken", + "default": "", + }), + }, + "optional": {} + } + + RETURN_TYPES = ("RUNWARESPEECHINPUT",) + RETURN_NAMES = ("Speech Input",) + FUNCTION = "createSpeechInput" + CATEGORY = "Runware" + DESCRIPTION = "Create a speech input configuration with TTS provider settings for video generation." + + def createSpeechInput(self, **kwargs) -> tuple[Dict[str, Any]]: + """Create speech input configuration""" + + speech_id = kwargs.get("id", "").strip() + provider = kwargs.get("provider", "elevenlabs").strip() + voiceId = kwargs.get("voiceId", "").strip() + text = kwargs.get("text", "").strip() + + if not speech_id or not provider or not voiceId or not text: + return ({},) + + speechInput: Dict[str, Any] = { + "id": speech_id, + "provider": provider, + "voiceId": voiceId, + "text": text, + } + + return (speechInput,) + + +# Node class mappings +NODE_CLASS_MAPPINGS = { + "RunwareSpeechInput": RunwareSpeechInput, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "RunwareSpeechInput": "Runware Video Speech Input", +} + diff --git a/modules/speechInputCombine.py b/modules/speechInputCombine.py new file mode 100644 index 0000000..041d38d --- /dev/null +++ b/modules/speechInputCombine.py @@ -0,0 +1,60 @@ +""" +Runware Speech Input Combine Node +Combines multiple speech inputs into a list +""" + +from typing import List, Dict, Any + + +class RunwareSpeechInputCombine: + """Runware Speech Input Combine Node""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "Speech Input 1": ("RUNWARESPEECHINPUT", { + "tooltip": "Connect a Runware Speech Input node.", + }), + }, + "optional": { + "Speech Input 2": ("RUNWARESPEECHINPUT", { + "tooltip": "Connect a Runware Speech Input node.", + }), + "Speech Input 3": ("RUNWARESPEECHINPUT", { + "tooltip": "Connect a Runware Speech Input node.", + }), + "Speech Input 4": ("RUNWARESPEECHINPUT", { + "tooltip": "Connect a Runware Speech Input node.", + }), + }, + } + + DESCRIPTION = "Combine multiple speech inputs to connect with Runware Video Inference Inputs." + FUNCTION = "speechInputCombine" + RETURN_TYPES = ("RUNWARESPEECHINPUT",) + RETURN_NAMES = ("Speech Inputs",) + CATEGORY = "Runware" + + def speechInputCombine(self, **kwargs): + """Combine speech inputs into a list""" + speechInputs = [] + + for i in range(1, 5): + speechKey = f"Speech Input {i}" + speechInput = kwargs.get(speechKey) + if speechInput is not None and isinstance(speechInput, dict) and len(speechInput) > 0: + speechInputs.append(speechInput) + + return (speechInputs,) + + +# Node class mappings +NODE_CLASS_MAPPINGS = { + "RunwareSpeechInputCombine": RunwareSpeechInputCombine, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "RunwareSpeechInputCombine": "Runware Speech Inputs Combine", +} + diff --git a/modules/videoInference.py b/modules/videoInference.py index bfd560d..0e8525a 100644 --- a/modules/videoInference.py +++ b/modules/videoInference.py @@ -65,8 +65,12 @@ class txt2vid: "min": 1, "max": 100, }), + "useBatchSize": ("BOOLEAN", { + "tooltip": "Enable to include batchSize parameter in API request. Disable if your model doesn't support batch size.", + "default": False, + }), "batchSize": ("INT", { - "tooltip": "The number of videos to generate in a single request.", + "tooltip": "The number of videos to generate in a single request. Only used when 'Use Batch Size' is enabled.", "default": 1, "min": 1, "max": 4, @@ -163,11 +167,12 @@ class txt2vid: fps = kwargs.get("fps", 24) useFps = kwargs.get("useFps", True) outputFormat = kwargs.get("outputFormat", "mp4") - batchSize = kwargs.get("batchSize", 1) seed = kwargs.get("seed", 1) useSeed = kwargs.get("useSeed", True) steps = kwargs.get("steps", 20) useSteps = kwargs.get("useSteps", False) + useBatchSize = kwargs.get("useBatchSize", False) + batchSize = kwargs.get("batchSize", 1) acceleration = kwargs.get("acceleration", "none") # Handle model input - could be dict or string @@ -190,11 +195,14 @@ class txt2vid: "taskUUID": rwUtils.genRandUUID(), "model": model, "outputFormat": outputFormat, - "numberResults": batchSize, "includeCost": True, } ] + # Add batchSize (numberResults) only if enabled + if useBatchSize: + genConfig[0]["numberResults"] = batchSize + # Add resolution if enabled if useResolution and resolution: genConfig[0]["resolution"] = resolution diff --git a/modules/videoInferenceInputs.py b/modules/videoInferenceInputs.py index cdee2c8..48343de 100644 --- a/modules/videoInferenceInputs.py +++ b/modules/videoInferenceInputs.py @@ -17,7 +17,13 @@ class videoInferenceInputs: "tooltip": "Connect the Runware Video Inputs Frame node to provide timeline-constrained frame images." }), "Audio": ("STRING", { - "tooltip": "Connect the mediaUUID output from Runware Media Upload node with audio file. This audio will be synchronized with the generated video." + "tooltip": "Connect the mediaUUID output from Runware Media Upload node with audio file. This audio will be synchronized with the generated video. (Legacy: single audio string)" + }), + "Audio Inputs": ("RUNWAREAUDIOINPUT", { + "tooltip": "Connect Runware Video Audio Input node or Runware Audio Inputs Combine node to provide audio inputs with reference IDs." + }), + "Speech Inputs": ("RUNWARESPEECHINPUT", { + "tooltip": "Connect Runware Video Speech Input node or Runware Speech Inputs Combine node to provide TTS speech inputs with reference IDs." }), "Video": ("STRING", { "tooltip": "Connect the mediaUUID output from Runware Media Upload node with a reference/input video." @@ -47,6 +53,8 @@ class videoInferenceInputs: """Create video inference inputs from provided parameters""" image = kwargs.get("Image", None) audio = kwargs.get("Audio", None) + audioInputs = kwargs.get("Audio Inputs", None) + speechInputs = kwargs.get("Speech Inputs", None) video = kwargs.get("Video", None) mask = kwargs.get("Mask", None) frame = kwargs.get("Frame", None) @@ -59,8 +67,25 @@ class videoInferenceInputs: if image is not None: inputs["image"] = rwUtils.convertTensor2IMG(image) - if audio is not None and audio.strip() != "": + # Handle audio: can be either a string (legacy), a single dict, or a list of audio inputs + if audioInputs is not None: + if isinstance(audioInputs, list) and len(audioInputs) > 0: + # List from combine node + inputs["audio"] = audioInputs + elif isinstance(audioInputs, dict) and len(audioInputs) > 0: + # Single audio input dict + inputs["audio"] = [audioInputs] + elif audio is not None and audio.strip() != "": inputs["audio"] = audio.strip() + + # Handle speech inputs: can be a single dict or a list of speech inputs + if speechInputs is not None: + if isinstance(speechInputs, list) and len(speechInputs) > 0: + # List from combine node + inputs["speech"] = speechInputs + elif isinstance(speechInputs, dict) and len(speechInputs) > 0: + # Single speech input dict + inputs["speech"] = [speechInputs] if video is not None and video.strip() != "": inputs["video"] = video.strip() diff --git a/modules/videoModelSearch.py b/modules/videoModelSearch.py index e5b7213..5498e04 100644 --- a/modules/videoModelSearch.py +++ b/modules/videoModelSearch.py @@ -79,6 +79,11 @@ class videoModelSearch: "lumaai:2@1 (Luma Ray 2)", "lumaai:2@2 (Luma Ray 2 Flash)", ], + "Sync": [ + "sync:lipsync-2@1 (Sync LipSync 2)", + "sync:lipsync-2-pro@1 (Sync LipSync 2 Pro)", + "sync:react-1@1 (Sync React-1)", + ], } # Model dimensions mapping @@ -158,6 +163,11 @@ class videoModelSearch: "lumaai:1@1": {"width": 1080, "height": 720}, "lumaai:2@1": {"width": 1080, "height": 720}, "lumaai:2@2": {"width": 1080, "height": 720}, + + # Sync Models + "sync:lipsync-2@1": {"width": 0, "height": 0}, + "sync:lipsync-2-pro@1": {"width": 0, "height": 0}, + "sync:react-1@1": {"width": 0, "height": 0}, } # Model resolutions mapping @@ -237,6 +247,11 @@ class videoModelSearch: "lumaai:1@1": "720p", "lumaai:2@1": "720p", "lumaai:2@2": "720p", + + # Sync Models + "sync:lipsync-2@1": "720p", + "sync:lipsync-2-pro@1": "720p", + "sync:react-1@1": "720p", } MODEL_ARCHITECTURES = [ @@ -252,7 +267,8 @@ class videoModelSearch: "Lightricks", "Ovi", "Runway", - "Luma" + "Luma", + "Sync" ] DEFAULT_DIMENSIONS = {"width": 1024, "height": 576} diff --git a/workflows/Runware_Basic_Image_Inference_Wan_2_5_Preview.json b/workflows/Runware_Basic_Image_Inference_Wan_2_5_Preview.json deleted file mode 100644 index edecfd0..0000000 --- a/workflows/Runware_Basic_Image_Inference_Wan_2_5_Preview.json +++ /dev/null @@ -1 +0,0 @@ -{"id":"0685c977-b8ed-4ce4-8b4c-d8463ed12b69","revision":0,"last_node_id":92,"last_link_id":70,"nodes":[{"id":80,"type":"Runware Model Search","pos":[10.674552917480469,-324.2225646972656],"size":[270,154],"flags":{},"order":0,"mode":0,"inputs":[{"localized_name":"Model Search","name":"Model Search","type":"STRING","widget":{"name":"Model Search"},"link":null},{"localized_name":"Model Architecture","name":"Model Architecture","type":"COMBO","widget":{"name":"Model Architecture"},"link":null},{"localized_name":"ModelType","name":"ModelType","type":"COMBO","widget":{"name":"ModelType"},"link":null},{"localized_name":"ModelList","name":"ModelList","type":"COMBO","widget":{"name":"ModelList"},"link":null},{"localized_name":"Use Search Value","name":"Use Search Value","type":"BOOLEAN","widget":{"name":"Use Search Value"},"link":null}],"outputs":[{"localized_name":"Runware Model","name":"Runware Model","type":"RUNWAREMODEL","links":[67]}],"properties":{"cnr_id":"ComfyUI-Runware","ver":"de0a4c2e258a1f3a104d0f9994f9b7ea66e56736","Node name for S&R":"Runware Model Search"},"widgets_values":["runware:201@10","All","Base Model","vyro:1@5 (Imagineart 1.5 Preview 1.5)",true],"bgcolor":"#5345bf"},{"id":90,"type":"Runware Image Inference Inputs","pos":[-4.500412464141846,-106.89787292480469],"size":[303.0162048339844,210],"flags":{},"order":3,"mode":0,"inputs":[{"localized_name":"image","name":"image","shape":7,"type":"IMAGE","link":68},{"localized_name":"Reference Image 1","name":"Reference Image 1","shape":7,"type":"IMAGE","link":null},{"localized_name":"Reference Image 2","name":"Reference Image 2","shape":7,"type":"IMAGE","link":null},{"localized_name":"Reference Image 3","name":"Reference Image 3","shape":7,"type":"IMAGE","link":null},{"localized_name":"Reference Image 4","name":"Reference Image 4","shape":7,"type":"IMAGE","link":null},{"localized_name":"Reference Tag 1","name":"Reference Tag 1","shape":7,"type":"STRING","widget":{"name":"Reference Tag 1"},"link":null},{"localized_name":"Reference Tag 2","name":"Reference Tag 2","shape":7,"type":"STRING","widget":{"name":"Reference Tag 2"},"link":null},{"localized_name":"Reference Tag 3","name":"Reference Tag 3","shape":7,"type":"STRING","widget":{"name":"Reference Tag 3"},"link":null},{"localized_name":"Reference Tag 4","name":"Reference Tag 4","shape":7,"type":"STRING","widget":{"name":"Reference Tag 4"},"link":null}],"outputs":[{"localized_name":"Inference Inputs","name":"Inference Inputs","type":"RUNWAREIMAGEINFERENCEINPUTS","links":[69]}],"properties":{"cnr_id":"ComfyUI-Runware","ver":"03084f77c620512811a9c1cbc51548bcccfa7f38","Node name for S&R":"Runware Image Inference Inputs"},"widgets_values":["","","",""],"bgcolor":"#5345bf"},{"id":91,"type":"LoadImage","pos":[9.291159629821777,175.53916931152344],"size":[274.080078125,314],"flags":{},"order":2,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[68]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"cnr_id":"comfy-core","ver":"0.3.46","Node name for S&R":"LoadImage"},"widgets_values":["woman.webp","image"]},{"id":88,"type":"Runware Image Inference","pos":[328.7291259765625,-296.4019775390625],"size":[400,998],"flags":{},"order":4,"mode":0,"inputs":[{"localized_name":"Model","name":"Model","type":"RUNWAREMODEL","link":67},{"localized_name":"Accelerator","name":"Accelerator","shape":7,"type":"RUNWAREACCELERATOR","link":null},{"localized_name":"Lora","name":"Lora","shape":7,"type":"RUNWARELORA","link":null},{"localized_name":"Outpainting","name":"Outpainting","shape":7,"type":"RUNWAREOUTPAINT","link":null},{"localized_name":"IPAdapters","name":"IPAdapters","shape":7,"type":"RUNWAREIPAdapter","link":null},{"localized_name":"ControlNet","name":"ControlNet","shape":7,"type":"RUNWARECONTROLNET","link":null},{"localized_name":"Refiner","name":"Refiner","shape":7,"type":"RUNWAREREFINER","link":null},{"localized_name":"seedImage","name":"seedImage","shape":7,"type":"IMAGE","link":null},{"localized_name":"maskImage","name":"maskImage","shape":7,"type":"MASK","link":null},{"localized_name":"Embeddings","name":"Embeddings","shape":7,"type":"RUNWAREEMBEDDING","link":null},{"localized_name":"VAE","name":"VAE","shape":7,"type":"RUNWAREVAE","link":null},{"localized_name":"referenceImages","name":"referenceImages","shape":7,"type":"RUNWAREREFERENCEIMAGES","link":null},{"localized_name":"inputs","name":"inputs","shape":7,"type":"RUNWAREIMAGEINFERENCEINPUTS","link":69},{"localized_name":"providerSettings","name":"providerSettings","shape":7,"type":"RUNWAREPROVIDERSETTINGS","link":70},{"localized_name":"safetyInputs","name":"safetyInputs","shape":7,"type":"RUNWARESAFETYINPUTS","link":null},{"localized_name":"positivePrompt","name":"positivePrompt","type":"STRING","widget":{"name":"positivePrompt"},"link":null},{"localized_name":"negativePrompt","name":"negativePrompt","type":"STRING","widget":{"name":"negativePrompt"},"link":null},{"localized_name":"Multi Inference Mode","name":"Multi Inference Mode","type":"BOOLEAN","widget":{"name":"Multi Inference Mode"},"link":null},{"localized_name":"Prompt Weighting","name":"Prompt Weighting","type":"COMBO","widget":{"name":"Prompt Weighting"},"link":null},{"localized_name":"dimensions","name":"dimensions","type":"COMBO","widget":{"name":"dimensions"},"link":null},{"localized_name":"width","name":"width","type":"INT","widget":{"name":"width"},"link":null},{"localized_name":"height","name":"height","type":"INT","widget":{"name":"height"},"link":null},{"localized_name":"useResolution","name":"useResolution","type":"BOOLEAN","widget":{"name":"useResolution"},"link":null},{"localized_name":"resolution","name":"resolution","type":"COMBO","widget":{"name":"resolution"},"link":null},{"localized_name":"useSteps","name":"useSteps","type":"BOOLEAN","widget":{"name":"useSteps"},"link":null},{"localized_name":"steps","name":"steps","type":"INT","widget":{"name":"steps"},"link":null},{"localized_name":"useScheduler","name":"useScheduler","type":"BOOLEAN","widget":{"name":"useScheduler"},"link":null},{"localized_name":"scheduler","name":"scheduler","type":"COMBO","widget":{"name":"scheduler"},"link":null},{"localized_name":"useCFGScale","name":"useCFGScale","type":"BOOLEAN","widget":{"name":"useCFGScale"},"link":null},{"localized_name":"cfgScale","name":"cfgScale","type":"FLOAT","widget":{"name":"cfgScale"},"link":null},{"localized_name":"useSeed","name":"useSeed","type":"BOOLEAN","widget":{"name":"useSeed"},"link":null},{"localized_name":"seed","name":"seed","type":"INT","widget":{"name":"seed"},"link":null},{"localized_name":"useClipSkip","name":"useClipSkip","type":"BOOLEAN","widget":{"name":"useClipSkip"},"link":null},{"localized_name":"clipSkip","name":"clipSkip","type":"INT","widget":{"name":"clipSkip"},"link":null},{"localized_name":"strength","name":"strength","type":"FLOAT","widget":{"name":"strength"},"link":null},{"localized_name":"Mask Margin","name":"Mask Margin","type":"BOOLEAN","widget":{"name":"Mask Margin"},"link":null},{"localized_name":"maskMargin","name":"maskMargin","type":"INT","widget":{"name":"maskMargin"},"link":null},{"localized_name":"outputFormat","name":"outputFormat","type":"COMBO","widget":{"name":"outputFormat"},"link":null},{"localized_name":"batchSize","name":"batchSize","type":"INT","widget":{"name":"batchSize"},"link":null},{"localized_name":"acceleration","name":"acceleration","type":"COMBO","widget":{"name":"acceleration"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[66]},{"localized_name":"RW-Task","name":"RW-Task","type":"RUNWARETASK","links":null}],"properties":{"cnr_id":"ComfyUI-Runware","ver":"bdc86e015971881247addb07bb8f22f7d85c0062","Node name for S&R":"Runware Image Inference"},"widgets_values":["Change the floral dress to a vintage-style lace long dress with exquisite embroidery details on the collar and cuffs.","",false,"Disabled","Square HD (1024x1024)",1024,1024,false,"1k",true,4,true,"Default",false,6.5,true,2089980774,"randomize",true,0,0.8,false,32,"WEBP",1,"none"],"bgcolor":"#5345bf"},{"id":76,"type":"PreviewImage","pos":[771.1234741210938,-288.4895935058594],"size":[442.2727355957031,469.6363525390625],"flags":{},"order":5,"mode":0,"inputs":[{"localized_name":"images","name":"images","type":"IMAGE","link":66}],"outputs":[],"properties":{"cnr_id":"comfy-core","ver":"0.3.46","Node name for S&R":"PreviewImage"},"widgets_values":[]},{"id":92,"type":"Runware Alibaba Provider Settings","pos":[779.4414672851562,250.9171905517578],"size":[317.0162048339844,82],"flags":{},"order":1,"mode":0,"inputs":[{"localized_name":"usePromptEnhancer","name":"usePromptEnhancer","shape":7,"type":"BOOLEAN","widget":{"name":"usePromptEnhancer"},"link":null},{"localized_name":"promptEnhancer","name":"promptEnhancer","shape":7,"type":"BOOLEAN","widget":{"name":"promptEnhancer"},"link":null}],"outputs":[{"localized_name":"Provider Settings","name":"Provider Settings","type":"RUNWAREPROVIDERSETTINGS","links":[70]}],"properties":{"cnr_id":"ComfyUI-Runware","ver":"03084f77c620512811a9c1cbc51548bcccfa7f38","Node name for S&R":"Runware Alibaba Provider Settings"},"widgets_values":[true,true],"bgcolor":"#5345bf"}],"links":[[66,88,0,76,0,"IMAGE"],[67,80,0,88,0,"RUNWAREMODEL"],[68,91,0,90,0,"IMAGE"],[69,90,0,88,12,"RUNWAREIMAGEINFERENCEINPUTS"],[70,92,0,88,13,"RUNWAREPROVIDERSETTINGS"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.6528554800764321,"offset":[456.35554570381703,391.2377164824513]}},"version":0.4} \ No newline at end of file diff --git a/workflows/Runware_Video_Inference_Sync_LipSync_2.json b/workflows/Runware_Video_Inference_Sync_LipSync_2.json new file mode 100644 index 0000000..4d6829e --- /dev/null +++ b/workflows/Runware_Video_Inference_Sync_LipSync_2.json @@ -0,0 +1 @@ +{"id":"c1c8e6f2-14fb-4684-b152-f1f85161695d","revision":0,"last_node_id":10,"last_link_id":12,"nodes":[{"id":5,"type":"Runware Video Speech Input","pos":[87.85826873779297,335.40789794921875],"size":[400,200],"flags":{},"order":0,"mode":0,"inputs":[{"localized_name":"id","name":"id","type":"STRING","widget":{"name":"id"},"link":null},{"localized_name":"provider","name":"provider","type":"STRING","widget":{"name":"provider"},"link":null},{"localized_name":"voiceId","name":"voiceId","type":"STRING","widget":{"name":"voiceId"},"link":null},{"localized_name":"text","name":"text","type":"STRING","widget":{"name":"text"},"link":null}],"outputs":[{"localized_name":"Speech Input","name":"Speech Input","type":"RUNWARESPEECHINPUT","links":[3]}],"properties":{"cnr_id":"ComfyUI-Runware","ver":"cb36affb8b9c231980996976a5c2becab169443f","Node name for S&R":"Runware Video Speech Input"},"widgets_values":["","elevenlabs","",""],"bgcolor":"#5345bf"},{"id":8,"type":"LoadVideo","pos":[-176.57504272460938,-236.3052978515625],"size":[274.080078125,442.0882263183594],"flags":{},"order":1,"mode":0,"inputs":[{"localized_name":"file","name":"file","type":"COMBO","widget":{"name":"file"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"VIDEO","name":"VIDEO","type":"VIDEO","links":[7]}],"properties":{"cnr_id":"comfy-core","ver":"0.3.46","Node name for S&R":"LoadVideo"},"widgets_values":["T-ONORYMYLoEOB9lXryA2_IKEy3yAyi1evJGBAkXGZx_output.mp4","image"]},{"id":9,"type":"Runware Media Upload","pos":[-174.35516357421875,93.08289337158203],"size":[270,58],"flags":{},"order":5,"mode":0,"inputs":[{"localized_name":"media","name":"media","shape":7,"type":"VIDEO,AUDIO","link":7},{"localized_name":"mediaUUID","name":"mediaUUID","shape":7,"type":"STRING","widget":{"name":"mediaUUID"},"link":null}],"outputs":[{"localized_name":"mediaUUID","name":"mediaUUID","type":"STRING","links":[8]}],"properties":{"cnr_id":"ComfyUI-Runware","ver":"cb36affb8b9c231980996976a5c2becab169443f","Node name for S&R":"Runware Media Upload"},"widgets_values":["3e19dba2-98ca-40cc-8184-3f2375f74acc"],"bgcolor":"#5345bf"},{"id":4,"type":"Runware Video Audio Input","pos":[-178.2024688720703,206.13197326660156],"size":[270.0943298339844,82],"flags":{},"order":2,"mode":0,"inputs":[{"localized_name":"id","name":"id","type":"STRING","widget":{"name":"id"},"link":null},{"localized_name":"source","name":"source","type":"STRING","widget":{"name":"source"},"link":null}],"outputs":[{"localized_name":"Audio Input","name":"Audio Input","type":"RUNWAREAUDIOINPUT","links":[4]}],"properties":{"cnr_id":"ComfyUI-Runware","ver":"cb36affb8b9c231980996976a5c2becab169443f","Node name for S&R":"Runware Video Audio Input"},"widgets_values":["audio-track-1","https://v3.fal.media/files/monkey/k4iyN8bJZWwJXMKH-pO9r_speech.mp3"],"bgcolor":"#5345bf"},{"id":3,"type":"Runware Video Inference Inputs","pos":[150.19256591796875,47.37628936767578],"size":[320.0390625,222],"flags":{},"order":6,"mode":0,"inputs":[{"localized_name":"Image","name":"Image","shape":7,"type":"IMAGE","link":null},{"localized_name":"Frame Images","name":"Frame Images","shape":7,"type":"RUNWAREVIDEOINPUTSFRAMEIMAGES","link":null},{"localized_name":"Audio Inputs","name":"Audio Inputs","shape":7,"type":"RUNWAREAUDIOINPUT","link":4},{"localized_name":"Speech Inputs","name":"Speech Inputs","shape":7,"type":"RUNWARESPEECHINPUT","link":3},{"localized_name":"Mask","name":"Mask","shape":7,"type":"IMAGE","link":null},{"localized_name":"Frame","name":"Frame","shape":7,"type":"IMAGE","link":null},{"localized_name":"References","name":"References","shape":7,"type":"RUNWAREVIDEOINPUTSREFERENCEIMAGES","link":null},{"localized_name":"Reference Videos","name":"Reference Videos","shape":7,"type":"RUNWAREREFERENCEVIDEOS","link":null},{"localized_name":"Audio","name":"Audio","shape":7,"type":"STRING","widget":{"name":"Audio"},"link":null},{"localized_name":"Video","name":"Video","shape":7,"type":"STRING","widget":{"name":"Video"},"link":8}],"outputs":[{"localized_name":"Video Inference Inputs","name":"Video Inference Inputs","type":"RUNWAREVIDEOINFERENCEINPUTS","links":[10]}],"properties":{"cnr_id":"ComfyUI-Runware","ver":"cb36affb8b9c231980996976a5c2becab169443f","Node name for S&R":"Runware Video Inference Inputs"},"widgets_values":["",""],"bgcolor":"#5345bf"},{"id":10,"type":"Runware Video Inference","pos":[533.83203125,-291.465576171875],"size":[400,760],"flags":{},"order":7,"mode":0,"inputs":[{"localized_name":"Model","name":"Model","type":"RUNWAREVIDEOMODEL","link":9},{"localized_name":"frameImages","name":"frameImages","shape":7,"type":"RUNWAREFRAMEIMAGES","link":null},{"localized_name":"referenceImages","name":"referenceImages","shape":7,"type":"RUNWAREREFERENCEIMAGES","link":null},{"localized_name":"providerSettings","name":"providerSettings","shape":7,"type":"RUNWAREPROVIDERSETTINGS","link":12},{"localized_name":"inputAudios","name":"inputAudios","shape":7,"type":"RUNWAREINPUTAUDIOS","link":null},{"localized_name":"referenceVideos","name":"referenceVideos","shape":7,"type":"RUNWAREREFERENCEVIDEOS","link":null},{"localized_name":"inputs","name":"inputs","shape":7,"type":"RUNWAREVIDEOINFERENCEINPUTS","link":10},{"localized_name":"safetyInputs","name":"safetyInputs","shape":7,"type":"RUNWARESAFETYINPUTS","link":null},{"localized_name":"videoAdvancedFeatureInputs","name":"videoAdvancedFeatureInputs","shape":7,"type":"RUNWAREVIDEOADVANCEDFEATUREINPUTS","link":null},{"localized_name":"Accelerator","name":"Accelerator","shape":7,"type":"RUNWAREACCELERATOR","link":null},{"localized_name":"Multi Inference Mode","name":"Multi Inference Mode","type":"BOOLEAN","widget":{"name":"Multi Inference Mode"},"link":null},{"localized_name":"Prompt Weighting","name":"Prompt Weighting","type":"COMBO","widget":{"name":"Prompt Weighting"},"link":null},{"localized_name":"useDuration","name":"useDuration","type":"BOOLEAN","widget":{"name":"useDuration"},"link":null},{"localized_name":"duration","name":"duration","type":"INT","widget":{"name":"duration"},"link":null},{"localized_name":"useFps","name":"useFps","type":"BOOLEAN","widget":{"name":"useFps"},"link":null},{"localized_name":"fps","name":"fps","type":"INT","widget":{"name":"fps"},"link":null},{"localized_name":"outputFormat","name":"outputFormat","type":"COMBO","widget":{"name":"outputFormat"},"link":null},{"localized_name":"useSeed","name":"useSeed","type":"BOOLEAN","widget":{"name":"useSeed"},"link":null},{"localized_name":"seed","name":"seed","type":"INT","widget":{"name":"seed"},"link":null},{"localized_name":"useSteps","name":"useSteps","type":"BOOLEAN","widget":{"name":"useSteps"},"link":null},{"localized_name":"steps","name":"steps","type":"INT","widget":{"name":"steps"},"link":null},{"localized_name":"useBatchSize","name":"useBatchSize","type":"BOOLEAN","widget":{"name":"useBatchSize"},"link":null},{"localized_name":"batchSize","name":"batchSize","type":"INT","widget":{"name":"batchSize"},"link":null},{"localized_name":"acceleration","name":"acceleration","type":"COMBO","widget":{"name":"acceleration"},"link":null},{"localized_name":"positivePrompt","name":"positivePrompt","shape":7,"type":"STRING","widget":{"name":"positivePrompt"},"link":null},{"localized_name":"negativePrompt","name":"negativePrompt","shape":7,"type":"STRING","widget":{"name":"negativePrompt"},"link":null},{"localized_name":"speechVoice","name":"speechVoice","shape":7,"type":"STRING","widget":{"name":"speechVoice"},"link":null},{"localized_name":"speechText","name":"speechText","shape":7,"type":"STRING","widget":{"name":"speechText"},"link":null}],"outputs":[{"localized_name":"VIDEO","name":"VIDEO","type":"VIDEO","links":[11]},{"localized_name":"RW-Task","name":"RW-Task","type":"RUNWARETASK","links":null}],"properties":{"cnr_id":"ComfyUI-Runware","ver":"cb36affb8b9c231980996976a5c2becab169443f","Node name for S&R":"Runware Video Inference"},"widgets_values":[false,"Disabled",false,5,false,24,"mp4",false,542548713,"randomize",false,20,false,1,"none","","","",""],"bgcolor":"#5345bf"},{"id":1,"type":"Runware Video Model Search","pos":[153.17233276367188,-306.729248046875],"size":[287.380859375,290],"flags":{},"order":4,"mode":0,"inputs":[{"localized_name":"Model Search","name":"Model Search","type":"STRING","widget":{"name":"Model Search"},"link":null},{"localized_name":"Model Architecture","name":"Model Architecture","type":"COMBO","widget":{"name":"Model Architecture"},"link":null},{"localized_name":"VideoList","name":"VideoList","type":"COMBO","widget":{"name":"VideoList"},"link":null},{"localized_name":"Use Search Value","name":"Use Search Value","type":"BOOLEAN","widget":{"name":"Use Search Value"},"link":null},{"localized_name":"useCustomDimensions","name":"useCustomDimensions","type":"COMBO","widget":{"name":"useCustomDimensions"},"link":null},{"localized_name":"Width","name":"Width","type":"INT","widget":{"name":"Width"},"link":null},{"localized_name":"Height","name":"Height","type":"INT","widget":{"name":"Height"},"link":null},{"localized_name":"useResolution","name":"useResolution","type":"BOOLEAN","widget":{"name":"useResolution"},"link":null},{"localized_name":"resolution","name":"resolution","type":"COMBO","widget":{"name":"resolution"},"link":null}],"outputs":[{"localized_name":"Runware Video Model","name":"Runware Video Model","type":"RUNWAREVIDEOMODEL","links":[9]},{"localized_name":"Width","name":"Width","type":"INT","links":null},{"localized_name":"Height","name":"Height","type":"INT","links":null}],"properties":{"cnr_id":"ComfyUI-Runware","ver":"cb36affb8b9c231980996976a5c2becab169443f","Node name for S&R":"Runware Video Model Search"},"widgets_values":["","Sync","sync:lipsync-2-pro@1 (Sync LipSync 2 Pro)",false,"Model Default",0,0,false,"1080p"],"bgcolor":"#5345bf"},{"id":7,"type":"Runware Sync Provider Settings","pos":[976.6459350585938,213.06065368652344],"size":[330.76953125,358],"flags":{},"order":3,"mode":0,"inputs":[{"localized_name":"Sync Segment 1","name":"Sync Segment 1","shape":7,"type":"RUNWARESYNCSEGMENT","link":null},{"localized_name":"Sync Segment 2","name":"Sync Segment 2","shape":7,"type":"RUNWARESYNCSEGMENT","link":null},{"localized_name":"Sync Segment 3","name":"Sync Segment 3","shape":7,"type":"RUNWARESYNCSEGMENT","link":null},{"localized_name":"Sync Segment 4","name":"Sync Segment 4","shape":7,"type":"RUNWARESYNCSEGMENT","link":null},{"localized_name":"syncMode","name":"syncMode","type":"COMBO","widget":{"name":"syncMode"},"link":null},{"localized_name":"useEditRegion","name":"useEditRegion","shape":7,"type":"BOOLEAN","widget":{"name":"useEditRegion"},"link":null},{"localized_name":"editRegion","name":"editRegion","shape":7,"type":"COMBO","widget":{"name":"editRegion"},"link":null},{"localized_name":"useEmotionPrompt","name":"useEmotionPrompt","shape":7,"type":"BOOLEAN","widget":{"name":"useEmotionPrompt"},"link":null},{"localized_name":"emotionPrompt","name":"emotionPrompt","shape":7,"type":"COMBO","widget":{"name":"emotionPrompt"},"link":null},{"localized_name":"useTemperature","name":"useTemperature","shape":7,"type":"BOOLEAN","widget":{"name":"useTemperature"},"link":null},{"localized_name":"temperature","name":"temperature","shape":7,"type":"FLOAT","widget":{"name":"temperature"},"link":null},{"localized_name":"useActiveSpeakerDetection","name":"useActiveSpeakerDetection","shape":7,"type":"BOOLEAN","widget":{"name":"useActiveSpeakerDetection"},"link":null},{"localized_name":"activeSpeakerDetection","name":"activeSpeakerDetection","shape":7,"type":"BOOLEAN","widget":{"name":"activeSpeakerDetection"},"link":null},{"localized_name":"useOcclusionDetectionEnabled","name":"useOcclusionDetectionEnabled","shape":7,"type":"BOOLEAN","widget":{"name":"useOcclusionDetectionEnabled"},"link":null},{"localized_name":"occlusionDetectionEnabled","name":"occlusionDetectionEnabled","shape":7,"type":"BOOLEAN","widget":{"name":"occlusionDetectionEnabled"},"link":null}],"outputs":[{"localized_name":"Provider Settings","name":"Provider Settings","type":"RUNWAREPROVIDERSETTINGS","links":[12]}],"properties":{"cnr_id":"ComfyUI-Runware","ver":"cb36affb8b9c231980996976a5c2becab169443f","Node name for S&R":"Runware Sync Provider Settings"},"widgets_values":["bounce",false,"face",false,"happy",true,0.5,true,true,true,false],"bgcolor":"#5345bf"},{"id":6,"type":"SaveVideo","pos":[979.4523315429688,-311.74322509765625],"size":[270,460.60870361328125],"flags":{},"order":8,"mode":0,"inputs":[{"localized_name":"video","name":"video","type":"VIDEO","link":11},{"localized_name":"filename_prefix","name":"filename_prefix","type":"STRING","widget":{"name":"filename_prefix"},"link":null},{"localized_name":"format","name":"format","type":"COMBO","widget":{"name":"format"},"link":null},{"localized_name":"codec","name":"codec","type":"COMBO","widget":{"name":"codec"},"link":null}],"outputs":[],"properties":{"cnr_id":"comfy-core","ver":"0.3.46","Node name for S&R":"SaveVideo"},"widgets_values":["video/ComfyUI","auto","auto"]}],"links":[[3,5,0,3,3,"RUNWARESPEECHINPUT"],[4,4,0,3,2,"RUNWAREAUDIOINPUT"],[7,8,0,9,0,"VIDEO,AUDIO"],[8,9,0,3,9,"STRING"],[9,1,0,10,0,"RUNWAREVIDEOMODEL"],[10,3,0,10,6,"RUNWAREVIDEOINFERENCEINPUTS"],[11,10,0,6,0,"VIDEO"],[12,7,0,10,3,"RUNWAREPROVIDERSETTINGS"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.6528554800764321,"offset":[413.21573396982853,427.2992484270766]}},"version":0.4} \ No newline at end of file