Merge pull request #43 from Runware/feature-Runway-Luma

Add Runway and Luma support
This commit is contained in:
Sirshendu Ganguly
2025-11-11 23:16:34 -05:00
committed by GitHub
37 changed files with 1026 additions and 200 deletions
+7 -2
View File
@@ -28,7 +28,6 @@ from .modules.imageInferenceInputs import imageInferenceInputs
from .modules.videoInference import txt2vid
from .modules.videoModelSearch import videoModelSearch
from .modules.frameImages import RunwareFrameImages
from .modules.providerSettings.providerSettings import RunwareProviderSettings
from .modules.providerSettings.pixverseProviderSettings import RunwarePixverseProviderSettings
from .modules.providerSettings.openaiProviderSettings import RunwareOpenAIProviderSettings
from .modules.providerSettings.bytedanceImageProviderSettings import RunwareBytedanceProviderSettings
@@ -36,6 +35,7 @@ from .modules.providerSettings.briaImageProviderSettings import RunwareBriaProvi
from .modules.audioInference import RunwareAudioInference
from .modules.audioModelSearch import RunwareAudioModelSearch
from .modules.audioSections import RunwareAudioSections
from .modules.audioInferenceInputs import audioInferenceInputs
from .modules.providerSettings.elevenlabsProviderSettings import RunwareElevenLabsProviderSettings
from .modules.providerSettings.elevenlabsProviderSettingsSections import RunwareElevenLabsProviderSettingsSections
from .modules.acceleratorOptions import acceleratorOptions
@@ -48,10 +48,12 @@ from .modules.providerSettings.lightricksProviderSettings import RunwareLightric
from .modules.providerSettings.minimaxProviderSettings import RunwareMiniMaxProviderSettings
from .modules.providerSettings.klingProviderSettings import RunwareKlingProviderSettings
from .modules.providerSettings.runwayProviderSettings import RunwareRunwayProviderSettings
from .modules.providerSettings.lumaProviderSettings import RunwareLumaProviderSettings
from .modules.vectorize import vectorize
from .modules.videoBgRemoval import videoBgRemoval
from .modules.videoUpscaler import videoUpscaler
from .modules.videoInputsReferences import videoInputsReferences
from .modules.videoInputsFrame import RunwareVideoInputsFrameImages
from .modules.safetyInputs import safetyInputs
from .modules.videoAdvancedFeatureInputs import videoAdvancedFeatureInputs
@@ -96,7 +98,6 @@ NODE_CLASS_MAPPINGS = {
"Runware Video Inference": txt2vid,
"Runware Video Model Search": videoModelSearch,
"Runware Frame Images": RunwareFrameImages,
"Runware Provider Settings": RunwareProviderSettings,
"Runware Pixverse Provider Settings": RunwarePixverseProviderSettings,
"Runware OpenAI Provider Settings": RunwareOpenAIProviderSettings,
"Runware Bytedance Provider Settings": RunwareBytedanceProviderSettings,
@@ -104,6 +105,7 @@ NODE_CLASS_MAPPINGS = {
"Runware Audio Inference": RunwareAudioInference,
"Runware Audio Model Search": RunwareAudioModelSearch,
"Runware Audio Sections": RunwareAudioSections,
"Runware Audio Inference Inputs": audioInferenceInputs,
"Runware ElevenLabs Provider Settings": RunwareElevenLabsProviderSettings,
"Runware ElevenLabs Provider Settings Sections": RunwareElevenLabsProviderSettingsSections,
"Runware Accelerator Options": acceleratorOptions,
@@ -114,12 +116,14 @@ NODE_CLASS_MAPPINGS = {
"Runware Video Inference Inputs": videoInferenceInputs,
"Runware Lightricks Provider Settings": RunwareLightricksProviderSettings,
"Runware MiniMax Provider Settings": RunwareMiniMaxProviderSettings,
"Runware Luma Provider Settings": RunwareLumaProviderSettings,
"Runware KlingAI Provider Settings": RunwareKlingProviderSettings,
"Runware Runway Provider Settings": RunwareRunwayProviderSettings,
"Runware Vectorize": vectorize,
"Runware Video Background Removal": videoBgRemoval,
"Runware Video Upscaler": videoUpscaler,
"Runware Video Inputs References": videoInputsReferences,
"Runware Video Inputs Frame Images": RunwareVideoInputsFrameImages,
"Runware Safety Inputs": safetyInputs,
"Runware Video Advanced Feature Inputs": videoAdvancedFeatureInputs,
}
@@ -132,6 +136,7 @@ NODE_DISPLAY_NAME_MAPPINGS = {
"Runware Multi Inference": "Runware Multi Inference [BETA]",
"Runware Video Model Search": "Runware Video Model",
"Runware Audio Model Search": "Runware Audio Model",
"Runware Video Transcription": "Runware Video Caption",
}
WEB_DIRECTORY = "./clientlibs"
+12 -4
View File
@@ -1,6 +1,6 @@
import { app } from "../../scripts/app.js";
import { api } from "../../scripts/api.js";
import { promptEnhanceHandler, syncDimensionsNodeHandler, searchNodeHandler, APIKeyHandler, captionNodeHandler, mediaUUIDHandler, videoTranscriptionHandler, handleCustomErrors, videoInferenceDimensionsHandler, videoModelSearchFilterHandler, useParameterToggleHandler, imageInferenceToggleHandler, upscalerToggleHandler, audioInferenceToggleHandler, acceleratorOptionsToggleHandler, bytedanceProviderSettingsToggleHandler, openaiProviderSettingsToggleHandler } from "./utils.js";
import { promptEnhanceHandler, syncDimensionsNodeHandler, searchNodeHandler, APIKeyHandler, captionNodeHandler, mediaUUIDHandler, videoTranscriptionHandler, handleCustomErrors, videoInferenceDimensionsHandler, videoModelSearchFilterHandler, audioModelSearchFilterHandler, useParameterToggleHandler, imageInferenceToggleHandler, upscalerToggleHandler, audioInferenceToggleHandler, acceleratorOptionsToggleHandler, bytedanceProviderSettingsToggleHandler, openaiProviderSettingsToggleHandler, klingProviderSettingsToggleHandler, lumaProviderSettingsToggleHandler } from "./utils.js";
import { RUNWARE_NODE_TYPES, RUNWARE_NODE_PROPS, SEARCH_TERMS } from "./types.js";
const nodeInitList = [];
@@ -41,11 +41,10 @@ app.registerExtension({
}
}
if(crNodeProps.colorModeOnly === true) return;
const nodeWidgets = node.widgets;
if(nodeWidgets.length <= 0) return;
// Handle node-specific handlers first - each node has its own dedicated handler
// These must be called BEFORE the colorModeOnly check
if(nodeClass === RUNWARE_NODE_TYPES.IMAGEINFERENCE) {
imageInferenceToggleHandler(node);
} else if(nodeClass === RUNWARE_NODE_TYPES.VIDEOINFERENCE) {
@@ -60,10 +59,19 @@ app.registerExtension({
bytedanceProviderSettingsToggleHandler(node);
} else if(nodeClass === RUNWARE_NODE_TYPES.OPENAIPROVIDERSETTINGS) {
openaiProviderSettingsToggleHandler(node);
} else if(nodeClass === RUNWARE_NODE_TYPES.KLINGPROVIDERSETTINGS) {
klingProviderSettingsToggleHandler(node);
} else if(nodeClass === RUNWARE_NODE_TYPES.LUMAPROVIDERSETTINGS) {
lumaProviderSettingsToggleHandler(node);
}
if(crNodeProps.colorModeOnly === true) return;
if(nodeWidgets.length <= 0) return;
if(nodeClass === RUNWARE_NODE_TYPES.VIDEOMODELSEARCH) {
videoModelSearchFilterHandler(node);
} else if(nodeClass === RUNWARE_NODE_TYPES.AUDIOMODELSEARCH) {
audioModelSearchFilterHandler(node);
}
for(const nodeWidget of nodeWidgets) {
+21 -2
View File
@@ -81,7 +81,7 @@ const RUNWARE_NODE_TYPES = {
AUDIOINFERENCE: "Runware Audio Inference",
AUDIOMODELSEARCH: "Runware Audio Model Search",
AUDIOSECTIONS: "Runware Audio Sections",
PROVIDERSETTINGS: "Runware Provider Settings",
AUDIOINFERENCEINPUTS: "Runware Audio Inference Inputs",
PIXVERSEPROVIDERSETTINGS: "Runware Pixverse Provider Settings",
OPENAIPROVIDERSETTINGS: "Runware OpenAI Provider Settings",
BYTEDANCEPROVIDERSETTINGS: "Runware Bytedance Provider Settings",
@@ -89,6 +89,7 @@ const RUNWARE_NODE_TYPES = {
ELEVENLABSPROVIDERSETTINGS: "Runware ElevenLabs Provider Settings",
ELEVENLABSPROVIDERSETTINGSSECTIONS: "Runware ElevenLabs Provider Settings Sections",
VIDUPROVIDERSETTINGS: "Runware Vidu Provider Settings",
LORAMIX: "Runware Lora Mix",
MEDIAUPLOAD: "Runware Media Upload",
INPUTAUDIOS: "Runware Input Audios",
REFERENCEVIDEOS: "Runware Reference Videos",
@@ -96,10 +97,12 @@ const RUNWARE_NODE_TYPES = {
LIGHTRICKSPROVIDERSETTINGS: "Runware Lightricks Provider Settings",
KLINGPROVIDERSETTINGS: "Runware KlingAI Provider Settings",
RUNWAYPROVIDERSETTINGS: "Runware Runway Provider Settings",
LUMAPROVIDERSETTINGS: "Runware Luma Provider Settings",
VECTORIZE: "Runware Vectorize",
VIDEOBGREMOVAL: "Runware Video Background Removal",
VIDEOUPSCALER: "Runware Video Upscaler",
VIDEOINPUTSREFERENCES: "Runware Video Inputs References",
VIDEOINPUTSFRAMEIMAGES: "Runware Video Inputs Frame Images",
ACCELERATOROPTIONS: "Runware Accelerator Options",
VIDEOTRANSCRIPTION: "Runware Video Transcription",
MINIMAXPROVIDERSETTINGS: "Runware MiniMax Provider Settings",
@@ -243,7 +246,7 @@ const RUNWARE_NODE_PROPS = {
bgColor: DEFAULT_BGCOLOR,
colorModeOnly: true,
},
[RUNWARE_NODE_TYPES.PROVIDERSETTINGS]: {
[RUNWARE_NODE_TYPES.IMAGEINFERENCEINPUTS]: {
bgColor: DEFAULT_BGCOLOR,
colorModeOnly: true,
},
@@ -275,6 +278,10 @@ const RUNWARE_NODE_PROPS = {
bgColor: DEFAULT_BGCOLOR,
colorModeOnly: true,
},
[RUNWARE_NODE_TYPES.LORAMIX]: {
bgColor: DEFAULT_BGCOLOR,
colorModeOnly: true,
},
[RUNWARE_NODE_TYPES.MEDIAUPLOAD]: {
bgColor: DEFAULT_BGCOLOR,
colorModeOnly: true,
@@ -303,6 +310,10 @@ const RUNWARE_NODE_PROPS = {
bgColor: DEFAULT_BGCOLOR,
colorModeOnly: true,
},
[RUNWARE_NODE_TYPES.LUMAPROVIDERSETTINGS]: {
bgColor: DEFAULT_BGCOLOR,
colorModeOnly: true,
},
[RUNWARE_NODE_TYPES.VECTORIZE]: {
bgColor: DEFAULT_BGCOLOR,
},
@@ -316,6 +327,14 @@ const RUNWARE_NODE_PROPS = {
bgColor: DEFAULT_BGCOLOR,
colorModeOnly: true,
},
[RUNWARE_NODE_TYPES.VIDEOINPUTSFRAMEIMAGES]: {
bgColor: DEFAULT_BGCOLOR,
colorModeOnly: true,
},
[RUNWARE_NODE_TYPES.AUDIOINFERENCEINPUTS]: {
bgColor: DEFAULT_BGCOLOR,
colorModeOnly: true,
},
[RUNWARE_NODE_TYPES.ACCELERATOROPTIONS]: {
bgColor: DEFAULT_BGCOLOR,
colorModeOnly: true,
+264
View File
@@ -940,6 +940,10 @@ function audioInferenceToggleHandler(audioInferenceNode) {
const positivePromptWidget = audioInferenceNode.widgets.find(w => w.name === "positivePrompt");
const useDurationWidget = audioInferenceNode.widgets.find(w => w.name === "useDuration");
const durationWidget = audioInferenceNode.widgets.find(w => w.name === "duration");
const useSampleRateWidget = audioInferenceNode.widgets.find(w => w.name === "useSampleRate");
const sampleRateWidget = audioInferenceNode.widgets.find(w => w.name === "sampleRate");
const useBitrateWidget = audioInferenceNode.widgets.find(w => w.name === "useBitrate");
const bitrateWidget = audioInferenceNode.widgets.find(w => w.name === "bitrate");
// Helper function to toggle widget enabled state (exact same pattern)
function toggleWidgetState(useWidget, paramWidget, paramName) {
@@ -990,6 +994,14 @@ function audioInferenceToggleHandler(audioInferenceNode) {
if (useDurationWidget && durationWidget) {
toggleWidgetState(useDurationWidget, durationWidget, "duration");
}
if (useSampleRateWidget && sampleRateWidget) {
toggleWidgetState(useSampleRateWidget, sampleRateWidget, "sampleRate");
}
if (useBitrateWidget && bitrateWidget) {
toggleWidgetState(useBitrateWidget, bitrateWidget, "bitrate");
}
}
function acceleratorOptionsToggleHandler(acceleratorNode) {
@@ -1523,6 +1535,7 @@ function videoModelSearchFilterHandler(videoModelSearchNode) {
const videoListWidget = videoModelSearchNode.widgets.find(w => w.name === "VideoList");
const widthWidget = videoModelSearchNode.widgets.find(w => w.name === "Width");
const heightWidget = videoModelSearchNode.widgets.find(w => w.name === "Height");
const useResolutionWidget = videoModelSearchNode.widgets.find(w => w.name === "useResolution");
if (!modelArchWidget || !videoListWidget) return;
@@ -1569,6 +1582,15 @@ function videoModelSearchFilterHandler(videoModelSearchNode) {
"Ovi": [
"runware:190@1 (Ovi)",
],
"Runway": [
"runway:2@1 (Runway Aleph)",
"runway:1@1 (Runway Gen-4 Turbo)",
],
"Luma": [
"lumaai:1@1 (Luma Ray 1.6)",
"lumaai:2@1 (Luma Ray 2)",
"lumaai:2@2 (Luma Ray 2 Flash)",
],
};
const MODEL_DIMENSIONS = {
@@ -1614,16 +1636,46 @@ function videoModelSearchFilterHandler(videoModelSearchNode) {
"lightricks:2@0": {"width": 1920, "height": 1080},
"lightricks:2@1": {"width": 1920, "height": 1080},
"runware:190@1": {"width": 0, "height": 0},
"runway:2@1": {"width": 1280, "height": 720},
"runway:1@1": {"width": 1280, "height": 720},
"lumaai:1@1": {"width": 1080, "height": 720},
"lumaai:2@1": {"width": 1080, "height": 720},
"lumaai:2@2": {"width": 1080, "height": 720},
};
const DEFAULT_DIMENSIONS = {"width": 1024, "height": 576};
function setWidthHeightEnabled(enabled) {
if (!widthWidget || !heightWidget) return;
[widthWidget, heightWidget].forEach(widget => {
widget.disabled = !enabled;
if (widget.inputEl) {
widget.inputEl.disabled = !enabled;
widget.inputEl.style.opacity = enabled ? "1" : "0.5";
widget.inputEl.style.cursor = enabled ? "text" : "not-allowed";
widget.inputEl.readOnly = !enabled;
}
});
}
function updateDimensions() {
if (!widthWidget || !heightWidget || !videoListWidget) return;
const selectedModel = videoListWidget.value;
if (!selectedModel) return;
if (useResolutionWidget && !useResolutionWidget.value) {
setWidthHeightEnabled(false);
if (widthWidget.callback) widthWidget.callback(0, "customSetOperation");
widthWidget.value = 0;
if (heightWidget.callback) heightWidget.callback(0, "customSetOperation");
heightWidget.value = 0;
videoModelSearchNode.setDirtyCanvas(true);
return;
}
setWidthHeightEnabled(true);
const modelCode = selectedModel.split(" (")[0];
const dims = MODEL_DIMENSIONS[modelCode] || DEFAULT_DIMENSIONS;
@@ -1666,10 +1718,219 @@ function videoModelSearchFilterHandler(videoModelSearchNode) {
if (videoListWidget) {
appendWidgetCB(videoListWidget, updateDimensions);
}
if (useResolutionWidget) {
appendWidgetCB(useResolutionWidget, () => {
updateDimensions();
});
}
filterModelList();
updateDimensions();
}
function audioModelSearchFilterHandler(audioModelSearchNode) {
const modelProviderWidget = audioModelSearchNode.widgets.find(w => w.name === "Model Provider");
const audioListWidget = audioModelSearchNode.widgets.find(w => w.name === "AudioList");
if (!modelProviderWidget || !audioListWidget) return;
const AUDIO_MODELS = {
"ElevenLabs": [
"elevenlabs:1@1 (ElevenLabs Multilingual v2)",
"elevenlabs:2@1 (ElevenLabs Multilingual v2 Turbo)",
"elevenlabs:3@1 (ElevenLabs Monolingual v1)",
],
"KlingAI": [
"klingai:8@1 (KlingAI Audio)",
],
};
function filterModelList() {
const selectedProvider = modelProviderWidget.value;
let filteredModels = [];
if (selectedProvider === "All") {
Object.values(AUDIO_MODELS).forEach(models => filteredModels.push(...models));
} else if (AUDIO_MODELS[selectedProvider]) {
filteredModels = AUDIO_MODELS[selectedProvider];
}
if (filteredModels.length > 0) {
const currentValue = audioListWidget.value;
audioListWidget.options.values = filteredModels;
if (!filteredModels.includes(currentValue)) {
audioListWidget.value = filteredModels[0];
}
audioModelSearchNode.setDirtyCanvas(true);
}
}
appendWidgetCB(modelProviderWidget, filterModelList);
filterModelList();
}
function klingProviderSettingsToggleHandler(klingNode) {
// Find all "use" parameter widgets for KlingAI Provider Settings (EXACT same pattern as imageInferenceToggleHandler)
const useCameraControlWidget = klingNode.widgets.find(w => w.name === "useCameraControl");
const cameraControlWidget = klingNode.widgets.find(w => w.name === "cameraControl");
const useSoundVolumeWidget = klingNode.widgets.find(w => w.name === "useSoundVolume");
const soundVolumeWidget = klingNode.widgets.find(w => w.name === "soundVolume");
const useOriginalAudioVolumeWidget = klingNode.widgets.find(w => w.name === "useOriginalAudioVolume");
const originalAudioVolumeWidget = klingNode.widgets.find(w => w.name === "originalAudioVolume");
const useSoundEffectPromptWidget = klingNode.widgets.find(w => w.name === "useSoundEffectPrompt");
const soundEffectPromptWidget = klingNode.widgets.find(w => w.name === "soundEffectPrompt");
const useBgmPromptWidget = klingNode.widgets.find(w => w.name === "useBgmPrompt");
const bgmPromptWidget = klingNode.widgets.find(w => w.name === "bgmPrompt");
const useAsmrModeWidget = klingNode.widgets.find(w => w.name === "useAsmrMode");
const asmrModeWidget = klingNode.widgets.find(w => w.name === "asmrMode");
// Helper function to toggle widget enabled state (EXACT same pattern as imageInferenceToggleHandler)
function toggleWidgetState(useWidget, paramWidget, paramName) {
if (!useWidget || !paramWidget) return;
function toggleEnabled() {
const enabled = useWidget.value === true;
// Disable/enable widgets using inputEl if available (exact same pattern)
if (paramWidget.inputEl) {
paramWidget.inputEl.disabled = !enabled;
paramWidget.inputEl.style.opacity = enabled ? "1" : "0.5";
paramWidget.inputEl.style.cursor = enabled ? "text" : "not-allowed";
paramWidget.inputEl.readOnly = !enabled;
}
// Also set widget property
paramWidget.disabled = !enabled;
// Fallback: try to find inputs via DOM if inputEl is not available
if (!paramWidget.inputEl) {
const nodeElement = klingNode.htmlElements?.widgetsContainer || klingNode.htmlElements;
if (nodeElement) {
const input = nodeElement.querySelector(`input[name="${paramName}"], textarea[name="${paramName}"], select[name="${paramName}"]`);
if (input) {
input.disabled = !enabled;
input.style.opacity = enabled ? "1" : "0.5";
input.style.cursor = enabled ? "text" : "not-allowed";
input.readOnly = !enabled;
if (input.tagName === "SELECT") {
input.style.pointerEvents = enabled ? "auto" : "none";
}
}
}
}
klingNode.setDirtyCanvas(true);
}
// Set up callback (exact same pattern as imageInferenceToggleHandler)
appendWidgetCB(useWidget, () => {
setTimeout(toggleEnabled, 50);
});
// Initial call to set initial state (exact same pattern)
setTimeout(toggleEnabled, 100);
}
// Set up all toggle handlers (exact same pattern as imageInferenceToggleHandler)
if (useCameraControlWidget && cameraControlWidget) {
toggleWidgetState(useCameraControlWidget, cameraControlWidget, "cameraControl");
}
if (useSoundVolumeWidget && soundVolumeWidget) {
toggleWidgetState(useSoundVolumeWidget, soundVolumeWidget, "soundVolume");
}
if (useOriginalAudioVolumeWidget && originalAudioVolumeWidget) {
toggleWidgetState(useOriginalAudioVolumeWidget, originalAudioVolumeWidget, "originalAudioVolume");
}
if (useSoundEffectPromptWidget && soundEffectPromptWidget) {
toggleWidgetState(useSoundEffectPromptWidget, soundEffectPromptWidget, "soundEffectPrompt");
}
if (useBgmPromptWidget && bgmPromptWidget) {
toggleWidgetState(useBgmPromptWidget, bgmPromptWidget, "bgmPrompt");
}
if (useAsmrModeWidget && asmrModeWidget) {
toggleWidgetState(useAsmrModeWidget, asmrModeWidget, "asmrMode");
}
}
function lumaProviderSettingsToggleHandler(lumaNode) {
// Find all "use" parameter widgets for Luma Provider Settings
const useLoopWidget = lumaNode.widgets.find(w => w.name === "useLoop");
const loopWidget = lumaNode.widgets.find(w => w.name === "loop");
const useConcept1Widget = lumaNode.widgets.find(w => w.name === "useConcept1");
const concept1Widget = lumaNode.widgets.find(w => w.name === "concept1");
const useConcept2Widget = lumaNode.widgets.find(w => w.name === "useConcept2");
const concept2Widget = lumaNode.widgets.find(w => w.name === "concept2");
const useConcept3Widget = lumaNode.widgets.find(w => w.name === "useConcept3");
const concept3Widget = lumaNode.widgets.find(w => w.name === "concept3");
const useConcept4Widget = lumaNode.widgets.find(w => w.name === "useConcept4");
const concept4Widget = lumaNode.widgets.find(w => w.name === "concept4");
function toggleWidgetState(useWidget, paramWidget, paramName) {
if (!useWidget || !paramWidget) return;
function toggleEnabled() {
const enabled = useWidget.value === true;
if (paramWidget.inputEl) {
paramWidget.inputEl.disabled = !enabled;
paramWidget.inputEl.style.opacity = enabled ? "1" : "0.5";
paramWidget.inputEl.style.cursor = enabled ? "text" : "not-allowed";
paramWidget.inputEl.readOnly = !enabled;
}
paramWidget.disabled = !enabled;
if (!paramWidget.inputEl) {
const nodeElement = lumaNode.htmlElements?.widgetsContainer || lumaNode.htmlElements;
if (nodeElement) {
const input = nodeElement.querySelector(`input[name="${paramName}"], textarea[name="${paramName}"], select[name="${paramName}"]`);
if (input) {
input.disabled = !enabled;
input.style.opacity = enabled ? "1" : "0.5";
input.style.cursor = enabled ? "text" : "not-allowed";
input.readOnly = !enabled;
if (input.tagName === "SELECT") {
input.style.pointerEvents = enabled ? "auto" : "none";
}
}
}
}
lumaNode.setDirtyCanvas(true);
}
appendWidgetCB(useWidget, () => {
setTimeout(toggleEnabled, 50);
});
setTimeout(toggleEnabled, 100);
}
if (useLoopWidget && loopWidget) {
toggleWidgetState(useLoopWidget, loopWidget, "loop");
}
if (useConcept1Widget && concept1Widget) {
toggleWidgetState(useConcept1Widget, concept1Widget, "concept1");
}
if (useConcept2Widget && concept2Widget) {
toggleWidgetState(useConcept2Widget, concept2Widget, "concept2");
}
if (useConcept3Widget && concept3Widget) {
toggleWidgetState(useConcept3Widget, concept3Widget, "concept3");
}
if (useConcept4Widget && concept4Widget) {
toggleWidgetState(useConcept4Widget, concept4Widget, "concept4");
}
}
export {
notifyUser,
promptEnhanceHandler,
@@ -1682,6 +1943,7 @@ export {
APIKeyHandler,
videoInferenceDimensionsHandler,
videoModelSearchFilterHandler,
audioModelSearchFilterHandler,
useParameterToggleHandler,
imageInferenceToggleHandler,
upscalerToggleHandler,
@@ -1689,4 +1951,6 @@ export {
acceleratorOptionsToggleHandler,
bytedanceProviderSettingsToggleHandler,
openaiProviderSettingsToggleHandler,
klingProviderSettingsToggleHandler,
lumaProviderSettingsToggleHandler,
};
+60 -12
View File
@@ -1,4 +1,3 @@
import json
import uuid
import requests
import torch
@@ -47,6 +46,10 @@ class RunwareAudioInference:
"step": 1,
"tooltip": "Length of generated audio in seconds (10-300)"
}),
"useSampleRate": ("BOOLEAN", {
"default": True,
"tooltip": "Enable/disable sampleRate parameter in API request"
}),
"sampleRate": ("INT", {
"default": 22050,
"min": 8000,
@@ -54,6 +57,10 @@ class RunwareAudioInference:
"step": 1,
"tooltip": "Sample rate of the generated audio in Hz (8000-48000)"
}),
"useBitrate": ("BOOLEAN", {
"default": True,
"tooltip": "Enable/disable bitrate parameter in API request"
}),
"bitrate": ("INT", {
"default": 32,
"min": 32,
@@ -77,6 +84,9 @@ class RunwareAudioInference:
}),
},
"optional": {
"inputs": ("RUNWAREAUDIOINFERENCEINPUTS", {
"tooltip": "Custom inputs for audio generation (e.g., video URL for audio extraction)"
}),
"providerSettings": ("RUNWAREPROVIDERSETTINGS", {
"tooltip": "Provider-specific configuration settings"
}),
@@ -110,12 +120,15 @@ class RunwareAudioInference:
"model": kwargs.get("model", ""),
"duration": kwargs.get("duration", 30),
"useDuration": kwargs.get("useDuration", True),
"sampleRate": kwargs.get("sampleRate", 44100),
"bitrate": kwargs.get("bitrate", 128),
"sampleRate": kwargs.get("sampleRate", 22050),
"useSampleRate": kwargs.get("useSampleRate", True),
"bitrate": kwargs.get("bitrate", 32),
"useBitrate": kwargs.get("useBitrate", True),
"outputType": kwargs.get("outputType", "URL"),
"outputFormat": kwargs.get("outputFormat", "MP3"),
"negativePrompt": kwargs.get("negativePrompt", ""),
"numberResults": kwargs.get("numberResults", 1),
"inputs": kwargs.get("inputs", None),
"providerSettings": kwargs.get("providerSettings", None),
}
@@ -124,8 +137,9 @@ class RunwareAudioInference:
if not providerSettings or not isinstance(providerSettings, dict):
return False
elevenlabsSettings = providerSettings.get("elevenlabs", {})
musicSettings = elevenlabsSettings.get("music", {})
# Provider settings is now flat (e.g., {"music": {...}})
# Check if it has music.compositionPlan.sections
musicSettings = providerSettings.get("music", {})
compositionPlan = musicSettings.get("compositionPlan", {})
sections = compositionPlan.get("sections", [])
@@ -144,12 +158,19 @@ class RunwareAudioInference:
"deliveryMethod": "sync",
"includeCost": True,
"numberResults": params["numberResults"],
"audioSettings": {
"sampleRate": params["sampleRate"],
"bitrate": params["bitrate"]
}
}]
# Build audioSettings conditionally based on use flags
audioSettings = {}
if params["useSampleRate"]:
audioSettings["sampleRate"] = params["sampleRate"]
if params["useBitrate"]:
audioSettings["bitrate"] = params["bitrate"]
# Only add audioSettings if at least one setting is enabled
if audioSettings:
genConfig[0]["audioSettings"] = audioSettings
# Handle sections - disable duration if sections are provided
hasSections = self._hasSections(params["providerSettings"])
if hasSections:
@@ -173,18 +194,45 @@ class RunwareAudioInference:
if params["negativePrompt"]:
genConfig[0]["negativePrompt"] = params["negativePrompt"]
# Handle inputs - merge custom inputs from Audio Inference Inputs node
if params["inputs"] is not None:
# Merge inputs from audio inference inputs node
if "inputs" not in genConfig[0]:
genConfig[0]["inputs"] = {}
# Merge each input from inputs
for key, value in params["inputs"].items():
genConfig[0]["inputs"][key] = value
print(f"[DEBUG] Audio inference inputs merged: {rwUtils.sanitize_for_logging(params['inputs'])}")
print(f"[DEBUG] Final genConfig inputs: {rwUtils.sanitize_for_logging(genConfig[0].get('inputs', {}))}")
# Handle providerSettings - extract provider name from model and wrap with provider name (same pattern as video inference)
if params["providerSettings"] is not None:
genConfig[0]["providerSettings"] = params["providerSettings"]
# Extract provider name from model (e.g., "klingai:8@1" -> "klingai", "elevenlabs:1@1" -> "elevenlabs")
provider_name = params["model"].split(":")[0] if ":" in params["model"] else params["model"]
# If providerSettings is a dictionary, create the correct API format
if isinstance(params["providerSettings"], dict):
# Create the providerSettings object with provider name as key
final_provider_settings = {
provider_name: params["providerSettings"]
}
genConfig[0]["providerSettings"] = final_provider_settings
print(f"[DEBUG] Provider settings wrapped with provider name: {rwUtils.sanitize_for_logging(final_provider_settings)}")
else:
# If it's just a string, use it directly
genConfig[0]["providerSettings"] = params["providerSettings"]
print(f"[DEBUG] Sending Audio Inference Request:")
print(f"[DEBUG] Request Payload: {json.dumps(genConfig, indent=2)}")
print(f"[DEBUG] Request Payload: {rwUtils.safe_json_dumps(genConfig, indent=2)}")
return genConfig
def _logResponse(self, genResult):
"""Log the API response for debugging"""
print(f"[DEBUG] Received Audio Inference Response:")
print(f"[DEBUG] Response: {json.dumps(genResult, indent=2)}")
print(f"[DEBUG] Response: {rwUtils.safe_json_dumps(genResult, indent=2)}")
def _validateResponse(self, genResult):
"""Validate API response and raise errors if needed"""
+46
View File
@@ -0,0 +1,46 @@
from .utils import runwareUtils as rwUtils
from typing import Dict, Any
class audioInferenceInputs:
"""Audio Inference Inputs node for configuring audio generation inputs"""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {},
"optional": {
"Video": ("STRING", {
"tooltip": "Video URL or mediaUUID for audio generation. Can be a direct URL or a mediaUUID from Runware Media Upload node.",
"default": "",
}),
}
}
DESCRIPTION = "Configure custom inputs for Runware Audio Inference, including video input for audio extraction or generation."
FUNCTION = "createInputs"
RETURN_TYPES = ("RUNWAREAUDIOINFERENCEINPUTS",)
RETURN_NAMES = ("Audio Inference Inputs",)
CATEGORY = "Runware"
def createInputs(self, **kwargs) -> tuple[Dict[str, Any]]:
"""Create audio inference inputs from provided parameters"""
video = kwargs.get("Video", None)
inputs = {}
if video is not None and video.strip() != "":
inputs["video"] = video.strip()
return (inputs,)
# Node class mappings
NODE_CLASS_MAPPINGS = {
"RunwareAudioInferenceInputs": audioInferenceInputs,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"RunwareAudioInferenceInputs": "Runware Audio Inference Inputs",
}
+60 -14
View File
@@ -1,32 +1,78 @@
class RunwareAudioModelSearch:
"""Audio Model Search node for searching audio models"""
def __init__(self):
pass
# Define the audio models by provider
AUDIO_MODELS = {
"ElevenLabs": [
"elevenlabs:1@1 (ElevenLabs Multilingual v2)",
"elevenlabs:2@1 (ElevenLabs Multilingual v2 Turbo)",
"elevenlabs:3@1 (ElevenLabs Monolingual v1)",
],
"KlingAI": [
"klingai:8@1 (KlingAI Audio)",
],
}
MODEL_PROVIDERS = [
"All",
"ElevenLabs",
"KlingAI",
]
@classmethod
def INPUT_TYPES(cls):
allModels = cls._getAllModels()
defaultModel = "elevenlabs:1@1 (ElevenLabs Multilingual v2)"
return {
"required": {
"searchQuery": ("STRING", {
"default": "elevenlabs:1@1",
"tooltip": "Search for audio models by name, provider, or capabilities"
"Model Search": ("STRING", {
"tooltip": "Search For Specific Audio Model By Name Or Code (eg: elevenlabs, klingai).",
}),
"provider": (["elevenlabs"], {
"default": "elevenlabs",
"tooltip": "Currently only ElevenLabs is supported"
"Model Provider": (cls.MODEL_PROVIDERS, {
"tooltip": "Choose Audio Model Provider To Filter Results.",
"default": "ElevenLabs",
}),
"AudioList": (allModels, {
"tooltip": "Audio Model Results Will Show UP Here So You Could Choose From.",
"default": defaultModel,
}),
"Use Search Value": ("BOOLEAN", {
"tooltip": "When Enabled, the value you've set in the search input will be used instead.\n\nThis is useful in case the model search API is down or you prefer to set the model manually.",
"default": False,
"label_on": "Enabled",
"label_off": "Disabled",
}),
}
}
@classmethod
def _getAllModels(cls):
"""Get all models from all providers"""
allModels = []
for models in cls.AUDIO_MODELS.values():
allModels.extend(models)
return allModels
RETURN_TYPES = ("RUNWAREAUDIOMODEL",)
RETURN_NAMES = ("model",)
FUNCTION = "searchModels"
CATEGORY = "Runware/Audio"
CATEGORY = "Runware"
DESCRIPTION = "Directly Search and Connect Audio Models to Runware Audio Inference Nodes In ComfyUI."
@classmethod
def VALIDATE_INPUTS(cls, AudioList):
return True
def searchModels(self, **kwargs):
"""Search for audio models"""
searchQuery = kwargs.get("searchQuery", "")
provider = kwargs.get("provider", "elevenlabs")
"""Search and return audio model"""
enableSearchValue = kwargs.get("Use Search Value", False)
searchInput = kwargs.get("Model Search")
return (searchQuery,)
if enableSearchValue:
modelAirCode = searchInput
else:
currentModel = kwargs.get("AudioList")
modelAirCode = currentModel.split(" (")[0]
return (modelAirCode,)
+2 -3
View File
@@ -1,5 +1,4 @@
from .utils import runwareUtils as rwUtils
import json
class bgremoval:
RUNWARE_RMBG_MODELS = {
@@ -155,14 +154,14 @@ class bgremoval:
# Debug: Print the request being sent
print(f"[DEBUG] Sending Background Removal Request:", flush=True)
print(f"[DEBUG] Request Payload: {json.dumps([genConfig], indent=2)}", flush=True)
print(f"[DEBUG] Request Payload: {rwUtils.safe_json_dumps([genConfig], indent=2)}", flush=True)
try:
genResult = rwUtils.inferenecRequest([genConfig])
# Debug: Print the response received
print(f"[DEBUG] Received Background Removal Response:", flush=True)
print(f"[DEBUG] Response: {json.dumps(genResult, indent=2)}", flush=True)
print(f"[DEBUG] Response: {rwUtils.safe_json_dumps(genResult, indent=2)}", flush=True)
except Exception as e:
print(f"[DEBUG] Error in Background Removal Request: {str(e)}", flush=True)
raise
+2 -5
View File
@@ -1,5 +1,4 @@
from .utils import runwareUtils as rwUtils
import json
class txt2img:
@@ -350,15 +349,13 @@ class txt2img:
else:
# Debug: Print the request being sent
print(f"[DEBUG] Sending Image Inference Request:")
print(f"[DEBUG] Request Payload: {json.dumps(genConfig, indent=2)}")
print(f"[DEBUG] CFGScale value from kwargs: {cfgScale}")
print(f"[DEBUG] Request Payload: {rwUtils.safe_json_dumps(genConfig, indent=2)}")
genResult = rwUtils.inferenecRequest(genConfig)
# Debug: Print the response received
print(f"[DEBUG] Received Image Inference Response:")
print(f"[DEBUG] Response: {json.dumps(genResult, indent=2)}")
print(f"[DEBUG] Response: {rwUtils.safe_json_dumps(genResult, indent=2)}")
images = rwUtils.convertImageB64List(genResult)
return (images, None)
+55 -10
View File
@@ -15,15 +15,31 @@ class imageInferenceInputs:
"Reference Image 1": ("IMAGE", {
"tooltip": "Specifies Reference Image 1 for the inputs. These reference images help guide the image generation process."
}),
"Reference Tag 1": ("STRING", {
"tooltip": "Optional tag describing Reference Image 1. Leave empty to omit.",
"default": "",
}),
"Reference Image 2": ("IMAGE", {
"tooltip": "Specifies Reference Image 2 for the inputs. These reference images help guide the image generation process."
}),
"Reference Tag 2": ("STRING", {
"tooltip": "Optional tag describing Reference Image 2. Leave empty to omit.",
"default": "",
}),
"Reference Image 3": ("IMAGE", {
"tooltip": "Specifies Reference Image 3 for the inputs. These reference images help guide the image generation process."
}),
"Reference Tag 3": ("STRING", {
"tooltip": "Optional tag describing Reference Image 3. Leave empty to omit.",
"default": "",
}),
"Reference Image 4": ("IMAGE", {
"tooltip": "Specifies Reference Image 4 for the inputs. These reference images help guide the image generation process."
}),
"Reference Tag 4": ("STRING", {
"tooltip": "Optional tag describing Reference Image 4. Leave empty to omit.",
"default": "",
}),
}
}
@@ -37,28 +53,57 @@ class imageInferenceInputs:
"""Create image inference inputs from provided parameters"""
image = kwargs.get("image", None)
refImage1 = kwargs.get("Reference Image 1", None)
refTag1 = kwargs.get("Reference Tag 1", "")
refImage2 = kwargs.get("Reference Image 2", None)
refTag2 = kwargs.get("Reference Tag 2", "")
refImage3 = kwargs.get("Reference Image 3", None)
refTag3 = kwargs.get("Reference Tag 3", "")
refImage4 = kwargs.get("Reference Image 4", None)
refTag4 = kwargs.get("Reference Tag 4", "")
inputs = {}
if image is not None:
inputs["image"] = rwUtils.convertTensor2IMG(image)
references = self._collectReferences(refImage1, refImage2, refImage3, refImage4)
references = self._collectReferences(
refImage1, refTag1,
refImage2, refTag2,
refImage3, refTag3,
refImage4, refTag4,
)
if len(references) > 0:
inputs["references"] = references
inputs["referenceImages"] = references
return (inputs,)
def _collectReferences(self, refImage1, refImage2, refImage3, refImage4):
def _collectReferences(self, *referenceArgs):
"""Collect and convert reference images to list"""
# Build list of (image, tag) pairs from the flat arguments
referencePairs = list(zip(referenceArgs[0::2], referenceArgs[1::2]))
# Determine if any tag is provided
hasTags = any(
isinstance(tag, str) and tag.strip() != ""
for _, tag in referencePairs
)
if not hasTags:
# Use legacy behavior: simple list of images
return [
rwUtils.convertTensor2IMG(image)
for image, _ in referencePairs
if image is not None
]
references = []
refImages = [refImage1, refImage2, refImage3, refImage4]
for refImage in refImages:
if refImage is not None:
references.append(rwUtils.convertTensor2IMG(refImage))
for image, tag in referencePairs:
if image is None:
continue
entry = {"image": rwUtils.convertTensor2IMG(image)}
if isinstance(tag, str) and tag.strip() != "":
entry["tag"] = tag.strip()
references.append(entry)
return references
+8 -6
View File
@@ -11,6 +11,8 @@ from .utils.runwareUtils import (
RUNWARE_API_BASE_URL,
generalRequestWrapper,
genRandUUID,
sanitize_for_logging,
safe_json_dumps,
sendMediaUUID,
)
@@ -202,7 +204,7 @@ class RunwareMediaUpload:
mediaData = self._stripDataUriPrefix(mediaDataUri)
uploadConfig = self._buildUploadConfig(taskUuid, mediaData)
print(f"[Debug] Upload config: {uploadConfig[0]}")
print(f"[Debug] Upload config: {sanitize_for_logging(uploadConfig[0])}")
print(f"[Debug] API URL: {RUNWARE_API_BASE_URL}")
uploadResult = self._makeUploadRequest(uploadConfig)
@@ -279,10 +281,10 @@ class RunwareMediaUpload:
def _validateUploadResponse(self, uploadResult):
"""Validate upload response"""
print(f"[Debug] Upload response: {uploadResult}")
print(f"[Debug] Upload response: {safe_json_dumps(uploadResult, indent=2) if isinstance(uploadResult, (dict, list)) else uploadResult}")
if "errors" in uploadResult:
print(f"[Debug] Upload error: {uploadResult}")
print(f"[Debug] Upload error: {safe_json_dumps(uploadResult, indent=2) if isinstance(uploadResult, (dict, list)) else uploadResult}")
raise Exception(f"Upload failed: {uploadResult}")
def _pollForResult(self, taskUuid):
@@ -344,14 +346,14 @@ class RunwareMediaUpload:
print(f"[Debug] Failed to parse JSON in poll: {str(e)}")
return None
print(f"[Debug] Poll response: {pollResultJson}")
print(f"[Debug] Poll response: {safe_json_dumps(pollResultJson, indent=2) if isinstance(pollResultJson, (dict, list)) else pollResultJson}")
if "errors" in pollResultJson:
print(f"[Debug] Poll error: {pollResultJson}")
print(f"[Debug] Poll error: {safe_json_dumps(pollResultJson, indent=2) if isinstance(pollResultJson, (dict, list)) else pollResultJson}")
return None
if "data" in pollResultJson and len(pollResultJson["data"]) > 0:
print(f"[Debug] Poll data: {pollResultJson['data'][0]}")
print(f"[Debug] Poll data: {sanitize_for_logging(pollResultJson['data'][0])}")
return pollResultJson["data"][0]
return None
@@ -57,11 +57,9 @@ class RunwareElevenLabsProviderSettings:
}
}
# Create provider settings
# Return flat dictionary - will be wrapped by inference node with provider name (same pattern as video provider settings)
providerSettings = {
"elevenlabs": {
"music": music
}
"music": music
}
return (providerSettings,)
@@ -3,7 +3,7 @@ Runware KlingAI Provider Settings Node
Provides KlingAI-specific settings for video generation
"""
from typing import Dict, Any
from typing import Optional, Dict, Any
class RunwareKlingProviderSettings:
@@ -14,6 +14,18 @@ class RunwareKlingProviderSettings:
return {
"required": {},
"optional": {
"useCameraControl": ("BOOLEAN", {
"tooltip": "Enable to include cameraControl parameter in provider settings",
"default": False,
}),
"cameraControl": (["none", "static", "dynamic", "zoom_in", "zoom_out", "pan_left", "pan_right", "tilt_up", "tilt_down", "smooth"], {
"tooltip": "Camera control mode for KlingAI video generation",
"default": "none",
}),
"useSoundVolume": ("BOOLEAN", {
"tooltip": "Enable to include soundVolume parameter in provider settings",
"default": False,
}),
"soundVolume": ("FLOAT", {
"tooltip": "Background sound volume to mix with generated audio (provider key: soundVolume).",
"default": 1.0,
@@ -21,6 +33,10 @@ class RunwareKlingProviderSettings:
"max": 2.0,
"step": 0.1,
}),
"useOriginalAudioVolume": ("BOOLEAN", {
"tooltip": "Enable to include originalAudioVolume parameter in provider settings",
"default": False,
}),
"originalAudioVolume": ("FLOAT", {
"tooltip": "Original video volume mix level (provider key: originalAudioVolume). No effect if source has no audio.",
"default": 1.0,
@@ -28,32 +44,92 @@ class RunwareKlingProviderSettings:
"max": 2.0,
"step": 0.1,
}),
"useSoundEffectPrompt": ("BOOLEAN", {
"tooltip": "Enable to include soundEffectPrompt parameter in provider settings",
"default": False,
}),
"soundEffectPrompt": ("STRING", {
"multiline": True,
"tooltip": "Prompt describing the sound effect to generate",
"default": "",
}),
"useBgmPrompt": ("BOOLEAN", {
"tooltip": "Enable to include bgmPrompt parameter in provider settings",
"default": False,
}),
"bgmPrompt": ("STRING", {
"multiline": True,
"tooltip": "Prompt describing the background music to generate",
"default": "",
}),
"useAsmrMode": ("BOOLEAN", {
"tooltip": "Enable to include asmrMode parameter in provider settings",
"default": False,
}),
"asmrMode": ("BOOLEAN", {
"tooltip": "Enable ASMR mode for audio generation",
"default": False,
}),
}
}
RETURN_TYPES = ("RUNWAREPROVIDERSETTINGS",)
RETURN_NAMES = ("Provider Settings",)
FUNCTION = "create_provider_settings"
CATEGORY = "Runware"
DESCRIPTION = "Configure KlingAI-specific provider settings for video generation including sound and original audio volume controls."
FUNCTION = "createProviderSettings"
CATEGORY = "Runware/Provider Settings"
DESCRIPTION = "Configure KlingAI-specific provider settings for video generation including camera control, audio volume, sound effects, and ASMR mode."
def create_provider_settings(self, **kwargs) -> tuple[Dict[str, Any]]:
def createProviderSettings(self, **kwargs) -> tuple[Dict[str, Any]]:
"""Create KlingAI provider settings"""
sound_volume = kwargs.get("soundVolume", None)
original_audio_volume = kwargs.get("originalAudioVolume", None)
# Get control parameters
useCameraControl = kwargs.get("useCameraControl", False)
useSoundVolume = kwargs.get("useSoundVolume", False)
useOriginalAudioVolume = kwargs.get("useOriginalAudioVolume", False)
useSoundEffectPrompt = kwargs.get("useSoundEffectPrompt", False)
useBgmPrompt = kwargs.get("useBgmPrompt", False)
useAsmrMode = kwargs.get("useAsmrMode", False)
kling_settings: Dict[str, Any] = {}
# Get value parameters
cameraControl = kwargs.get("cameraControl", "none")
soundVolume = kwargs.get("soundVolume", 1.0)
originalAudioVolume = kwargs.get("originalAudioVolume", 1.0)
soundEffectPrompt = kwargs.get("soundEffectPrompt", "")
bgmPrompt = kwargs.get("bgmPrompt", "")
asmrMode = kwargs.get("asmrMode", False)
if sound_volume is not None:
kling_settings["soundVolume"] = float(sound_volume)
if original_audio_volume is not None:
kling_settings["originalAudioVolume"] = float(original_audio_volume)
# Build settings dictionary - only include what is enabled
klingSettings: Dict[str, Any] = {}
# Remove unset values
kling_settings = {k: v for k, v in kling_settings.items() if v is not None}
# Add cameraControl only if useCameraControl is enabled and not "none"
if useCameraControl and cameraControl != "none":
klingSettings["cameraControl"] = cameraControl
return (kling_settings,)
# Add soundVolume only if useSoundVolume is enabled
if useSoundVolume:
klingSettings["soundVolume"] = float(soundVolume)
# Add originalAudioVolume only if useOriginalAudioVolume is enabled
if useOriginalAudioVolume:
klingSettings["originalAudioVolume"] = float(originalAudioVolume)
# Add soundEffectPrompt only if useSoundEffectPrompt is enabled and not empty
if useSoundEffectPrompt and soundEffectPrompt and soundEffectPrompt.strip():
klingSettings["soundEffectPrompt"] = soundEffectPrompt.strip()
# Add bgmPrompt only if useBgmPrompt is enabled and not empty
if useBgmPrompt and bgmPrompt and bgmPrompt.strip():
klingSettings["bgmPrompt"] = bgmPrompt.strip()
# Add asmrMode only if useAsmrMode is enabled
if useAsmrMode:
klingSettings["asmrMode"] = asmrMode
# Clean up None values
klingSettings = {k: v for k, v in klingSettings.items() if v is not None}
# Return flat dictionary - will be wrapped by inference node with provider name (same pattern as video provider settings)
return (klingSettings,)
# Node class mappings
@@ -0,0 +1,146 @@
from typing import Any, Dict, List
class RunwareLumaProviderSettings:
"""Provider settings node for LumaAI video generation parameters."""
CONCEPT_OPTIONS = [
"",
"aerial",
"aerial_drone",
"bolt_cam",
"crane_down",
"crane_up",
"dolly_zoom",
"elevator_doors",
"eye_level",
"ground_level",
"handheld",
"high_angle",
"low_angle",
"orbit_left",
"orbit_right",
"over_the_shoulder",
"overhead",
"pan_left",
"pan_right",
"pedestal_down",
"pedestal_up",
"pov",
"pull_out",
"push_in",
"roll_left",
"roll_right",
"selfie",
"static",
"tilt_down",
"tilt_up",
"tiny_planet",
"truck_left",
"truck_right",
"zoom_in",
"zoom_out",
]
@classmethod
def INPUT_TYPES(cls):
return {
"required": {},
"optional": {
"useLoop": ("BOOLEAN", {
"tooltip": "Enable to include the loop parameter in provider settings.",
"default": False,
}),
"loop": ("BOOLEAN", {
"tooltip": "When enabled, generated video will loop seamlessly.",
"default": False,
}),
"useConcept1": ("BOOLEAN", {
"tooltip": "Enable to include Concept 1 in the concepts list.",
"default": False,
}),
"concept1": (cls.CONCEPT_OPTIONS, {
"tooltip": "Concept key to guide the video generation (e.g., aerial, bolt_cam).",
"default": "",
}),
"useConcept2": ("BOOLEAN", {
"tooltip": "Enable to include Concept 2 in the concepts list.",
"default": False,
}),
"concept2": (cls.CONCEPT_OPTIONS, {
"tooltip": "Concept key to guide the video generation.",
"default": "",
}),
"useConcept3": ("BOOLEAN", {
"tooltip": "Enable to include Concept 3 in the concepts list.",
"default": False,
}),
"concept3": (cls.CONCEPT_OPTIONS, {
"tooltip": "Concept key to guide the video generation.",
"default": "",
}),
"useConcept4": ("BOOLEAN", {
"tooltip": "Enable to include Concept 4 in the concepts list.",
"default": False,
}),
"concept4": (cls.CONCEPT_OPTIONS, {
"tooltip": "Concept key to guide the video generation.",
"default": "",
}),
}
}
DESCRIPTION = "Configure LumaAI provider settings including loop and concept guidance."
FUNCTION = "createProviderSettings"
RETURN_TYPES = ("RUNWAREPROVIDERSETTINGS",)
RETURN_NAMES = ("Provider Settings",)
CATEGORY = "Runware/Provider Settings"
def createProviderSettings(self, **kwargs) -> tuple[Dict[str, Any]]:
"""Build LumaAI provider settings payload."""
useLoop = kwargs.get("useLoop", False)
loop = kwargs.get("loop", False)
concepts = self._collectConcepts(**kwargs)
lumaSettings: Dict[str, Any] = {}
if useLoop:
lumaSettings["loop"] = bool(loop)
if concepts:
lumaSettings["concepts"] = concepts
result = {}
if lumaSettings:
result["lumaai"] = lumaSettings
return (result if result else None,)
def _collectConcepts(self, **kwargs) -> List[Dict[str, str]]:
"""Collect concept entries from kwargs respecting use toggles."""
concepts: List[Dict[str, str]] = []
for index in range(1, 5):
use_key = f"useConcept{index}"
concept_key = f"concept{index}"
if kwargs.get(use_key, False):
concept_value = kwargs.get(concept_key, "")
if isinstance(concept_value, str):
normalized = concept_value.strip().lower()
if normalized != "":
concepts.append({"key": normalized})
return concepts
NODE_CLASS_MAPPINGS = {
"RunwareLumaProviderSettings": RunwareLumaProviderSettings,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"RunwareLumaProviderSettings": "Runware Luma Provider Settings",
}
@@ -1,36 +0,0 @@
from ..utils import runwareUtils as rwUtils
class RunwareProviderSettings:
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"customSettings": ("DICT", {
"tooltip": "Custom provider settings as a dictionary. Add any provider-specific parameters here.",
"default": {},
}),
}
}
DESCRIPTION = "Configure custom provider-specific settings for video and image generation. Add any provider parameters as a dictionary."
FUNCTION = "create_provider_settings"
RETURN_TYPES = ("RUNWAREPROVIDERSETTINGS",)
RETURN_NAMES = ("Provider Settings",)
CATEGORY = "Runware"
def create_provider_settings(self, **kwargs):
custom_settings = kwargs.get("customSettings", {})
print(f"[Provider Settings] Custom settings: {custom_settings}")
return (custom_settings,)
# Node class mappings for ComfyUI registration
NODE_CLASS_MAPPINGS = {
"RunwareProviderSettings": RunwareProviderSettings,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"RunwareProviderSettings": "Runware Provider Settings",
}
@@ -3,7 +3,7 @@ Runware Runway Provider Settings Node
Provides Runway-specific settings for video generation
"""
from typing import Optional, Dict, Any
from typing import Dict, Any
class RunwareRunwayProviderSettings:
@@ -14,9 +14,9 @@ class RunwareRunwayProviderSettings:
return {
"required": {},
"optional": {
"publicFigureThreshold": ("STRING", {
"tooltip": "Public figure threshold for content moderation (provider key: contentModeration.publicFigureThreshold).",
"default": None,
"publicFigureThreshold": (["auto", "low"], {
"tooltip": "Select the public figure detection threshold (provider key: contentModeration.publicFigureThreshold).",
"default": "auto",
}),
}
}
@@ -30,12 +30,11 @@ class RunwareRunwayProviderSettings:
def createProviderSettings(self, **kwargs) -> tuple[Dict[str, Any]]:
"""Create Runway provider settings"""
publicFigureThreshold = kwargs.get("publicFigureThreshold", None)
publicFigureThreshold = kwargs.get("publicFigureThreshold")
runwaySettings: Dict[str, Any] = {}
# Build contentModeration if publicFigureThreshold is provided
if publicFigureThreshold is not None and publicFigureThreshold.strip():
if isinstance(publicFigureThreshold, str) and publicFigureThreshold.strip() != "":
contentModeration: Dict[str, Any] = {}
contentModeration["publicFigureThreshold"] = publicFigureThreshold.strip()
runwaySettings["contentModeration"] = contentModeration
+2 -3
View File
@@ -1,5 +1,4 @@
from .utils import runwareUtils as rwUtils
import json
class upscaler:
@classmethod
@@ -238,14 +237,14 @@ class upscaler:
# Debug: Print the request being sent
print(f"[DEBUG] Sending Image Upscale Request:", flush=True)
print(f"[DEBUG] Request Payload: {json.dumps([genConfig], indent=2)}", flush=True)
print(f"[DEBUG] Request Payload: {rwUtils.safe_json_dumps([genConfig], indent=2)}", flush=True)
try:
genResult = rwUtils.inferenecRequest([genConfig])
# Debug: Print the response received
print(f"[DEBUG] Received Image Upscale Response:", flush=True)
print(f"[DEBUG] Response: {json.dumps(genResult, indent=2)}", flush=True)
print(f"[DEBUG] Response: {rwUtils.safe_json_dumps(genResult, indent=2)}", flush=True)
except Exception as e:
print(f"[DEBUG] Error in Image Upscale Request: {str(e)}", flush=True)
raise
+49 -6
View File
@@ -18,6 +18,7 @@ import json
import os
import io
import threading
import re
load_dotenv()
@@ -44,6 +45,50 @@ adapter = HTTPAdapter(pool_connections=10, pool_maxsize=10)
session.mount("http://", adapter)
session.mount("https://", adapter)
_BASE64_PATTERN = re.compile(r'^(?:data:[^;]+;base64,)?[A-Za-z0-9+/=\s]+$')
def _truncate_base64_string(value: str, preview_length: int = 20) -> str:
"""Return a short preview for base64/data URI strings."""
if value is None:
return value
prefix = ""
payload = value
if value.startswith("data:") and "base64," in value:
prefix, payload = value.split("base64,", 1)
prefix += "base64,"
payload = payload.replace("\n", "").replace("\r", "")
if len(payload) > preview_length:
return f"{prefix}{payload[:preview_length]}...(truncated)"
return value
def sanitize_for_logging(data, preview_length: int = 20, max_string_length: int = 2048):
"""
Sanitize data structures for debug logging by truncating base64 strings.
"""
if isinstance(data, dict):
return {key: sanitize_for_logging(value, preview_length, max_string_length) for key, value in data.items()}
if isinstance(data, list):
return [sanitize_for_logging(item, preview_length, max_string_length) for item in data]
if isinstance(data, tuple):
return tuple(sanitize_for_logging(item, preview_length, max_string_length) for item in data)
if isinstance(data, str):
stripped = data.strip()
if (len(stripped) > preview_length and _BASE64_PATTERN.match(stripped)
and not stripped.lower().startswith("http")):
return _truncate_base64_string(stripped, preview_length)
if len(data) > max_string_length:
return f"{data[:max_string_length]}...(truncated)"
return data
return data
def safe_json_dumps(data, **kwargs) -> str:
"""Dump JSON after sanitizing potential base64 payloads."""
sanitized = sanitize_for_logging(data)
return json.dumps(sanitized, **kwargs)
def generalRequestWrapper(recaller, *args, **kwargs):
for attempt in range(MAX_RETRIES + 1):
try:
@@ -303,7 +348,7 @@ def inferenecRequest(genConfig):
print(f"[Debugging] Runware Raw Response Content: {genResult.content}")
raise Exception("Error: Invalid JSON response from API!")
if "errors" in genResult:
print(f"[DEBUG] API Error Response: {genResult}")
print(f"[DEBUG] API Error Response: {safe_json_dumps(genResult, indent=2) if isinstance(genResult, dict) else genResult}")
raise Exception(genResult["errors"][0]["message"])
else:
return genResult
@@ -400,14 +445,12 @@ def convertTensor2IMG(tensorImage):
image = Image.fromarray(imageNP)
with io.BytesIO() as buffer:
image.save(
buffer, format="webp", quality=100, subsampling=0, method=6, exact=True
)
image.save(buffer, format="PNG")
imageRawData = buffer.getvalue()
imgBytes = len(imageRawData)
imgSize = int(imgBytes / 1024)
imgb64 = base64.b64encode(imageRawData).decode("utf-8")
imgDataUri = f"data:image/webp;base64,{imgb64}"
imgDataUri = f"data:image/png;base64,{imgb64}"
if ENABLE_IMAGES_CACHING and imgSize >= MIN_IMAGE_CACHE_SIZE:
try:
@@ -606,7 +649,7 @@ def pollVideoResult(taskUUID):
return None
if "errors" in pollResult:
print(f"[Debugging] Poll error: {pollResult}")
print(f"[Debugging] Poll error: {safe_json_dumps(pollResult, indent=2) if isinstance(pollResult, (dict, list)) else pollResult}")
return pollResult
else:
return pollResult
+2 -3
View File
@@ -1,5 +1,4 @@
from .utils import runwareUtils as rwUtils
import json
import base64
import io
@@ -51,12 +50,12 @@ class vectorize:
genConfig = self._buildGenConfig(model, imageUuid)
print(f"[DEBUG] Sending Vectorize Request:")
print(f"[DEBUG] Request Payload: {json.dumps(genConfig, indent=2)}")
print(f"[DEBUG] Request Payload: {rwUtils.safe_json_dumps(genConfig, indent=2)}")
genResult = rwUtils.inferenecRequest(genConfig)
print(f"[DEBUG] Received Vectorize Response:")
print(f"[DEBUG] Response: {json.dumps(genResult, indent=2)}")
print(f"[DEBUG] Response: {rwUtils.safe_json_dumps(genResult, indent=2)}")
self._validateResponse(genResult)
+2 -3
View File
@@ -1,5 +1,4 @@
from .utils import runwareUtils as rwUtils
import json
import comfy.model_management
@@ -69,12 +68,12 @@ class videoBgRemoval:
genConfig = self._buildGenConfig(videoUuid, modelAir, outputFormat, rgbaColor)
print(f"[DEBUG] Sending Video Background Removal Request:")
print(f"[DEBUG] Request Payload: {json.dumps(genConfig, indent=2)}")
print(f"[DEBUG] Request Payload: {rwUtils.safe_json_dumps(genConfig, indent=2)}")
genResult = rwUtils.inferenecRequest(genConfig)
print(f"[DEBUG] Received Video Background Removal Response:")
print(f"[DEBUG] Response: {json.dumps(genResult, indent=2)}")
print(f"[DEBUG] Response: {rwUtils.safe_json_dumps(genResult, indent=2)}")
self._validateResponse(genResult)
+37 -31
View File
@@ -1,6 +1,5 @@
from .utils import runwareUtils as rwUtils
from .videoModelSearch import videoModelSearch
import json
import comfy.model_management
@@ -180,8 +179,8 @@ class txt2vid:
videoAdvancedFeatureInputs = kwargs.get("videoAdvancedFeatureInputs", None)
runwareAccelerator = kwargs.get("Accelerator", None)
useCustomDimensions = kwargs.get("useCustomDimensions", False)
customWidth = kwargs.get("width", 864)
customHeight = kwargs.get("height", 480)
customWidth = kwargs.get("width", None)
customHeight = kwargs.get("height", None)
useDuration = kwargs.get("useDuration", True)
duration = kwargs.get("duration", 5)
fps = kwargs.get("fps", 24)
@@ -197,18 +196,23 @@ class txt2vid:
# Handle model input - could be dict or string
if isinstance(runwareVideoModel, dict):
model = runwareVideoModel.get("model", "")
model_use_resolution = runwareVideoModel.get("useResolution", True)
else:
model = runwareVideoModel
# Determine dimensions based on custom setting
model_use_resolution = True
# Determine dimensions based on custom setting and model defaults
if useCustomDimensions:
width = customWidth
height = customHeight
else:
elif model_use_resolution:
modelDimensions = videoModelSearch.MODEL_DIMENSIONS.get(model, {"width": 1024, "height": 576})
width = modelDimensions["width"]
height = modelDimensions["height"]
width = modelDimensions.get("width", None)
height = modelDimensions.get("height", None)
else:
width = None
height = None
genConfig = [
{
"taskType": "videoInference",
@@ -219,7 +223,10 @@ class txt2vid:
"includeCost": True,
}
]
if height != 0 and width != 0:
if (
width is not None and height is not None
and width > 0 and height > 0
):
genConfig[0]['height'] = height
genConfig[0]['width'] = width
@@ -253,25 +260,25 @@ class txt2vid:
# Add frameImages if provided from Runware Frame Images node
if frameImages is not None and len(frameImages) > 0:
genConfig[0]["frameImages"] = frameImages
print(f"[Debugging] Frame images array: {frameImages}")
print(f"[Debugging] Frame images array: {rwUtils.sanitize_for_logging(frameImages)}")
# Add referenceImages if provided from Runware Reference Images node
if referenceImages is not None and len(referenceImages) > 0:
genConfig[0]["referenceImages"] = referenceImages
print(f"[Debugging] Reference images array: {referenceImages}")
print(f"[Debugging] Reference images array: {rwUtils.sanitize_for_logging(referenceImages)}")
# Add inputAudios if provided
if inputAudios is not None and len(inputAudios) > 0:
genConfig[0]["inputAudios"] = inputAudios
print(f"[Debugging] Input audios array: {inputAudios}")
print(f"[Debugging] Input audios array: {rwUtils.sanitize_for_logging(inputAudios)}")
# Add referenceVideos if provided and not empty
if referenceVideos is not None:
# Handle both single mediaUUID (string) and multiple mediaUUIDs (list)
if isinstance(referenceVideos, str) and referenceVideos.strip() != "":
genConfig[0]["referenceVideos"] = [referenceVideos.strip()]
print(f"[Debugging] Reference videos: {genConfig[0].get('referenceVideos', [])}")
print(f"[Debugging] Reference videos: {rwUtils.sanitize_for_logging(genConfig[0].get('referenceVideos', []))}")
elif isinstance(referenceVideos, list) and len(referenceVideos) > 0:
genConfig[0]["referenceVideos"] = referenceVideos
print(f"[Debugging] Reference videos: {genConfig[0].get('referenceVideos', [])}")
print(f"[Debugging] Reference videos: {rwUtils.sanitize_for_logging(genConfig[0].get('referenceVideos', []))}")
# Add speech parameters if both voice and text are provided
if speechVoice and speechVoice.strip() != "" and speechText and speechText.strip() != "":
@@ -291,24 +298,23 @@ class txt2vid:
for key, value in inputs.items():
genConfig[0]["inputs"][key] = value
print(f"[Debugging] Video inference inputs merged: {inputs}")
print(f"[Debugging] Final genConfig inputs: {genConfig[0].get('inputs', {})}")
print(f"[Debugging] Video inference inputs merged: {rwUtils.sanitize_for_logging(inputs)}")
print(f"[Debugging] Final genConfig inputs: {rwUtils.sanitize_for_logging(genConfig[0].get('inputs', {}))}")
# Handle providerSettings - extract provider name from model and merge with custom settings
if providerSettings is not None:
# Extract provider name from model (e.g., "pixverse:1@1" -> "pixverse")
provider_name = model.split(":")[0] if ":" in model else model
# If providerSettings is a dictionary, create the correct API format
if isinstance(providerSettings, dict):
# Create the providerSettings object with provider name as key
final_provider_settings = {
provider_name: providerSettings
}
genConfig[0]["providerSettings"] = final_provider_settings
print(f"[Debugging] Provider settings: {final_provider_settings}")
if provider_name in providerSettings:
final_provider_settings = providerSettings
else:
final_provider_settings = {provider_name: providerSettings}
if final_provider_settings:
genConfig[0]["providerSettings"] = final_provider_settings
print(f"[Debugging] Provider settings: {rwUtils.sanitize_for_logging(final_provider_settings)}")
else:
# If it's just a string, use it directly
genConfig[0]["providerSettings"] = providerSettings
# Add safety inputs if provided
@@ -339,15 +345,15 @@ class txt2vid:
try:
# Debug: Print the request being sent
print(f"[DEBUG] Sending Video Inference Request:")
print(f"[DEBUG] Request Payload: {json.dumps(genConfig, indent=2)}")
print(f"[DEBUG] Request Payload: {rwUtils.safe_json_dumps(genConfig, indent=2)}")
genResult = rwUtils.inferenecRequest(genConfig)
# Debug: Print the response received
print(f"[DEBUG] Received Video Inference Response:")
print(f"[DEBUG] Response: {json.dumps(genResult, indent=2)}")
print(f"[DEBUG] Response: {rwUtils.safe_json_dumps(genResult, indent=2)}")
print(f"[Debugging] Generation config: {genConfig}")
print(f"[Debugging] Generation config: {rwUtils.safe_json_dumps(genConfig, indent=2)}")
except Exception as e:
# Check if it's a dimension error and provide helpful information
error_msg = str(e)
@@ -376,7 +382,7 @@ class txt2vid:
# Poll for video result
pollResult = rwUtils.pollVideoResult(taskUUID)
print(f"[Debugging] Poll result: {pollResult}")
print(f"[Debugging] Poll result: {rwUtils.safe_json_dumps(pollResult, indent=2) if isinstance(pollResult, (dict, list)) else pollResult}")
# Check for errors first
if pollResult and "errors" in pollResult and len(pollResult["errors"]) > 0:
+9 -2
View File
@@ -13,6 +13,9 @@ class videoInferenceInputs:
"Image": ("IMAGE", {
"tooltip": "Portrait image for video generation. This will be the main subject that will speak and move in the generated video."
}),
"Frame Images": ("RUNWAREVIDEOINPUTSFRAMEIMAGES", {
"tooltip": "Connect the Runware Video Inputs Frame node to provide timeline-constrained frame images."
}),
"Audio": ("STRING", {
"tooltip": "Connect the mediaUUID output from Runware Media Upload node with audio file. This audio will be synchronized with the generated video."
}),
@@ -25,7 +28,7 @@ class videoInferenceInputs:
"Frame": ("IMAGE", {
"tooltip": "Frame image for video generation. Connect a Load Image node to provide the frame image."
}),
"References": ("RUNWAREVIDEOINPUTSREFERENCES", {
"References": ("RUNWAREVIDEOINPUTSREFERENCEIMAGES", {
"tooltip": "Connect the Video Inputs References node to provide reference images."
}),
}
@@ -44,6 +47,7 @@ class videoInferenceInputs:
video = kwargs.get("Video", None)
mask = kwargs.get("Mask", None)
frame = kwargs.get("Frame", None)
frameImages = kwargs.get("Frame Images", None)
references = kwargs.get("References", None)
inputs = {}
@@ -63,7 +67,10 @@ class videoInferenceInputs:
if frame is not None:
inputs["frame"] = rwUtils.convertTensor2IMG(frame)
if frameImages is not None and len(frameImages) > 0:
inputs["frameImages"] = frameImages
if references is not None and len(references) > 0:
inputs["references"] = references
inputs["referenceImages"] = references
return (inputs,)
+70
View File
@@ -0,0 +1,70 @@
from .utils import runwareUtils as rwUtils
class RunwareVideoInputsFrameImages:
"""Video Inputs Frame node for adapting frame images structure for video inference"""
MAX_FRAMES = 4
@classmethod
def INPUT_TYPES(cls):
optionalInputs = {}
for i in range(1, cls.MAX_FRAMES + 1):
optionalInputs[f"image{i}"] = ("IMAGE", {
"tooltip": "Frame image to include in video inference inputs."
})
optionalInputs[f"frame{i} position"] = ("STRING", {
"default": "",
"tooltip": "Optional frame label (e.g., 'first', 'last', specific frame number). Leave blank for provider defaults."
})
return {
"required": {},
"optional": optionalInputs
}
DESCRIPTION = "Convert Runware Frame Images into the expected structure for video inference inputs."
FUNCTION = "createFrameInputs"
RETURN_TYPES = ("RUNWAREVIDEOINPUTSFRAMEIMAGES",)
RETURN_NAMES = ("Video Inputs Frame Images",)
CATEGORY = "Runware"
def createFrameInputs(self, **kwargs):
frameImages = []
for i in range(1, self.MAX_FRAMES + 1):
imageKey = f"image{i}"
frameKey = f"frame{i} position"
image = kwargs.get(imageKey)
frame = kwargs.get(frameKey, "")
if image is None:
continue
frameData = self._createFrameEntry(image, frame)
frameImages.append(frameData)
return (frameImages,)
def _createFrameEntry(self, image, frameLabel):
imageData = rwUtils.convertTensor2IMG(image)
entry = {"image": imageData}
if isinstance(frameLabel, str) and frameLabel.strip() != "":
if frameLabel.strip().isdigit():
entry["frame"] = int(frameLabel.strip())
else:
entry["frame"] = frameLabel.strip()
return entry
NODE_CLASS_MAPPINGS = {
"RunwareVideoInputsFrameImages": RunwareVideoInputsFrameImages,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"RunwareVideoInputsFrameImages": "Runware Video Inputs Frame Images",
}
+7 -10
View File
@@ -28,8 +28,8 @@ class videoInputsReferences:
DESCRIPTION = "Configure multiple reference images with optional types for video inference inputs."
FUNCTION = "createReferences"
RETURN_TYPES = ("RUNWAREVIDEOINPUTSREFERENCES",)
RETURN_NAMES = ("Video Inputs References",)
RETURN_TYPES = ("RUNWAREVIDEOINPUTSREFERENCEIMAGES",)
RETURN_NAMES = ("Video Inputs Reference Images",)
CATEGORY = "Runware"
def createReferences(self, **kwargs) -> tuple[List[Dict[str, Any]]]:
@@ -50,14 +50,11 @@ class videoInputsReferences:
return (references,)
def _createReference(self, image, refType):
"""Create reference object from image and type"""
"""Create reference entry (API expects direct strings)"""
imageUrl = rwUtils.convertTensor2IMG(image)
reference = {"image": imageUrl}
if refType and refType.strip() != "":
reference["type"] = refType.strip()
return reference
# API expects referenceImages to be a list of strings (UUID/URL/data URI)
# so we return the converted string directly.
return imageUrl
# Node class mappings
@@ -66,5 +63,5 @@ NODE_CLASS_MAPPINGS = {
}
NODE_DISPLAY_NAME_MAPPINGS = {
"RunwareVideoInputsReferences": "Runware Video Inputs References",
"RunwareVideoInputsReferences": "Runware Video Inputs Reference Images",
}
+40 -7
View File
@@ -65,6 +65,15 @@ class videoModelSearch:
"Ovi": [
"runware:190@1 (Ovi)",
],
"Runway": [
"runway:2@1 (Runway Aleph)",
"runway:1@1 (Runway Gen-4 Turbo)",
],
"Luma": [
"lumaai:1@1 (Luma Ray 1.6)",
"lumaai:2@1 (Luma Ray 2)",
"lumaai:2@2 (Luma Ray 2 Flash)",
],
}
# Model dimensions mapping
@@ -130,6 +139,15 @@ class videoModelSearch:
# Ovi Models
"runware:190@1": {"width": 0, "height": 0},
# Runway Models
"runway:2@1": {"width": 1280, "height": 720},
"runway:1@1": {"width": 1280, "height": 720},
# Luma Models
"lumaai:1@1": {"width": 1080, "height": 720},
"lumaai:2@1": {"width": 1080, "height": 720},
"lumaai:2@2": {"width": 1080, "height": 720},
}
MODEL_ARCHITECTURES = [
@@ -143,7 +161,9 @@ class videoModelSearch:
"Wan",
"OpenAI",
"Lightricks",
"Ovi"
"Ovi",
"Runway",
"Luma"
]
DEFAULT_DIMENSIONS = {"width": 1024, "height": 576}
@@ -174,6 +194,12 @@ class videoModelSearch:
"label_on": "Enabled",
"label_off": "Disabled",
}),
"useResolution": ("BOOLEAN", {
"tooltip": "Enable to automatically use the model's recommended resolution. Disable to output 0x0 and set custom resolution downstream.",
"default": True,
"label_on": "Enabled",
"label_off": "Disabled",
}),
"Width": ("INT", {
"default": defaultDims["width"],
"min": 0,
@@ -215,6 +241,7 @@ class videoModelSearch:
searchInput = kwargs.get("Model Search")
customWidth = kwargs.get("Width", 0)
customHeight = kwargs.get("Height", 0)
useResolution = kwargs.get("useResolution", True)
if enableSearchValue:
modelAirCode = searchInput
@@ -224,13 +251,19 @@ class videoModelSearch:
dimensions = self.MODEL_DIMENSIONS.get(modelAirCode, self.DEFAULT_DIMENSIONS)
width = customWidth if customWidth > 0 else dimensions["width"]
height = customHeight if customHeight > 0 else dimensions["height"]
if width == 0 and height == 0:
width = None
height = None
if useResolution:
width = customWidth if customWidth > 0 else dimensions["width"]
height = customHeight if customHeight > 0 else dimensions["height"]
if width == 0 and height == 0:
width = None
height = None
else:
width = 0
height = 0
return ({
"model": modelAirCode,
"useResolution": useResolution,
"useResolution": useResolution,
}, width, height)
+13 -8
View File
@@ -1,9 +1,13 @@
from .utils import runwareUtils as rwUtils
import json
import comfy.model_management
class videoTranscription:
RUNWARE_VIDEO_TRANSCRIPTION_MODELS = {
"Memories Video Captioning": "memories:1@1",
"Memories Age Detection": "memories:2@1",
}
@classmethod
def INPUT_TYPES(cls):
return {
@@ -25,9 +29,9 @@ class videoTranscription:
"placeholder": "Enter video URL, file path, or Runware video UUID",
"tooltip": "The video to transcribe. Can be a URL, file path, or Runware video UUID."
}),
"model": ("STRING", {
"default": "memories:1@1",
"tooltip": "The model to use for video transcription. Default: memories:1@1"
"model": (list(cls.RUNWARE_VIDEO_TRANSCRIPTION_MODELS.keys()), {
"default": "Memories Video Captioning",
"tooltip": "Choose the model to use for video transcription."
}),
},
"hidden": { "node_id": "UNIQUE_ID" }
@@ -37,7 +41,7 @@ class videoTranscription:
FUNCTION = "transcribeVideo"
RETURN_TYPES = ("STRING",)
RETURN_NAMES = ("Video Transcription",)
RETURN_NAMES = ("Video Caption",)
CATEGORY = "Runware"
OUTPUT_NODE = True
@@ -50,7 +54,8 @@ class videoTranscription:
def transcribeVideo(self, **kwargs):
video = kwargs.get("video", "")
model = kwargs.get("model", "memories:1@1")
modelKey = kwargs.get("model", "Memories Video Captioning")
model = self.RUNWARE_VIDEO_TRANSCRIPTION_MODELS.get(modelKey, modelKey)
if not video or not video.strip():
raise Exception("Video URL, file path, or UUID is required")
@@ -70,14 +75,14 @@ class videoTranscription:
# Debug: Print the request being sent
print(f"[DEBUG] Sending Video Transcription Request:")
print(f"[DEBUG] Request Payload: {json.dumps(genConfig, indent=2)}")
print(f"[DEBUG] Request Payload: {rwUtils.safe_json_dumps(genConfig, indent=2)}")
try:
genResult = rwUtils.inferenecRequest(genConfig)
# Debug: Print the response received
print(f"[DEBUG] Received Video Transcription Response:")
print(f"[DEBUG] Response: {json.dumps(genResult, indent=2)}")
print(f"[DEBUG] Response: {rwUtils.safe_json_dumps(genResult, indent=2)}")
except Exception as e:
raise e
+3 -3
View File
@@ -1,10 +1,10 @@
from .utils import runwareUtils as rwUtils
import json
import comfy.model_management
class videoUpscaler:
RUNWARE_VUPSCALER_MODELS = {
"Bria Video Upscaler": "bria:50@1",
"Bytedance Video Upscaler": "bytedance:50@1",
}
@classmethod
@@ -69,13 +69,13 @@ class videoUpscaler:
try:
# Debug: Print the request being sent
print(f"[DEBUG] Sending Video Upscale Request:")
print(f"[DEBUG] Request Payload: {json.dumps(genConfig, indent=2)}")
print(f"[DEBUG] Request Payload: {rwUtils.safe_json_dumps(genConfig, indent=2)}")
genResult = rwUtils.inferenecRequest(genConfig)
# Debug: Print the response received
print(f"[DEBUG] Received Video Upscale Response:")
print(f"[DEBUG] Response: {json.dumps(genResult, indent=2)}")
print(f"[DEBUG] Response: {rwUtils.safe_json_dumps(genResult, indent=2)}")
# Check for errors
if "errors" in genResult:
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+1 -1
View File
@@ -1 +1 @@
{"id":"df246f14-f1cb-4379-91af-5185b0c663d8","revision":0,"last_node_id":4,"last_link_id":2,"nodes":[{"id":3,"type":"LoadVideo","pos":[-436.6539001464844,430.0327453613281],"size":[274.080078125,442.0882263183594],"flags":{},"order":0,"mode":0,"inputs":[{"localized_name":"file","name":"file","type":"COMBO","widget":{"name":"file"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"VIDEO","name":"VIDEO","type":"VIDEO","links":[1]}],"properties":{"cnr_id":"comfy-core","ver":"0.3.46","Node name for S&R":"LoadVideo"},"widgets_values":["backgroundRemoval.mp4","image"]},{"id":4,"type":"Runware Media Upload","pos":[-151.37985229492188,276.1902160644531],"size":[270,58],"flags":{},"order":1,"mode":0,"inputs":[{"localized_name":"media","name":"media","shape":7,"type":"VIDEO,AUDIO","link":1},{"localized_name":"mediaUUID","name":"mediaUUID","shape":7,"type":"STRING","widget":{"name":"mediaUUID"},"link":null}],"outputs":[{"localized_name":"mediaUUID","name":"mediaUUID","type":"STRING","links":[2]}],"properties":{"cnr_id":"ComfyUI-Runware","ver":"056c15a1b72b1555184d0447562c095797825d6e","Node name for S&R":"Runware Media Upload"},"widgets_values":["1ba124eb-81aa-44f6-ab7c-cfa043065738"],"bgcolor":"#5345bf"},{"id":1,"type":"Runware Video Transcription","pos":[-75.60359191894531,496.2238464355469],"size":[400,200],"flags":{},"order":2,"mode":0,"inputs":[{"localized_name":"Always Retranscribe","name":"Always Retranscribe","type":"BOOLEAN","widget":{"name":"Always Retranscribe"},"link":null},{"localized_name":"prompt","name":"prompt","type":"STRING","widget":{"name":"prompt"},"link":null},{"localized_name":"video","name":"video","type":"STRING","widget":{"name":"video"},"link":2},{"localized_name":"model","name":"model","type":"STRING","widget":{"name":"model"},"link":null}],"outputs":[{"localized_name":"Video Transcription","name":"Video Transcription","type":"STRING","links":null}],"properties":{"cnr_id":"ComfyUI-Runware","ver":"056c15a1b72b1555184d0447562c095797825d6e","Node name for S&R":"Runware Video Transcription"},"widgets_values":[false,"[0] (0s-7s): A young girl with braided hair and large, expressive eyes smiles at the camera. She wears a white tank top and a gold chain. The background appears to be a kitchen.","","memories:1@1"]}],"links":[[1,3,0,4,0,"VIDEO,AUDIO"],[2,4,0,1,2,"STRING"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.8264462809917354,"offset":[793.6275234675816,-100.02884140963462]}},"version":0.4}
{"id":"df246f14-f1cb-4379-91af-5185b0c663d8","revision":0,"last_node_id":5,"last_link_id":3,"nodes":[{"id":4,"type":"Runware Media Upload","pos":[-151.37985229492188,276.1902160644531],"size":[270,58],"flags":{},"order":1,"mode":0,"inputs":[{"localized_name":"media","name":"media","shape":7,"type":"VIDEO,AUDIO","link":1},{"localized_name":"mediaUUID","name":"mediaUUID","shape":7,"type":"STRING","widget":{"name":"mediaUUID"},"link":null}],"outputs":[{"localized_name":"mediaUUID","name":"mediaUUID","type":"STRING","links":[3]}],"properties":{"cnr_id":"ComfyUI-Runware","ver":"056c15a1b72b1555184d0447562c095797825d6e","Node name for S&R":"Runware Media Upload"},"widgets_values":["1ba124eb-81aa-44f6-ab7c-cfa043065738"],"bgcolor":"#5345bf"},{"id":3,"type":"LoadVideo","pos":[-438.6059875488281,321.4447326660156],"size":[274.080078125,442.0882263183594],"flags":{},"order":0,"mode":0,"inputs":[{"localized_name":"file","name":"file","type":"COMBO","widget":{"name":"file"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"VIDEO","name":"VIDEO","type":"VIDEO","links":[1]}],"properties":{"cnr_id":"comfy-core","ver":"0.3.46","Node name for S&R":"LoadVideo"},"widgets_values":["backgroundRemoval.mp4","image"]},{"id":5,"type":"Runware Video Transcription","pos":[-123.58058166503906,383.5848083496094],"size":[400,200],"flags":{},"order":2,"mode":0,"inputs":[{"localized_name":"Always Retranscribe","name":"Always Retranscribe","type":"BOOLEAN","widget":{"name":"Always Retranscribe"},"link":null},{"localized_name":"prompt","name":"prompt","type":"STRING","widget":{"name":"prompt"},"link":null},{"localized_name":"video","name":"video","type":"STRING","widget":{"name":"video"},"link":3},{"localized_name":"model","name":"model","type":"COMBO","widget":{"name":"model"},"link":null}],"outputs":[{"localized_name":"Video Caption","name":"Video Caption","type":"STRING","links":null}],"properties":{"cnr_id":"ComfyUI-Runware","ver":"056ade066187313b5c2f9a3afa5cbbee19227d3e","Node name for S&R":"Runware Video Transcription"},"widgets_values":[false,"","","Memories Age Detection"],"bgcolor":"#5345bf"}],"links":[[1,3,0,4,0,"VIDEO,AUDIO"],[3,4,0,5,2,"STRING"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.8264462809917354,"offset":[793.6275234675816,-100.02884140963462]}},"version":0.4}