Add video background removal and video upscaler nodes
- Add Runware Video Background Removal node with Bria support - Add Runware Video Upscaler node with Bria support - Add Runware Vectorize node for image vectorization - Add Lightricks Provider Settings node - Add Video Inference Inputs node for structured inputs - Add Input Audios and Reference Videos nodes - Update video inference with speech support and reference videos - Add new video models: OmniHuman 1/1.5, PixVerse LipSync, LTX - Update Bytedance provider settings with boolean controls - Add corresponding workflow examples
This commit is contained in:
@@ -3,7 +3,6 @@ Runware Bytedance Image Provider Settings Node
|
||||
Provides Bytedance-specific settings for image generation
|
||||
"""
|
||||
|
||||
import torch
|
||||
from typing import Optional, Dict, Any
|
||||
|
||||
class RunwareBytedanceProviderSettings:
|
||||
@@ -14,16 +13,32 @@ class RunwareBytedanceProviderSettings:
|
||||
return {
|
||||
"required": {},
|
||||
"optional": {
|
||||
"useCameraFixed": ("BOOLEAN", {
|
||||
"tooltip": "Enable to include cameraFixed parameter in provider settings",
|
||||
"default": False,
|
||||
}),
|
||||
"cameraFixed": ("BOOLEAN", {
|
||||
"tooltip": "Enable to fix camera position during video generation",
|
||||
"default": False,
|
||||
}),
|
||||
"useMaxSequentialImages": ("BOOLEAN", {
|
||||
"tooltip": "Enable to include maxSequentialImages parameter in provider settings",
|
||||
"default": False,
|
||||
}),
|
||||
"maxSequentialImages": ("INT", {
|
||||
"tooltip": "Maximum number of sequential images to generate (1-15)",
|
||||
"default": 1,
|
||||
"min": 1,
|
||||
"max": 15,
|
||||
}),
|
||||
"useFastMode": ("BOOLEAN", {
|
||||
"tooltip": "Enable to include fastMode parameter in provider settings",
|
||||
"default": False,
|
||||
}),
|
||||
"fastMode": ("BOOLEAN", {
|
||||
"tooltip": "When enabled, speeds up generation by sacrificing some effects. Default: false. RTF: 25-28 (fast) vs 35 (normal). Supported by OmniHuman 1.5.",
|
||||
"default": False,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,21 +50,31 @@ class RunwareBytedanceProviderSettings:
|
||||
def create_provider_settings(self, **kwargs) -> tuple[Dict[str, Any]]:
|
||||
"""Create Bytedance provider settings"""
|
||||
|
||||
# Get parameters
|
||||
# Get control parameters
|
||||
useCameraFixed = kwargs.get("useCameraFixed", False)
|
||||
useMaxSequentialImages = kwargs.get("useMaxSequentialImages", False)
|
||||
useFastMode = kwargs.get("useFastMode", False)
|
||||
|
||||
# Get value parameters
|
||||
cameraFixed = kwargs.get("cameraFixed", False)
|
||||
maxSequentialImages = kwargs.get("maxSequentialImages", 1)
|
||||
fastMode = kwargs.get("fastMode", False)
|
||||
|
||||
# Build settings dictionary
|
||||
# Build settings dictionary - only include what is enabled
|
||||
settings = {}
|
||||
|
||||
# Add cameraFixed only if it's true
|
||||
if cameraFixed is True:
|
||||
# Add cameraFixed only if useCameraFixed is enabled
|
||||
if useCameraFixed:
|
||||
settings["cameraFixed"] = cameraFixed
|
||||
|
||||
# Add maxSequentialImages if provided and valid
|
||||
if maxSequentialImages is not None and 1 <= maxSequentialImages <= 15:
|
||||
# Add maxSequentialImages only if useMaxSequentialImages is enabled
|
||||
if useMaxSequentialImages:
|
||||
settings["maxSequentialImages"] = maxSequentialImages
|
||||
|
||||
# Add fastMode only if useFastMode is enabled
|
||||
if useFastMode:
|
||||
settings["fastMode"] = fastMode
|
||||
|
||||
# Clean up None values
|
||||
settings = {k: v for k, v in settings.items() if v is not None}
|
||||
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
from .utils import runwareUtils as rwUtils
|
||||
from typing import List, Dict, Any
|
||||
|
||||
class inputAudios:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {},
|
||||
"optional": {
|
||||
"Audio1": ("STRING", {
|
||||
"tooltip": "MediaUUID for the first audio file from Runware Media Upload node.",
|
||||
}),
|
||||
"Audio2": ("STRING", {
|
||||
"tooltip": "MediaUUID for the second audio file from Runware Media Upload node.",
|
||||
}),
|
||||
"Audio3": ("STRING", {
|
||||
"tooltip": "MediaUUID for the third audio file from Runware Media Upload node.",
|
||||
}),
|
||||
"Audio4": ("STRING", {
|
||||
"tooltip": "MediaUUID for the fourth audio file from Runware Media Upload node.",
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
DESCRIPTION = "Configure multiple audio inputs (mediaUUIDs) for video generation with audio synchronization."
|
||||
FUNCTION = "createInputAudios"
|
||||
RETURN_TYPES = ("RUNWAREINPUTAUDIOS",)
|
||||
RETURN_NAMES = ("Input Audios",)
|
||||
CATEGORY = "Runware"
|
||||
|
||||
def createInputAudios(self, **kwargs) -> tuple[List[str]]:
|
||||
audio_list = []
|
||||
for i in range(1, 5):
|
||||
audio_key = f"Audio{i}"
|
||||
audio_uuid = kwargs.get(audio_key)
|
||||
if audio_uuid and audio_uuid.strip() != "":
|
||||
audio_list.append(audio_uuid.strip())
|
||||
return (audio_list,)
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
"""
|
||||
Runware Lightricks Provider Settings Node
|
||||
Provides Lightricks-specific settings for video generation
|
||||
"""
|
||||
|
||||
from typing import Optional, Dict, Any
|
||||
|
||||
class RunwareLightricksProviderSettings:
|
||||
"""Runware Lightricks Provider Settings Node"""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {},
|
||||
"optional": {
|
||||
"generateAudio": ("BOOLEAN", {
|
||||
"tooltip": "Enable to generate audio for the video. Default: false.",
|
||||
"default": False,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("RUNWAREPROVIDERSETTINGS",)
|
||||
RETURN_NAMES = ("providerSettings",)
|
||||
FUNCTION = "create_provider_settings"
|
||||
CATEGORY = "Runware/Provider Settings"
|
||||
|
||||
def create_provider_settings(self, **kwargs) -> tuple[Dict[str, Any]]:
|
||||
"""Create Lightricks provider settings"""
|
||||
|
||||
# Get value parameters
|
||||
generateAudio = kwargs.get("generateAudio", False)
|
||||
|
||||
# Build settings dictionary
|
||||
settings = {}
|
||||
|
||||
# Add generateAudio if it's True
|
||||
if generateAudio:
|
||||
settings["generateAudio"] = generateAudio
|
||||
|
||||
return (settings,)
|
||||
|
||||
# Node class mappings
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"RunwareLightricksProviderSettings": RunwareLightricksProviderSettings,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"RunwareLightricksProviderSettings": "Runware Lightricks Provider Settings",
|
||||
}
|
||||
|
||||
@@ -29,7 +29,6 @@ class RunwareMediaUpload:
|
||||
"tooltip": "Media tensor from ComfyUI load video or audio node",
|
||||
}),
|
||||
"mediaUUID": ("STRING", {
|
||||
"multiline": True,
|
||||
"placeholder": "Generated media UUID will appear here automatically.",
|
||||
"tooltip": "This field will be automatically populated with the generated media UUID after upload.",
|
||||
}),
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
from .utils import runwareUtils as rwUtils
|
||||
from typing import List, Dict, Any
|
||||
|
||||
class referenceVideos:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {},
|
||||
"optional": {
|
||||
"Video1": ("STRING", {
|
||||
"tooltip": "MediaUUID for the first reference video from Runware Media Upload node.",
|
||||
}),
|
||||
"Video2": ("STRING", {
|
||||
"tooltip": "MediaUUID for the second reference video from Runware Media Upload node.",
|
||||
}),
|
||||
"Video3": ("STRING", {
|
||||
"tooltip": "MediaUUID for the third reference video from Runware Media Upload node.",
|
||||
}),
|
||||
"Video4": ("STRING", {
|
||||
"tooltip": "MediaUUID for the fourth reference video from Runware Media Upload node.",
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
DESCRIPTION = "Configure multiple reference video inputs (mediaUUIDs) for video generation."
|
||||
FUNCTION = "createReferenceVideos"
|
||||
RETURN_TYPES = ("RUNWAREREFERENCEVIDEOS",)
|
||||
RETURN_NAMES = ("Reference Videos",)
|
||||
CATEGORY = "Runware"
|
||||
|
||||
def createReferenceVideos(self, **kwargs) -> tuple[List[str]]:
|
||||
video_list = []
|
||||
for i in range(1, 5):
|
||||
video_key = f"Video{i}"
|
||||
video_uuid = kwargs.get(video_key)
|
||||
if video_uuid and video_uuid.strip() != "":
|
||||
video_list.append(video_uuid.strip())
|
||||
return (video_list,)
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
from .utils import runwareUtils as rwUtils
|
||||
import json
|
||||
|
||||
class vectorize:
|
||||
RUNWARE_VECTORIZE_MODELS = {
|
||||
"recraft:1@1": "recraft:1@1",
|
||||
"picsart:1@1": "picsart:1@1",
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"Image": ("IMAGE", {
|
||||
"tooltip": "Input image to be vectorized."
|
||||
}),
|
||||
"Model": (list(cls.RUNWARE_VECTORIZE_MODELS.keys()), {
|
||||
"tooltip": "Select the vectorization model to use.",
|
||||
"default": "recraft:1@1",
|
||||
}),
|
||||
},
|
||||
}
|
||||
|
||||
DESCRIPTION = "Convert images to vector format using Runware's vectorization service."
|
||||
FUNCTION = "vectorizeImage"
|
||||
RETURN_TYPES = ("IMAGE", "STRING")
|
||||
RETURN_NAMES = ("IMAGE", "Vector Data")
|
||||
CATEGORY = "Runware"
|
||||
|
||||
def vectorizeImage(self, **kwargs):
|
||||
image = kwargs.get("Image")
|
||||
modelName = kwargs.get("Model", "recraft:1@1")
|
||||
|
||||
# Get the model AIR code
|
||||
model = self.RUNWARE_VECTORIZE_MODELS.get(modelName, "recraft:1@1")
|
||||
|
||||
# Convert image to UUID
|
||||
image_uuid = rwUtils.convertTensor2IMG(image)
|
||||
|
||||
# Build the API request
|
||||
genConfig = [
|
||||
{
|
||||
"taskType": "vectorize",
|
||||
"taskUUID": rwUtils.genRandUUID(),
|
||||
"model": model,
|
||||
"inputs": {
|
||||
"image": image_uuid
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
try:
|
||||
# Debug: Print the request being sent
|
||||
print(f"[DEBUG] Sending Vectorize Request:")
|
||||
print(f"[DEBUG] Request Payload: {json.dumps(genConfig, indent=2)}")
|
||||
|
||||
genResult = rwUtils.inferenecRequest(genConfig)
|
||||
|
||||
# Debug: Print the response received
|
||||
print(f"[DEBUG] Received Vectorize Response:")
|
||||
print(f"[DEBUG] Response: {json.dumps(genResult, indent=2)}")
|
||||
|
||||
# Check for errors
|
||||
if "errors" in genResult:
|
||||
error_message = genResult["errors"][0]["message"]
|
||||
raise Exception(f"Vectorization failed: {error_message}")
|
||||
|
||||
# Extract the result
|
||||
if "data" in genResult and len(genResult["data"]) > 0:
|
||||
result_data = genResult["data"][0]
|
||||
|
||||
# Return the vectorized image URL
|
||||
if "imageURL" in result_data:
|
||||
vector_url = result_data["imageURL"]
|
||||
return (image, vector_url)
|
||||
else:
|
||||
raise Exception("imageURL not found in response")
|
||||
else:
|
||||
raise Exception("No data returned from vectorization API")
|
||||
|
||||
except Exception as e:
|
||||
print(f"[Error] Vectorization failed: {str(e)}")
|
||||
raise e
|
||||
|
||||
# Node class mappings
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"RunwareVectorize": vectorize,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"RunwareVectorize": "Runware Vectorize",
|
||||
}
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
from .utils import runwareUtils as rwUtils
|
||||
import json
|
||||
import comfy.model_management
|
||||
|
||||
class videoBgRemoval:
|
||||
RUNWARE_VRMBG_MODELS = {
|
||||
"bria:51@1": "bria:51@1",
|
||||
}
|
||||
|
||||
BACKGROUND_COLORS = {
|
||||
"Transparent": [0, 0, 0, 0],
|
||||
"Black": [0, 0, 0, 1],
|
||||
"White": [255, 255, 255, 0],
|
||||
"Gray": [128, 128, 128, 0],
|
||||
"Red": [255, 0, 0, 0],
|
||||
"Green": [0, 255, 0, 0],
|
||||
"Blue": [0, 0, 255, 0],
|
||||
"Yellow": [255, 255, 0, 0],
|
||||
"Cyan": [0, 255, 255, 0],
|
||||
"Magenta": [255, 0, 255, 0],
|
||||
"Orange": [255, 165, 0, 0],
|
||||
}
|
||||
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"Video": ("STRING", {
|
||||
"tooltip": "MediaUUID from Runware Media Upload node for the video to process (max 30 seconds). Supports resolution up to 16000x16000 (16K)."
|
||||
}),
|
||||
"Model": (list(cls.RUNWARE_VRMBG_MODELS.keys()), {
|
||||
"tooltip": "Select the video background removal model to use.",
|
||||
"default": "Bria Video Background Removal",
|
||||
}),
|
||||
},
|
||||
"optional": {
|
||||
"Background Color": (list(cls.BACKGROUND_COLORS.keys()), {
|
||||
"default": "Transparent",
|
||||
"tooltip": "Background color preset to use. Hex values are not supported.",
|
||||
}),
|
||||
"Output Format": (["mp4", "webm", "mov"], {
|
||||
"default": "mp4",
|
||||
"tooltip": "Choose the output video format.",
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
DESCRIPTION = "Remove complex or moving video backgrounds in real time, isolating subjects with clean alpha channels for seamless compositing."
|
||||
FUNCTION = "removeBackground"
|
||||
RETURN_TYPES = ("VIDEO",)
|
||||
RETURN_NAMES = ("VIDEO",)
|
||||
CATEGORY = "Runware"
|
||||
|
||||
def removeBackground(self, **kwargs):
|
||||
video_uuid = kwargs.get("Video")
|
||||
modelName = kwargs.get("Model", "Bria Video Background Removal")
|
||||
backgroundColor = kwargs.get("Background Color", "Transparent")
|
||||
outputFormat = kwargs.get("Output Format", "mp4")
|
||||
|
||||
# Validate mediaUUID
|
||||
if not video_uuid or video_uuid.strip() == "":
|
||||
raise ValueError("Video mediaUUID is required")
|
||||
|
||||
# Get the model AIR code
|
||||
modelAIR = self.RUNWARE_VRMBG_MODELS.get(modelName, "bria:51@1")
|
||||
|
||||
# Get background color RGBA
|
||||
rgba_color = self.BACKGROUND_COLORS.get(backgroundColor, [0, 0, 0, 0])
|
||||
|
||||
# Build the API request
|
||||
genConfig = [
|
||||
{
|
||||
"taskType": "removeBackground",
|
||||
"taskUUID": rwUtils.genRandUUID(),
|
||||
"inputs": {
|
||||
"video": video_uuid
|
||||
},
|
||||
"model": modelAIR,
|
||||
"outputFormat": outputFormat,
|
||||
"settings": {
|
||||
"rgba": rgba_color
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
try:
|
||||
# Debug: Print the request being sent
|
||||
print(f"[DEBUG] Sending Video Background Removal Request:")
|
||||
print(f"[DEBUG] Request Payload: {json.dumps(genConfig, indent=2)}")
|
||||
|
||||
genResult = rwUtils.inferenecRequest(genConfig)
|
||||
|
||||
# Debug: Print the response received
|
||||
print(f"[DEBUG] Received Video Background Removal Response:")
|
||||
print(f"[DEBUG] Response: {json.dumps(genResult, indent=2)}")
|
||||
|
||||
# Check for errors
|
||||
if "errors" in genResult:
|
||||
error_message = genResult["errors"][0]["message"]
|
||||
raise Exception(f"Video background removal failed: {error_message}")
|
||||
|
||||
# Extract task UUID for polling
|
||||
taskUUID = genConfig[0]["taskUUID"]
|
||||
|
||||
# Poll for video completion
|
||||
while True:
|
||||
# Check for interrupt before each poll
|
||||
comfy.model_management.throw_exception_if_processing_interrupted()
|
||||
|
||||
# Poll for video result
|
||||
pollResult = rwUtils.pollVideoResult(taskUUID)
|
||||
print(f"[Debugging] Poll result: {pollResult}")
|
||||
|
||||
# Check for errors first
|
||||
if pollResult and "errors" in pollResult and len(pollResult["errors"]) > 0:
|
||||
error_info = pollResult["errors"][0]
|
||||
error_message = error_info.get("message", "Unknown error")
|
||||
raise Exception(f"Video background removal failed: {error_message}")
|
||||
|
||||
if pollResult and "data" in pollResult and len(pollResult["data"]) > 0:
|
||||
video_data = pollResult["data"][0]
|
||||
|
||||
# Check status directly
|
||||
if "status" in video_data:
|
||||
status = video_data["status"]
|
||||
|
||||
if status == "success":
|
||||
if "videoURL" in video_data or "videoBase64Data" in video_data:
|
||||
videos = rwUtils.convertVideoB64List(pollResult, 1920, 1080)
|
||||
return videos
|
||||
|
||||
# If status is "processing", continue polling
|
||||
|
||||
# Check for interrupt before waiting
|
||||
comfy.model_management.throw_exception_if_processing_interrupted()
|
||||
|
||||
# Wait before next poll
|
||||
for _ in range(10): # 10 x 0.1 second = 1 second total
|
||||
comfy.model_management.throw_exception_if_processing_interrupted()
|
||||
rwUtils.time.sleep(0.1)
|
||||
|
||||
except Exception as e:
|
||||
print(f"[Error] Video background removal failed: {str(e)}")
|
||||
raise e
|
||||
|
||||
# Node class mappings
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"RunwareVideoBgRemoval": videoBgRemoval,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"RunwareVideoBgRemoval": "Runware Video Background Removal",
|
||||
}
|
||||
|
||||
+70
-18
@@ -84,16 +84,6 @@ class txt2vid:
|
||||
"Model": ("RUNWAREVIDEOMODEL", {
|
||||
"tooltip": "Connect a Runware Video Model From Runware Video Model Search Node.",
|
||||
}),
|
||||
"positivePrompt": ("STRING", {
|
||||
"multiline": True,
|
||||
"placeholder": "Positive Prompt: a text instruction to guide the model on generating the video. It is usually a sentence or a paragraph that provides positive guidance for the task. This parameter is essential to shape the desired results.\n\nYou Can Press (Ctrl + Alt + E) To Enhance The Prompt!",
|
||||
"tooltip": "Positive Prompt: a text instruction to guide the model on generating the video. You Can Also Press (Ctrl + Alt + E) To Enhance The Prompt!"
|
||||
}),
|
||||
"negativePrompt": ("STRING", {
|
||||
"multiline": True,
|
||||
"placeholder": "Negative Prompt: a text instruction to guide the model on generating the video. It is usually a sentence or a paragraph that provides negative guidance for the task. This parameter helps to avoid certain undesired results.",
|
||||
"tooltip": "Negative Prompt: a text instruction to guide the model on generating the video."
|
||||
}),
|
||||
"Multi Inference Mode": ("BOOLEAN", {
|
||||
"tooltip": "If Enabled the node will skip the video generation process and will only return the Runware Task Object to be used in the Multi Inference Node.",
|
||||
"default": False,
|
||||
@@ -162,6 +152,16 @@ class txt2vid:
|
||||
}),
|
||||
},
|
||||
"optional": {
|
||||
"positivePrompt": ("STRING", {
|
||||
"multiline": True,
|
||||
"placeholder": "Positive Prompt: a text instruction to guide the model on generating the video. It is usually a sentence or a paragraph that provides positive guidance for the task. This parameter is essential to shape the desired results.\n\nYou Can Press (Ctrl + Alt + E) To Enhance The Prompt!",
|
||||
"tooltip": "Positive Prompt: a text instruction to guide the model on generating the video. You Can Also Press (Ctrl + Alt + E) To Enhance The Prompt!"
|
||||
}),
|
||||
"negativePrompt": ("STRING", {
|
||||
"multiline": True,
|
||||
"placeholder": "Negative Prompt: a text instruction to guide the model on generating the video. It is usually a sentence or a paragraph that provides negative guidance for the task. This parameter helps to avoid certain undesired results.",
|
||||
"tooltip": "Negative Prompt: a text instruction to guide the model on generating the video."
|
||||
}),
|
||||
"frameImages": ("RUNWAREFRAMEIMAGES", {
|
||||
"tooltip": "Frame images configuration from Runware Frame Images node. Allows precise control over frame positioning.",
|
||||
}),
|
||||
@@ -174,17 +174,31 @@ class txt2vid:
|
||||
"inputAudios": ("RUNWAREINPUTAUDIOS", {
|
||||
"tooltip": "Connect input audio files for video generation with audio synchronization.",
|
||||
}),
|
||||
"referenceVideos": ("RUNWAREREFERENCEVIDEOS", {
|
||||
"tooltip": "Connect reference video files for video generation.",
|
||||
}),
|
||||
"speechVoice": ("STRING", {
|
||||
"tooltip": "Voice for speech synthesis. Specify the voice to use for text-to-speech generation.",
|
||||
}),
|
||||
"speechText": ("STRING", {
|
||||
"multiline": True,
|
||||
"tooltip": "Text for speech synthesis. The text that will be converted to speech using the specified voice.",
|
||||
}),
|
||||
"inputs": ("RUNWAREVIDEOINFERENCEINPUTS", {
|
||||
"tooltip": "Connect a Runware Video Inference Inputs node to provide custom inputs like image, audio, mask, and parameters for OmniHuman 1.5 and other video models.",
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def VALIDATE_INPUTS(cls, positivePrompt, negativePrompt):
|
||||
if (positivePrompt is not None and (positivePrompt == "" or len(positivePrompt) < 3 or len(positivePrompt) > 2000)):
|
||||
raise Exception(
|
||||
"Positive Prompt Must Be Between 3 And 2000 characters!")
|
||||
if (negativePrompt is not None and negativePrompt != "" and (len(negativePrompt) < 3 or len(negativePrompt) > 2000)):
|
||||
raise Exception(
|
||||
"Negative Prompt Must Be Between 3 And 2000 characters!")
|
||||
def VALIDATE_INPUTS(cls, positivePrompt=None, negativePrompt=None):
|
||||
# Only validate if prompts are provided and not empty
|
||||
if positivePrompt is not None and positivePrompt.strip() != "":
|
||||
if len(positivePrompt) < 3 or len(positivePrompt) > 2000:
|
||||
raise Exception("Positive Prompt Must Be Between 3 And 2000 characters!")
|
||||
if negativePrompt is not None and negativePrompt.strip() != "":
|
||||
if len(negativePrompt) < 3 or len(negativePrompt) > 2000:
|
||||
raise Exception("Negative Prompt Must Be Between 3 And 2000 characters!")
|
||||
return True
|
||||
|
||||
DESCRIPTION = "Generates Videos Lightning Fast With Runware Video Inference Engine."
|
||||
@@ -203,6 +217,10 @@ class txt2vid:
|
||||
frameImages = kwargs.get("frameImages", None)
|
||||
referenceImages = kwargs.get("referenceImages", None)
|
||||
inputAudios = kwargs.get("inputAudios", None)
|
||||
referenceVideos = kwargs.get("referenceVideos", None)
|
||||
speechVoice = kwargs.get("speechVoice", None)
|
||||
speechText = kwargs.get("speechText", None)
|
||||
inputs = kwargs.get("inputs", None)
|
||||
useCustomDimensions = kwargs.get("useCustomDimensions", False)
|
||||
customWidth = kwargs.get("width", 864)
|
||||
customHeight = kwargs.get("height", 480)
|
||||
@@ -235,7 +253,6 @@ class txt2vid:
|
||||
{
|
||||
"taskType": "videoInference",
|
||||
"taskUUID": rwUtils.genRandUUID(),
|
||||
"positivePrompt": positivePrompt,
|
||||
"height": height,
|
||||
"width": width,
|
||||
"model": model,
|
||||
@@ -245,6 +262,10 @@ class txt2vid:
|
||||
}
|
||||
]
|
||||
|
||||
# Only add positivePrompt if it's not empty
|
||||
if positivePrompt and positivePrompt.strip() != "":
|
||||
genConfig[0]["positivePrompt"] = positivePrompt
|
||||
|
||||
# Add fps parameter only if enabled
|
||||
if useFps:
|
||||
genConfig[0]["fps"] = fps
|
||||
@@ -276,6 +297,37 @@ class txt2vid:
|
||||
if inputAudios is not None and len(inputAudios) > 0:
|
||||
genConfig[0]["inputAudios"] = inputAudios
|
||||
print(f"[Debugging] Input audios array: {inputAudios}")
|
||||
|
||||
# Add referenceVideos if provided and not empty
|
||||
if referenceVideos is not None:
|
||||
# Handle both single mediaUUID (string) and multiple mediaUUIDs (list)
|
||||
if isinstance(referenceVideos, str) and referenceVideos.strip() != "":
|
||||
genConfig[0]["referenceVideos"] = [referenceVideos.strip()]
|
||||
print(f"[Debugging] Reference videos: {genConfig[0].get('referenceVideos', [])}")
|
||||
elif isinstance(referenceVideos, list) and len(referenceVideos) > 0:
|
||||
genConfig[0]["referenceVideos"] = referenceVideos
|
||||
print(f"[Debugging] Reference videos: {genConfig[0].get('referenceVideos', [])}")
|
||||
|
||||
# Add speech parameters if both voice and text are provided
|
||||
if speechVoice and speechVoice.strip() != "" and speechText and speechText.strip() != "":
|
||||
genConfig[0]["speech"] = {
|
||||
"voice": speechVoice.strip(),
|
||||
"text": speechText.strip()
|
||||
}
|
||||
print(f"[Debugging] Speech parameters: voice='{speechVoice.strip()}', text='{speechText.strip()[:50]}...'")
|
||||
|
||||
# Handle inputs - merge custom inputs from Video Inference Inputs node
|
||||
if inputs is not None:
|
||||
# Merge inputs from video inference inputs node
|
||||
if "inputs" not in genConfig[0]:
|
||||
genConfig[0]["inputs"] = {}
|
||||
|
||||
# Merge each input from inputs (only actual input data, not provider settings)
|
||||
for key, value in inputs.items():
|
||||
genConfig[0]["inputs"][key] = value
|
||||
|
||||
print(f"[Debugging] Video inference inputs merged: {inputs}")
|
||||
|
||||
# Handle providerSettings - extract provider name from model and merge with custom settings
|
||||
if providerSettings is not None:
|
||||
# Extract provider name from model (e.g., "pixverse:1@1" -> "pixverse")
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
from .utils import runwareUtils as rwUtils
|
||||
|
||||
class videoInferenceInputs:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {},
|
||||
"optional": {
|
||||
"Image": ("IMAGE", {
|
||||
"tooltip": "Portrait image for video generation. This will be the main subject that will speak and move in the generated video."
|
||||
}),
|
||||
"Audio": ("STRING", {
|
||||
"tooltip": "Connect the mediaUUID output from Runware Media Upload node with audio file. This audio will be synchronized with the generated video."
|
||||
}),
|
||||
"Mask": ("IMAGE", {
|
||||
"tooltip": "Mask image to specify a specific subject in the image to speak. Use white/black mask format."
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
DESCRIPTION = "Configure custom inputs for Runware Video Inference with OmniHuman 1.5 support, including image, audio, and mask inputs."
|
||||
FUNCTION = "createInputs"
|
||||
RETURN_TYPES = ("RUNWAREVIDEOINFERENCEINPUTS",)
|
||||
RETURN_NAMES = ("Video Inference Inputs",)
|
||||
CATEGORY = "Runware"
|
||||
|
||||
def createInputs(self, **kwargs):
|
||||
image = kwargs.get("Image", None)
|
||||
audio = kwargs.get("Audio", None)
|
||||
mask = kwargs.get("Mask", None)
|
||||
|
||||
inputs = {}
|
||||
|
||||
if image is not None:
|
||||
image_uuid = rwUtils.convertTensor2IMG(image)
|
||||
inputs["image"] = image_uuid
|
||||
|
||||
if audio is not None and audio.strip() != "":
|
||||
inputs["audio"] = audio.strip()
|
||||
|
||||
if mask is not None:
|
||||
mask_uuid = rwUtils.convertTensor2IMG(mask)
|
||||
inputs["mask"] = mask_uuid
|
||||
|
||||
return (inputs,)
|
||||
|
||||
@@ -21,6 +21,8 @@ class videoModelSearch:
|
||||
"Seedance": [
|
||||
"bytedance:2@1 (Seedance 1.0 Pro)",
|
||||
"bytedance:1@1 (Seedance 1.0 Lite)",
|
||||
"bytedance:5@1 (OmniHuman 1)",
|
||||
"bytedance:5@2 (OmniHuman 1.5)",
|
||||
],
|
||||
"MiniMax": [
|
||||
"minimax:1@1 (MiniMax 01 Base)",
|
||||
@@ -32,6 +34,7 @@ class videoModelSearch:
|
||||
"pixverse:1@1 (PixVerse v3.5)",
|
||||
"pixverse:1@2 (PixVerse v4)",
|
||||
"pixverse:1@3 (PixVerse v4.5)",
|
||||
"pixverse:lipsync@1 (PixVerse LipSync)",
|
||||
],
|
||||
"Vidu": [
|
||||
"vidu:1@0 (Vidu Q1 Classic)",
|
||||
@@ -47,6 +50,9 @@ class videoModelSearch:
|
||||
"openai:3@1 (OpenAI Sora 3.1)",
|
||||
"openai:3@0 (OpenAI Sora 3.0)",
|
||||
],
|
||||
"Lightricks": [
|
||||
"lightricks:2@1 (LTX)",
|
||||
],
|
||||
}
|
||||
|
||||
# Model dimensions mapping
|
||||
@@ -71,6 +77,8 @@ class videoModelSearch:
|
||||
# Seedance Models
|
||||
"bytedance:2@1": {"width": 864, "height": 480}, # Seedance 1.0 Pro
|
||||
"bytedance:1@1": {"width": 864, "height": 480}, # Seedance 1.0 Lite
|
||||
"bytedance:5@1": {"width": 1024, "height": 1024}, # OmniHuman 1
|
||||
"bytedance:5@2": {"width": 1024, "height": 1024}, # OmniHuman 1.5
|
||||
|
||||
# MiniMax Models
|
||||
"minimax:1@1": {"width": 1366, "height": 768}, # MiniMax 01 Base
|
||||
@@ -82,6 +90,7 @@ class videoModelSearch:
|
||||
"pixverse:1@1": {"width": 640, "height": 360}, # PixVerse v3.5
|
||||
"pixverse:1@2": {"width": 640, "height": 360}, # PixVerse v4
|
||||
"pixverse:1@3": {"width": 640, "height": 360}, # PixVerse v4.5
|
||||
"pixverse:lipsync@1": {"width": 640, "height": 360}, # PixVerse LipSync
|
||||
|
||||
# Vidu Models
|
||||
"vidu:1@0": {"width": 1920, "height": 1080}, # Vidu Q1 Classic
|
||||
@@ -96,6 +105,9 @@ class videoModelSearch:
|
||||
# OpenAI Models
|
||||
"openai:3@1": {"width": 1280, "height": 720}, # OpenAI Sora 3.1
|
||||
"openai:3@0": {"width": 1280, "height": 720}, # OpenAI Sora 3.0
|
||||
|
||||
# Lightricks Models
|
||||
"lightricks:2@1": {"width": 1920, "height": 1080}, # Lightricks v2.1
|
||||
}
|
||||
|
||||
|
||||
@@ -120,7 +132,8 @@ class videoModelSearch:
|
||||
"PixVerse",
|
||||
"Vidu",
|
||||
"Wan",
|
||||
"OpenAI"
|
||||
"OpenAI",
|
||||
"Lightricks"
|
||||
], {
|
||||
"tooltip": "Choose Video Model Architecture To Filter Results.",
|
||||
"default": "KlingAI",
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
from .utils import runwareUtils as rwUtils
|
||||
import json
|
||||
import comfy.model_management
|
||||
|
||||
class videoUpscaler:
|
||||
RUNWARE_VUPSCALER_MODELS = {
|
||||
"Bria Video Upscaler": "bria:50@1",
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"Video": ("STRING", {
|
||||
"tooltip": "MediaUUID from Runware Media Upload node for the video to upscale."
|
||||
}),
|
||||
"upscaleFactor": ([2, 4], {
|
||||
"tooltip": "Integer scale factor for upscaling (2x or 4x).",
|
||||
"default": 2,
|
||||
}),
|
||||
"Model": (list(cls.RUNWARE_VUPSCALER_MODELS.keys()), {
|
||||
"tooltip": "Select the video upscaler model to use.",
|
||||
"default": "Bria Video Upscaler",
|
||||
}),
|
||||
},
|
||||
"optional": {
|
||||
"Output Format": (["mp4", "webm"], {
|
||||
"default": "mp4",
|
||||
"tooltip": "Choose the output video format.",
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
DESCRIPTION = "Enhance the resolution and quality of your videos using Runware's advanced upscaling API."
|
||||
FUNCTION = "upscale"
|
||||
RETURN_TYPES = ("VIDEO",)
|
||||
RETURN_NAMES = ("VIDEO",)
|
||||
CATEGORY = "Runware"
|
||||
|
||||
def upscale(self, **kwargs):
|
||||
video_uuid = kwargs.get("Video")
|
||||
upscaleFactor = kwargs.get("upscaleFactor", 2)
|
||||
modelName = kwargs.get("Model", "Bria Video Upscaler")
|
||||
outputFormat = kwargs.get("Output Format", "mp4")
|
||||
|
||||
# Validate mediaUUID
|
||||
if not video_uuid or video_uuid.strip() == "":
|
||||
raise ValueError("Video mediaUUID is required")
|
||||
|
||||
# Get the model AIR code
|
||||
modelAIR = self.RUNWARE_VUPSCALER_MODELS.get(modelName, "bria:50@1")
|
||||
|
||||
# Build the API request
|
||||
genConfig = [
|
||||
{
|
||||
"taskType": "upscale",
|
||||
"taskUUID": rwUtils.genRandUUID(),
|
||||
"upscaleFactor": upscaleFactor,
|
||||
"inputs": {
|
||||
"video": video_uuid
|
||||
},
|
||||
"outputType": "URL",
|
||||
"outputFormat": outputFormat.upper(),
|
||||
"model": modelAIR,
|
||||
"includeCost": True
|
||||
}
|
||||
]
|
||||
|
||||
try:
|
||||
# Debug: Print the request being sent
|
||||
print(f"[DEBUG] Sending Video Upscale Request:")
|
||||
print(f"[DEBUG] Request Payload: {json.dumps(genConfig, indent=2)}")
|
||||
|
||||
genResult = rwUtils.inferenecRequest(genConfig)
|
||||
|
||||
# Debug: Print the response received
|
||||
print(f"[DEBUG] Received Video Upscale Response:")
|
||||
print(f"[DEBUG] Response: {json.dumps(genResult, indent=2)}")
|
||||
|
||||
# Check for errors
|
||||
if "errors" in genResult:
|
||||
error_message = genResult["errors"][0]["message"]
|
||||
raise Exception(f"Video upscale failed: {error_message}")
|
||||
|
||||
# Extract task UUID for polling
|
||||
taskUUID = genConfig[0]["taskUUID"]
|
||||
|
||||
# Poll for video completion
|
||||
while True:
|
||||
# Check for interrupt before each poll
|
||||
comfy.model_management.throw_exception_if_processing_interrupted()
|
||||
|
||||
# Poll for video result
|
||||
pollResult = rwUtils.pollVideoResult(taskUUID)
|
||||
print(f"[Debugging] Poll result: {pollResult}")
|
||||
|
||||
# Check for errors first
|
||||
if pollResult and "errors" in pollResult and len(pollResult["errors"]) > 0:
|
||||
error_info = pollResult["errors"][0]
|
||||
error_message = error_info.get("message", "Unknown error")
|
||||
raise Exception(f"Video upscale failed: {error_message}")
|
||||
|
||||
if pollResult and "data" in pollResult and len(pollResult["data"]) > 0:
|
||||
video_data = pollResult["data"][0]
|
||||
|
||||
# Check status directly
|
||||
if "status" in video_data:
|
||||
status = video_data["status"]
|
||||
|
||||
if status == "success":
|
||||
if "videoURL" in video_data or "videoBase64Data" in video_data:
|
||||
videos = rwUtils.convertVideoB64List(pollResult, 1920, 1080)
|
||||
return videos
|
||||
|
||||
# If status is "processing", continue polling
|
||||
|
||||
# Check for interrupt before waiting
|
||||
comfy.model_management.throw_exception_if_processing_interrupted()
|
||||
|
||||
# Wait before next poll
|
||||
for _ in range(10): # 10 x 0.1 second = 1 second total
|
||||
comfy.model_management.throw_exception_if_processing_interrupted()
|
||||
rwUtils.time.sleep(0.1)
|
||||
|
||||
except Exception as e:
|
||||
print(f"[Error] Video upscale failed: {str(e)}")
|
||||
raise e
|
||||
|
||||
# Node class mappings
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"RunwareVideoUpscaler": videoUpscaler,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"RunwareVideoUpscaler": "Runware Video Upscaler",
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user