Add Sync provider settings, audio/speech inputs, and useBatchSize parameter
This commit is contained in:
Sirshendu Ganguly
@@ -0,0 +1,157 @@
|
||||
"""
|
||||
Runware Sync Provider Settings Node
|
||||
Provides Sync-specific settings for video generation with lip-sync capabilities
|
||||
"""
|
||||
|
||||
from typing import Optional, Dict, Any, List
|
||||
|
||||
|
||||
class RunwareSyncProviderSettings:
|
||||
"""Runware Sync Provider Settings Node"""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"syncMode": (["bounce", "loop", "cut_off", "silence", "remap"], {
|
||||
"tooltip": "Synchronization strategy. Enum: bounce, loop, cut_off, silence, remap",
|
||||
"default": "bounce",
|
||||
}),
|
||||
},
|
||||
"optional": {
|
||||
"useEditRegion": ("BOOLEAN", {
|
||||
"tooltip": "Enable to include editRegion parameter",
|
||||
"default": False,
|
||||
}),
|
||||
"editRegion": (["lips", "face", "head"], {
|
||||
"tooltip": "Edit region for the model. Only works with react-1. Defaults to face, which affects lipsync + emotions in the face region. When head is selected, model generates natural talking head movements along with emotions + lipsync. Only used when 'Use Edit Region' is enabled.",
|
||||
"default": "face",
|
||||
}),
|
||||
"useEmotionPrompt": ("BOOLEAN", {
|
||||
"tooltip": "Enable to include emotionPrompt parameter",
|
||||
"default": False,
|
||||
}),
|
||||
"emotionPrompt": (["happy", "sad", "angry", "disgusted", "surprised", "neutral"], {
|
||||
"tooltip": "Emotion prompt for the generation. Only works for react-1 model. Only single word emotions are supported. Only used when 'Use Emotion Prompt' is enabled.",
|
||||
"default": "happy",
|
||||
}),
|
||||
"useTemperature": ("BOOLEAN", {
|
||||
"tooltip": "Enable to include temperature parameter",
|
||||
"default": False,
|
||||
}),
|
||||
"temperature": ("FLOAT", {
|
||||
"tooltip": "Sampling temperature. 0 -> least expressive, 1 -> most expressive. Only used when 'Use Temperature' is enabled.",
|
||||
"default": 0.5,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"step": 0.1,
|
||||
}),
|
||||
"useActiveSpeakerDetection": ("BOOLEAN", {
|
||||
"tooltip": "Enable to include activeSpeakerDetection parameter",
|
||||
"default": False,
|
||||
}),
|
||||
"activeSpeakerDetection": ("BOOLEAN", {
|
||||
"tooltip": "Enable active speaker detection. Only used when 'Use Active Speaker Detection' is enabled.",
|
||||
"default": True,
|
||||
"label_on": "Enabled",
|
||||
"label_off": "Disabled",
|
||||
}),
|
||||
"useOcclusionDetectionEnabled": ("BOOLEAN", {
|
||||
"tooltip": "Enable to include occlusionDetectionEnabled parameter",
|
||||
"default": False,
|
||||
}),
|
||||
"occlusionDetectionEnabled": ("BOOLEAN", {
|
||||
"tooltip": "Enable occlusion detection. Only used when 'Use Occlusion Detection Enabled' is enabled.",
|
||||
"default": False,
|
||||
"label_on": "Enabled",
|
||||
"label_off": "Disabled",
|
||||
}),
|
||||
"Sync Segment 1": ("RUNWARESYNCSEGMENT", {
|
||||
"tooltip": "Connect a Runware Sync Segment node (optional)",
|
||||
}),
|
||||
"Sync Segment 2": ("RUNWARESYNCSEGMENT", {
|
||||
"tooltip": "Connect a Runware Sync Segment node (optional)",
|
||||
}),
|
||||
"Sync Segment 3": ("RUNWARESYNCSEGMENT", {
|
||||
"tooltip": "Connect a Runware Sync Segment node (optional)",
|
||||
}),
|
||||
"Sync Segment 4": ("RUNWARESYNCSEGMENT", {
|
||||
"tooltip": "Connect a Runware Sync Segment node (optional)",
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("RUNWAREPROVIDERSETTINGS",)
|
||||
RETURN_NAMES = ("Provider Settings",)
|
||||
FUNCTION = "createProviderSettings"
|
||||
CATEGORY = "Runware/Provider Settings"
|
||||
DESCRIPTION = "Configure Sync-specific provider settings for video generation including lip-sync, emotion prompts, and segment configurations."
|
||||
|
||||
def createProviderSettings(self, **kwargs) -> tuple[Dict[str, Any]]:
|
||||
"""Create Sync provider settings"""
|
||||
|
||||
# Get required parameters
|
||||
syncMode = kwargs.get("syncMode", "bounce")
|
||||
|
||||
# Get control parameters
|
||||
useEditRegion = kwargs.get("useEditRegion", False)
|
||||
useEmotionPrompt = kwargs.get("useEmotionPrompt", False)
|
||||
useTemperature = kwargs.get("useTemperature", False)
|
||||
useActiveSpeakerDetection = kwargs.get("useActiveSpeakerDetection", False)
|
||||
useOcclusionDetectionEnabled = kwargs.get("useOcclusionDetectionEnabled", False)
|
||||
|
||||
# Get value parameters
|
||||
editRegion = kwargs.get("editRegion", "face")
|
||||
emotionPrompt = kwargs.get("emotionPrompt", "happy")
|
||||
temperature = kwargs.get("temperature", 0.5)
|
||||
activeSpeakerDetection = kwargs.get("activeSpeakerDetection", True)
|
||||
occlusionDetectionEnabled = kwargs.get("occlusionDetectionEnabled", False)
|
||||
|
||||
# Get sync segments (up to 4)
|
||||
segments: List[Dict[str, Any]] = []
|
||||
for i in range(1, 5):
|
||||
segmentKey = f"Sync Segment {i}"
|
||||
segment = kwargs.get(segmentKey)
|
||||
if segment is not None and isinstance(segment, dict):
|
||||
segments.append(segment)
|
||||
|
||||
# Build settings dictionary
|
||||
syncSettings: Dict[str, Any] = {
|
||||
"syncMode": syncMode,
|
||||
}
|
||||
|
||||
# Add optional parameters only if enabled
|
||||
if useEditRegion:
|
||||
syncSettings["editRegion"] = editRegion
|
||||
if useEmotionPrompt:
|
||||
syncSettings["emotionPrompt"] = emotionPrompt
|
||||
if useTemperature:
|
||||
syncSettings["temperature"] = float(temperature)
|
||||
if useActiveSpeakerDetection:
|
||||
syncSettings["activeSpeakerDetection"] = activeSpeakerDetection
|
||||
if useOcclusionDetectionEnabled:
|
||||
syncSettings["occlusionDetectionEnabled"] = occlusionDetectionEnabled
|
||||
|
||||
# Add segments if any are provided
|
||||
if segments:
|
||||
syncSettings["segments"] = segments
|
||||
|
||||
# Clean up None values
|
||||
syncSettings = {k: v for k, v in syncSettings.items() if v is not None}
|
||||
|
||||
# Return settings wrapped with provider key "sync"
|
||||
if syncSettings:
|
||||
return ({"sync": syncSettings},)
|
||||
else:
|
||||
return ({},)
|
||||
|
||||
|
||||
# Node class mappings
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"RunwareSyncProviderSettings": RunwareSyncProviderSettings,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"RunwareSyncProviderSettings": "Runware Sync Provider Settings",
|
||||
}
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
"""
|
||||
Runware Sync Segment Node
|
||||
Creates a sync segment configuration for sync provider settings
|
||||
"""
|
||||
|
||||
from typing import Optional, Dict, Any
|
||||
|
||||
|
||||
class RunwareSyncSegment:
|
||||
"""Runware Sync Segment Node"""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"startTime": ("FLOAT", {
|
||||
"tooltip": "Start time of the segment in seconds",
|
||||
"default": 0.0,
|
||||
"min": 0.0,
|
||||
"step": 0.1,
|
||||
}),
|
||||
"endTime": ("FLOAT", {
|
||||
"tooltip": "End time of the segment in seconds",
|
||||
"default": 1.0,
|
||||
"min": 0.0,
|
||||
"step": 0.1,
|
||||
}),
|
||||
"ref": ("STRING", {
|
||||
"tooltip": "Reference identifier for the audio track (e.g., 'audio-track-1', 'speech-track-1')",
|
||||
"default": "audio-track-1",
|
||||
}),
|
||||
},
|
||||
"optional": {
|
||||
"useAudioStartTime": ("BOOLEAN", {
|
||||
"tooltip": "Enable to include audioStartTime parameter",
|
||||
"default": False,
|
||||
}),
|
||||
"audioStartTime": ("FLOAT", {
|
||||
"tooltip": "Start time in the audio track in seconds. Only used when 'Use Audio Start Time' is enabled.",
|
||||
"default": 0.0,
|
||||
"min": 0.0,
|
||||
"step": 0.1,
|
||||
}),
|
||||
"useAudioEndTime": ("BOOLEAN", {
|
||||
"tooltip": "Enable to include audioEndTime parameter",
|
||||
"default": False,
|
||||
}),
|
||||
"audioEndTime": ("FLOAT", {
|
||||
"tooltip": "End time in the audio track in seconds. Only used when 'Use Audio End Time' is enabled.",
|
||||
"default": 1.0,
|
||||
"min": 0.0,
|
||||
"step": 0.1,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("RUNWARESYNCSEGMENT",)
|
||||
RETURN_NAMES = ("Sync Segment",)
|
||||
FUNCTION = "createSyncSegment"
|
||||
CATEGORY = "Runware/Provider Settings"
|
||||
DESCRIPTION = "Create a sync segment configuration for sync provider settings."
|
||||
|
||||
def createSyncSegment(self, **kwargs) -> tuple[Dict[str, Any]]:
|
||||
"""Create sync segment configuration"""
|
||||
|
||||
# Get required parameters
|
||||
startTime = kwargs.get("startTime", 0.0)
|
||||
endTime = kwargs.get("endTime", 1.0)
|
||||
ref = kwargs.get("ref", "audio-track-1")
|
||||
|
||||
# Get optional parameters
|
||||
useAudioStartTime = kwargs.get("useAudioStartTime", False)
|
||||
audioStartTime = kwargs.get("audioStartTime", 0.0)
|
||||
useAudioEndTime = kwargs.get("useAudioEndTime", False)
|
||||
audioEndTime = kwargs.get("audioEndTime", 1.0)
|
||||
|
||||
# Build segment dictionary
|
||||
segment: Dict[str, Any] = {
|
||||
"startTime": float(startTime),
|
||||
"endTime": float(endTime),
|
||||
"ref": str(ref),
|
||||
}
|
||||
|
||||
# Add optional audio timing parameters
|
||||
if useAudioStartTime:
|
||||
segment["audioStartTime"] = float(audioStartTime)
|
||||
if useAudioEndTime:
|
||||
segment["audioEndTime"] = float(audioEndTime)
|
||||
|
||||
# Clean up None values
|
||||
segment = {k: v for k, v in segment.items() if v is not None}
|
||||
|
||||
return (segment,)
|
||||
|
||||
|
||||
# Node class mappings
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"RunwareSyncSegment": RunwareSyncSegment,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"RunwareSyncSegment": "Runware Sync Segment",
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user