From f303887f3695814e5a71661f0b2a7121ef6b60e8 Mon Sep 17 00:00:00 2001 From: Sirshendu Ganguly Date: Fri, 3 Apr 2026 20:57:31 +0530 Subject: [PATCH 1/3] Added Kling Elements --- __init__.py | 6 + clientlibs/types.js | 10 + modules/mediaUpload.py | 17 +- modules/utils/runwareUtils.py | 29 +++ modules/videoInferenceElements.py | 216 ++++++++++++++++++ modules/videoInferenceInputs.py | 12 + ...unware_Video_Inference_Kling_Elements.json | 1 + 7 files changed, 278 insertions(+), 13 deletions(-) create mode 100644 modules/videoInferenceElements.py create mode 100644 workflows/Runware_Video_Inference_Kling_Elements.json diff --git a/__init__.py b/__init__.py index 638bb9d..58d9eaa 100644 --- a/__init__.py +++ b/__init__.py @@ -64,6 +64,10 @@ from .modules.speechInputCombine import RunwareSpeechInputCombine from .modules.referenceVideos import referenceVideos from .modules.referenceVoices import referenceVoices from .modules.videoInferenceInputs import videoInferenceInputs +from .modules.videoInferenceElements import ( + RunwareVideoInferenceElements, + RunwareVideoInferenceElementsCombine, +) from .modules.providerSettings.lightricksProviderSettings import RunwareLightricksProviderSettings from .modules.providerSettings.minimaxProviderSettings import RunwareMiniMaxProviderSettings from .modules.providerSettings.klingProviderSettings import RunwareKlingProviderSettings @@ -181,6 +185,8 @@ NODE_CLASS_MAPPINGS = { "Runware Reference Videos": referenceVideos, "Runware Reference Voices": referenceVoices, "Runware Video Inference Inputs": videoInferenceInputs, + "Runware Video Inference Elements": RunwareVideoInferenceElements, + "Runware Video Inference Elements Combine": RunwareVideoInferenceElementsCombine, "Runware Video Inference Speech Input": RunwareVideoInferenceSpeechInput, "Runware Video Inference Settings": RunwareVideoSettings, "Runware Lightricks Provider Settings": RunwareLightricksProviderSettings, diff --git a/clientlibs/types.js b/clientlibs/types.js index f0aa88d..2f35991 100644 --- a/clientlibs/types.js +++ b/clientlibs/types.js @@ -108,6 +108,8 @@ const RUNWARE_NODE_TYPES = { REFERENCEVIDEOS: "Runware Reference Videos", REFERENCEVOICES: "Runware Reference Voices", VIDEOINFERENCEINPUTS: "Runware Video Inference Inputs", + VIDEOINFERENCEELEMENTS: "Runware Video Inference Elements", + VIDEOINFERENCEELEMENTSCOMBINE: "Runware Video Inference Elements Combine", VIDEOINFERENCESPEECHINPUT: "Runware Video Inference Speech Input", VIDEOSETTINGS: "Runware Video Inference Settings", LIGHTRICKSPROVIDERSETTINGS: "Runware Lightricks Provider Settings", @@ -407,6 +409,14 @@ const RUNWARE_NODE_PROPS = { bgColor: DEFAULT_BGCOLOR, colorModeOnly: true, }, + [RUNWARE_NODE_TYPES.VIDEOINFERENCEELEMENTS]: { + bgColor: DEFAULT_BGCOLOR, + colorModeOnly: true, + }, + [RUNWARE_NODE_TYPES.VIDEOINFERENCEELEMENTSCOMBINE]: { + bgColor: DEFAULT_BGCOLOR, + colorModeOnly: true, + }, [RUNWARE_NODE_TYPES.LIGHTRICKSPROVIDERSETTINGS]: { bgColor: DEFAULT_BGCOLOR, colorModeOnly: true, diff --git a/modules/mediaUpload.py b/modules/mediaUpload.py index 03975f6..2430bd2 100644 --- a/modules/mediaUpload.py +++ b/modules/mediaUpload.py @@ -14,6 +14,7 @@ from .utils.runwareUtils import ( sanitize_for_logging, safe_json_dumps, sendMediaUUID, + convertVideoInputToDataUri, ) @@ -106,19 +107,9 @@ class RunwareMediaUpload: def _convertVideoToBase64(self, videoTensor): """Convert video file to base64 data URI""" print(f"[Debug] VideoFromFile object detected") - - videoFile = self._getVideoFilePath(videoTensor) - videoPath = self._extractVideoPath(videoFile) - - print(f"[Debug] Processing video file: {videoPath}") - - with open(videoPath, 'rb') as f: - videoBytes = f.read() - - mimeType = self._getMimeType(videoPath) - videoBase64 = base64.b64encode(videoBytes).decode('utf-8') - - return f"data:{mimeType};base64,{videoBase64}" + out = convertVideoInputToDataUri(videoTensor) + print(f"[Debug] Processing video file (data URI length {len(out) if out else 0})") + return out def _getVideoFilePath(self, videoTensor): """Extract video file path from VideoFromFile object""" diff --git a/modules/utils/runwareUtils.py b/modules/utils/runwareUtils.py index dcd2743..2dd6f02 100644 --- a/modules/utils/runwareUtils.py +++ b/modules/utils/runwareUtils.py @@ -516,6 +516,35 @@ def convertTensor2IMG(tensorImage): return imgDataUri +def convertVideoInputToDataUri(video_tensor): + """Convert ComfyUI VIDEO (file-backed, e.g. Load Video) to data:;base64,... for API payloads.""" + if video_tensor is None: + return None + video_file = getattr(video_tensor, "_VideoFromFile__file", None) + if video_file is None: + for attr in ("video_path", "path", "file_path", "filename", "_file"): + if hasattr(video_tensor, attr): + video_file = getattr(video_tensor, attr) + break + if video_file is None: + raise ValueError( + "Could not resolve video file from VIDEO input. Connect a Load Video (file) output." + ) + path = video_file.name if hasattr(video_file, "name") else video_file + if not isinstance(path, str): + raise ValueError(f"Unexpected video file type: {type(video_file)}") + with open(path, "rb") as f: + video_bytes = f.read() + ext = os.path.splitext(path.lower())[1] + mime = { + ".mp4": "video/mp4", + ".avi": "video/avi", + ".mov": "video/quicktime", + ".webm": "video/webm", + }.get(ext, "video/mp4") + b64 = base64.b64encode(video_bytes).decode("utf-8") + return f"data:{mime};base64,{b64}" + def convertTensor2IMGBase64Only(tensorImage): """Convert tensor to base64 data URI without caching - for frame images""" diff --git a/modules/videoInferenceElements.py b/modules/videoInferenceElements.py new file mode 100644 index 0000000..e10e12f --- /dev/null +++ b/modules/videoInferenceElements.py @@ -0,0 +1,216 @@ +""" +Runware Video Inference Elements — builds inputs.elements[] for videoInference (Kling elements, etc.). +""" + +import re +from typing import Any, Dict, List, Optional, Tuple +from .utils import runwareUtils as rwUtils + +_REF_IMAGE_SLOTS = 3 +_REF_VIDEO_SLOTS = 4 + +_ALLOWED_TAGS = frozenset( + { + "Hottest", + "Character", + "Animal", + "Item", + "Costume", + "Scene", + "Effect", + "Others", + } +) + + +def _split_lines_or_csv(value: Optional[str]) -> List[str]: + if value is None: + return [] + s = str(value).strip() + if not s: + return [] + parts = re.split(r"[\n,]+", s) + return [p.strip() for p in parts if p.strip()] + + +def _parse_tags(value: Optional[str]) -> List[str]: + raw = _split_lines_or_csv(value) + if not raw: + return [] + out: List[str] = [] + for t in raw: + if t in _ALLOWED_TAGS: + if t not in out: + out.append(t) + return out + + +def _build_element_dict( + id_: str, + description: str, + frontalImage: Any, + refer_images: List[Any], + refer_videos: List[Any], + voice: str, + tags: str, +) -> Dict[str, Any]: + el: Dict[str, Any] = {} + id_s = (id_ or "").strip() + if id_s: + el["id"] = id_s + desc = (description or "").strip() + if desc: + el["description"] = desc + if isinstance(frontalImage, str): + front = frontalImage.strip() + if front: + el["frontalImage"] = front + elif frontalImage is not None: + el["frontalImage"] = frontalImage + imgs = [x for x in refer_images if x is not None] + if imgs: + el["images"] = imgs + vids = [x for x in refer_videos if x is not None] + if vids: + el["videos"] = vids + voices = _split_lines_or_csv(voice) + if voices: + el["voice"] = voices + tag_list = _parse_tags(tags) + if tag_list: + el["tags"] = tag_list + return el + + +class RunwareVideoInferenceElements: + """One Kling / video inference element for inputs.elements.""" + + @classmethod + def INPUT_TYPES(cls): + optional: Dict[str, tuple] = { + "id": ("STRING", { + "default": "", + "tooltip": "User-facing element id (max 20 chars on API). For reuse, set only id with no reference media.", + }), + "description": ("STRING", { + "default": "", + "multiline": True, + "tooltip": "Optional element description (e.g. for image_refer / new elements).", + }), + "frontalImage": ("IMAGE", { + "tooltip": "Frontal reference image for the element.", + }), + "voice": ("STRING", { + "default": "", + "multiline": True, + "tooltip": "Voice file URLs (mp3/wav/mp4/mov), one per line or comma-separated.", + }), + "tags": ("STRING", { + "default": "", + "multiline": True, + "tooltip": "Comma or newline-separated tags: Hottest, Character, Animal, Item, Costume, Scene, Effect, Others.", + }), + } + for i in range(1, _REF_IMAGE_SLOTS + 1): + optional[f"images_{i}"] = ("IMAGE", { + "tooltip": f"Optional reference image {i} for image_refer (up to {_REF_IMAGE_SLOTS}).", + }) + for i in range(1, _REF_VIDEO_SLOTS + 1): + optional[f"videos_{i}"] = ("VIDEO", { + "tooltip": f"Optional reference video {i} for video_refer (file-backed VIDEO, e.g. Load Video).", + }) + return {"required": {}, "optional": optional} + + RETURN_TYPES = ("RUNWAREVIDEOINFERENCEELEMENTS",) + RETURN_NAMES = ("Elements",) + FUNCTION = "build_element" + CATEGORY = "Runware" + DESCRIPTION = ( + "Define a single video inference element (id, description, frontalImage, images_1…images_3, videos_1…videos_4, voice, tags) " + "for inputs.elements on Kling and compatible models. Connect to Runware Video Inference Inputs → Elements, " + "or use Runware Video Inference Elements Combine for multiple elements." + ) + + def build_element(self, **kwargs) -> Tuple[List[Dict[str, Any]]]: + frontal_image = kwargs.get("frontalImage", None) + frontal_image_value = None + if frontal_image is not None: + frontal_image_value = rwUtils.convertTensor2IMG(frontal_image) + + refer_images: List[Any] = [] + for i in range(1, _REF_IMAGE_SLOTS + 1): + slot = kwargs.get(f"images_{i}") + if slot is not None: + refer_images.append(rwUtils.convertTensor2IMG(slot)) + + refer_videos: List[Any] = [] + for i in range(1, _REF_VIDEO_SLOTS + 1): + slot = kwargs.get(f"videos_{i}") + if slot is not None: + refer_videos.append(rwUtils.convertVideoInputToDataUri(slot)) + + el = _build_element_dict( + str(kwargs.get("id") or ""), + str(kwargs.get("description") or ""), + frontal_image_value, + refer_images, + refer_videos, + str(kwargs.get("voice") or ""), + str(kwargs.get("tags") or ""), + ) + if not el: + raise ValueError( + "Runware Video Inference Elements: provide at least id, description, frontalImage, " + "images_1…images_3, videos_1…videos_4, voice, or tags." + ) + return ([el],) + + +class RunwareVideoInferenceElementsCombine: + """Combine multiple Runware Video Inference Elements (each output is a list) into one list.""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "Elements 1": ("RUNWAREVIDEOINFERENCEELEMENTS", { + "tooltip": "Connect a Runware Video Inference Elements node (single-element list).", + }), + }, + "optional": { + "Elements 2": ("RUNWAREVIDEOINFERENCEELEMENTS", { + "tooltip": "Optional second Runware Video Inference Elements node.", + }), + "Elements 3": ("RUNWAREVIDEOINFERENCEELEMENTS", { + "tooltip": "Optional third Runware Video Inference Elements node.", + }), + "Elements 4": ("RUNWAREVIDEOINFERENCEELEMENTS", { + "tooltip": "Optional fourth Runware Video Inference Elements node.", + }), + }, + } + + RETURN_TYPES = ("RUNWAREVIDEOINFERENCEELEMENTS",) + RETURN_NAMES = ("Elements",) + FUNCTION = "combine" + CATEGORY = "Runware" + DESCRIPTION = ( + "Combine up to four Runware Video Inference Elements outputs into inputs.elements. " + "Connect the result to Runware Video Inference Inputs → Elements." + ) + + def combine(self, **kwargs) -> Tuple[List[Dict[str, Any]]]: + out: List[Dict[str, Any]] = [] + for i in range(1, 5): + key = f"Elements {i}" + chunk = kwargs.get(key) + if chunk is not None and isinstance(chunk, list): + out.extend( + e for e in chunk + if isinstance(e, dict) and len(e) > 0 + ) + if not out: + raise ValueError( + "Runware Video Inference Elements Combine: connect at least one non-empty Elements input." + ) + return (out,) diff --git a/modules/videoInferenceInputs.py b/modules/videoInferenceInputs.py index 1ab8e1f..b5edc1d 100644 --- a/modules/videoInferenceInputs.py +++ b/modules/videoInferenceInputs.py @@ -60,6 +60,9 @@ class videoInferenceInputs: "multiline": False, "tooltip": "Video ID from a previous generation. Used when extending or editing an existing OpenAI Sora video", }), + "Elements": ("RUNWAREVIDEOINFERENCEELEMENTS", { + "tooltip": "Connect Runware Video Inference Elements (one element) or Runware Video Inference Elements Combine (several) for inputs.elements (Kling, etc.).", + }), } } @@ -89,6 +92,7 @@ class videoInferenceInputs: draftId = kwargs.get("draftId", "") videoId = kwargs.get("videoId", "") avatar = kwargs.get("Avatar", "") + elements_list = kwargs.get("Elements", None) inputs = {} @@ -148,4 +152,12 @@ class videoInferenceInputs: if videoId is not None and videoId.strip() != "": inputs["videoId"] = videoId.strip() + if elements_list is not None and isinstance(elements_list, list): + merged_elements = [ + e for e in elements_list + if isinstance(e, dict) and len(e) > 0 + ] + if merged_elements: + inputs["elements"] = merged_elements + return (inputs,) diff --git a/workflows/Runware_Video_Inference_Kling_Elements.json b/workflows/Runware_Video_Inference_Kling_Elements.json new file mode 100644 index 0000000..b1b263b --- /dev/null +++ b/workflows/Runware_Video_Inference_Kling_Elements.json @@ -0,0 +1 @@ +{"id":"62fd2431-54ee-45ea-ad42-ad0ad64150bc","revision":0,"last_node_id":17,"last_link_id":13,"nodes":[{"id":4,"type":"SaveVideo","pos":[849.9493408203125,298.8796081542969],"size":[374,324.281005859375],"flags":{},"order":9,"mode":0,"inputs":[{"localized_name":"video","name":"video","type":"VIDEO","link":2},{"localized_name":"filename_prefix","name":"filename_prefix","type":"STRING","widget":{"name":"filename_prefix"},"link":null},{"localized_name":"format","name":"format","type":"COMBO","widget":{"name":"format"},"link":null},{"localized_name":"codec","name":"codec","type":"COMBO","widget":{"name":"codec"},"link":null}],"outputs":[],"properties":{"cnr_id":"comfy-core","ver":"0.3.46","Node name for S&R":"SaveVideo"},"widgets_values":["video/ComfyUI","auto","auto"]},{"id":11,"type":"LoadImage","pos":[-12.338081359863281,595.6199340820312],"size":[274.080078125,314],"flags":{},"order":0,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[7]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"cnr_id":"comfy-core","ver":"0.3.46","Node name for S&R":"LoadImage"},"widgets_values":["man_beard.jpg","image"]},{"id":10,"type":"Runware Frame Images","pos":[1.7095589637756348,998.5339965820312],"size":[270,190],"flags":{},"order":4,"mode":0,"inputs":[{"localized_name":"image1","name":"image1","shape":7,"type":"IMAGE","link":7},{"localized_name":"image2","name":"image2","shape":7,"type":"IMAGE","link":null},{"localized_name":"image3","name":"image3","shape":7,"type":"IMAGE","link":null},{"localized_name":"image4","name":"image4","shape":7,"type":"IMAGE","link":null},{"localized_name":"frame1_position","name":"frame1_position","shape":7,"type":"COMBO","widget":{"name":"frame1_position"},"link":null},{"localized_name":"frame2_position","name":"frame2_position","shape":7,"type":"COMBO","widget":{"name":"frame2_position"},"link":null},{"localized_name":"frame3_position","name":"frame3_position","shape":7,"type":"COMBO","widget":{"name":"frame3_position"},"link":null},{"localized_name":"frame4_position","name":"frame4_position","shape":7,"type":"COMBO","widget":{"name":"frame4_position"},"link":null}],"outputs":[{"localized_name":"Frame Images","name":"Frame Images","type":"RUNWAREFRAMEIMAGES","links":[8]}],"properties":{"cnr_id":"ComfyUI-Runware","ver":"fb5ec57d989698df6324392fe93591b178ded22a","Node name for S&R":"Runware Frame Images"},"widgets_values":["auto","auto","auto","auto"],"bgcolor":"#5345bf"},{"id":15,"type":"Runware Video Inference Inputs","pos":[823.6644287109375,728.4757080078125],"size":[320.0390625,354],"flags":{},"order":7,"mode":0,"inputs":[{"localized_name":"Image","name":"Image","shape":7,"type":"IMAGE","link":null},{"localized_name":"Frame Images","name":"Frame Images","shape":7,"type":"RUNWAREVIDEOINPUTSFRAMEIMAGES","link":null},{"localized_name":"Audio Inputs","name":"Audio Inputs","shape":7,"type":"RUNWAREAUDIOINPUT","link":null},{"localized_name":"Speech Inputs","name":"Speech Inputs","shape":7,"type":"RUNWARESPEECHINPUT","link":null},{"localized_name":"Mask","name":"Mask","shape":7,"type":"IMAGE","link":null},{"localized_name":"Frame","name":"Frame","shape":7,"type":"IMAGE","link":null},{"localized_name":"Background","name":"Background","shape":7,"type":"IMAGE","link":null},{"localized_name":"References","name":"References","shape":7,"type":"RUNWAREVIDEOINPUTSREFERENCEIMAGES","link":null},{"localized_name":"Reference Videos","name":"Reference Videos","shape":7,"type":"RUNWAREREFERENCEVIDEOS","link":null},{"localized_name":"Reference Voices","name":"Reference Voices","shape":7,"type":"RUNWAREREFERENCEVOICES","link":null},{"localized_name":"Elements","name":"Elements","shape":7,"type":"RUNWAREVIDEOINFERENCEELEMENTS","link":11},{"localized_name":"Avatar","name":"Avatar","shape":7,"type":"STRING","widget":{"name":"Avatar"},"link":null},{"localized_name":"Audio","name":"Audio","shape":7,"type":"STRING","widget":{"name":"Audio"},"link":null},{"localized_name":"Video","name":"Video","shape":7,"type":"STRING","widget":{"name":"Video"},"link":null},{"localized_name":"draftId","name":"draftId","shape":7,"type":"STRING","widget":{"name":"draftId"},"link":null},{"localized_name":"videoId","name":"videoId","shape":7,"type":"STRING","widget":{"name":"videoId"},"link":null}],"outputs":[{"localized_name":"Video Inference Inputs","name":"Video Inference Inputs","type":"RUNWAREVIDEOINFERENCEINPUTS","links":[9]}],"properties":{"cnr_id":"ComfyUI-Runware","ver":"fb5ec57d989698df6324392fe93591b178ded22a","Node name for S&R":"Runware Video Inference Inputs"},"widgets_values":["","","","",""],"bgcolor":"#5345bf"},{"id":13,"type":"Runware Video Inference Elements Combine","pos":[1234.8345947265625,710.3389892578125],"size":[319.7974548339844,86],"flags":{},"order":6,"mode":0,"inputs":[{"localized_name":"Elements 1","name":"Elements 1","type":"RUNWAREVIDEOINFERENCEELEMENTS","link":10},{"localized_name":"Elements 2","name":"Elements 2","shape":7,"type":"RUNWAREVIDEOINFERENCEELEMENTS","link":null},{"localized_name":"Elements 3","name":"Elements 3","shape":7,"type":"RUNWAREVIDEOINFERENCEELEMENTS","link":null},{"localized_name":"Elements 4","name":"Elements 4","shape":7,"type":"RUNWAREVIDEOINFERENCEELEMENTS","link":null}],"outputs":[{"localized_name":"Elements","name":"Elements","type":"RUNWAREVIDEOINFERENCEELEMENTS","links":[11]}],"properties":{"cnr_id":"ComfyUI-Runware","ver":"fb5ec57d989698df6324392fe93591b178ded22a","Node name for S&R":"Runware Video Inference Elements Combine"},"widgets_values":[],"bgcolor":"#5345bf"},{"id":17,"type":"LoadImage","pos":[1371.8580322265625,352.4707336425781],"size":[274.080078125,314.00006103515625],"flags":{},"order":1,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[12]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"cnr_id":"comfy-core","ver":"0.3.46","Node name for S&R":"LoadImage"},"widgets_values":["image_720_1280.png","image"]},{"id":16,"type":"LoadImage","pos":[1696.6624755859375,766.121826171875],"size":[274.080078125,314],"flags":{},"order":2,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[13]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"cnr_id":"comfy-core","ver":"0.3.46","Node name for S&R":"LoadImage"},"widgets_values":["image_720_1280.png","image"]},{"id":3,"type":"Runware Video Model Search","pos":[34.05699157714844,236.76271057128906],"size":[287.380859375,250],"flags":{},"order":3,"mode":0,"inputs":[{"localized_name":"Model Search","name":"Model Search","type":"STRING","widget":{"name":"Model Search"},"link":null},{"localized_name":"Model Architecture","name":"Model Architecture","type":"COMBO","widget":{"name":"Model Architecture"},"link":null},{"localized_name":"VideoList","name":"VideoList","type":"COMBO","widget":{"name":"VideoList"},"link":null},{"localized_name":"Use Search Value","name":"Use Search Value","type":"BOOLEAN","widget":{"name":"Use Search Value"},"link":null},{"localized_name":"useCustomDimensions","name":"useCustomDimensions","type":"COMBO","widget":{"name":"useCustomDimensions"},"link":null},{"localized_name":"Width","name":"Width","type":"INT","widget":{"name":"Width"},"link":null},{"localized_name":"Height","name":"Height","type":"INT","widget":{"name":"Height"},"link":null},{"localized_name":"useResolution","name":"useResolution","type":"BOOLEAN","widget":{"name":"useResolution"},"link":null},{"localized_name":"resolution","name":"resolution","type":"COMBO","widget":{"name":"resolution"},"link":null}],"outputs":[{"localized_name":"Runware Video Model","name":"Runware Video Model","type":"RUNWAREVIDEOMODEL","links":[1]}],"properties":{"cnr_id":"ComfyUI-Runware","ver":"a4a26667a03e63511d48db870f7e1acc31b5105b","Node name for S&R":"Runware Video Model Search"},"widgets_values":["","KlingAI","klingai:kling-video@o3-standard (Kling VIDEO O3 Standard)",false,"Custom",1280,720,false,"1080p"],"bgcolor":"#5345bf"},{"id":1,"type":"Runware Video Inference","pos":[386.22015380859375,243.1965789794922],"size":[400,876],"flags":{},"order":8,"mode":0,"inputs":[{"localized_name":"Model","name":"Model","type":"RUNWAREVIDEOMODEL","link":1},{"localized_name":"frameImages","name":"frameImages","shape":7,"type":"RUNWAREFRAMEIMAGES","link":8},{"localized_name":"referenceImages","name":"referenceImages","shape":7,"type":"RUNWAREREFERENCEIMAGES","link":null},{"localized_name":"providerSettings","name":"providerSettings","shape":7,"type":"RUNWAREPROVIDERSETTINGS","link":null},{"localized_name":"inputAudios","name":"inputAudios","shape":7,"type":"RUNWAREINPUTAUDIOS","link":null},{"localized_name":"referenceVideos","name":"referenceVideos","shape":7,"type":"RUNWAREREFERENCEVIDEOS","link":null},{"localized_name":"speech","name":"speech","shape":7,"type":"RUNWAREVIDEOINFERENCESPEECHINPUT","link":null},{"localized_name":"inputs","name":"inputs","shape":7,"type":"RUNWAREVIDEOINFERENCEINPUTS","link":9},{"localized_name":"safetyInputs","name":"safetyInputs","shape":7,"type":"RUNWARESAFETYINPUTS","link":null},{"localized_name":"videoAdvancedFeatureInputs","name":"videoAdvancedFeatureInputs","shape":7,"type":"RUNWAREVIDEOADVANCEDFEATUREINPUTS","link":null},{"localized_name":"Accelerator","name":"Accelerator","shape":7,"type":"RUNWAREACCELERATOR","link":null},{"localized_name":"settings","name":"settings","shape":7,"type":"RUNWAREVIDEOSETTINGS","link":null},{"localized_name":"Multi Inference Mode","name":"Multi Inference Mode","type":"BOOLEAN","widget":{"name":"Multi Inference Mode"},"link":null},{"localized_name":"Prompt Weighting","name":"Prompt Weighting","type":"COMBO","widget":{"name":"Prompt Weighting"},"link":null},{"localized_name":"useDuration","name":"useDuration","type":"BOOLEAN","widget":{"name":"useDuration"},"link":null},{"localized_name":"duration","name":"duration","type":"INT","widget":{"name":"duration"},"link":null},{"localized_name":"useFps","name":"useFps","type":"BOOLEAN","widget":{"name":"useFps"},"link":null},{"localized_name":"fps","name":"fps","type":"INT","widget":{"name":"fps"},"link":null},{"localized_name":"outputFormat","name":"outputFormat","type":"COMBO","widget":{"name":"outputFormat"},"link":null},{"localized_name":"useSeed","name":"useSeed","type":"BOOLEAN","widget":{"name":"useSeed"},"link":null},{"localized_name":"seed","name":"seed","type":"INT","widget":{"name":"seed"},"link":null},{"localized_name":"useSteps","name":"useSteps","type":"BOOLEAN","widget":{"name":"useSteps"},"link":null},{"localized_name":"steps","name":"steps","type":"INT","widget":{"name":"steps"},"link":null},{"localized_name":"useBatchSize","name":"useBatchSize","type":"BOOLEAN","widget":{"name":"useBatchSize"},"link":null},{"localized_name":"batchSize","name":"batchSize","type":"INT","widget":{"name":"batchSize"},"link":null},{"localized_name":"useSchedulers","name":"useSchedulers","type":"BOOLEAN","widget":{"name":"useSchedulers"},"link":null},{"localized_name":"scheduler","name":"scheduler","type":"COMBO","widget":{"name":"scheduler"},"link":null},{"localized_name":"useCFGScale","name":"useCFGScale","type":"BOOLEAN","widget":{"name":"useCFGScale"},"link":null},{"localized_name":"cfgScale","name":"cfgScale","type":"FLOAT","widget":{"name":"cfgScale"},"link":null},{"localized_name":"acceleration","name":"acceleration","type":"COMBO","widget":{"name":"acceleration"},"link":null},{"localized_name":"positivePrompt","name":"positivePrompt","shape":7,"type":"STRING","widget":{"name":"positivePrompt"},"link":null},{"localized_name":"negativePrompt","name":"negativePrompt","shape":7,"type":"STRING","widget":{"name":"negativePrompt"},"link":null}],"outputs":[{"localized_name":"VIDEO","name":"VIDEO","type":"VIDEO","links":[2]},{"localized_name":"RW-Task","name":"RW-Task","type":"RUNWARETASK","links":null},{"localized_name":"OUTPUT","name":"OUTPUT","type":"RUNWARETASK","links":null}],"properties":{"cnr_id":"ComfyUI-Runware","ver":"a4a26667a03e63511d48db870f7e1acc31b5105b","Node name for S&R":"Runware Video Inference"},"widgets_values":[false,"Disabled",true,12,false,24,"mp4",false,330035029,"randomize",false,20,false,1,false,"euler",false,5,"none","A girl walking with her father",""],"bgcolor":"#5345bf"},{"id":12,"type":"Runware Video Inference Elements","pos":[1189.997314453125,906.7557373046875],"size":[400,360],"flags":{},"order":5,"mode":0,"inputs":[{"localized_name":"frontalImage","name":"frontalImage","shape":7,"type":"IMAGE","link":13},{"localized_name":"images_1","name":"images_1","shape":7,"type":"IMAGE","link":12},{"localized_name":"images_2","name":"images_2","shape":7,"type":"IMAGE","link":null},{"localized_name":"images_3","name":"images_3","shape":7,"type":"IMAGE","link":null},{"localized_name":"videos_1","name":"videos_1","shape":7,"type":"VIDEO","link":null},{"localized_name":"videos_2","name":"videos_2","shape":7,"type":"VIDEO","link":null},{"localized_name":"videos_3","name":"videos_3","shape":7,"type":"VIDEO","link":null},{"localized_name":"videos_4","name":"videos_4","shape":7,"type":"VIDEO","link":null},{"localized_name":"id","name":"id","shape":7,"type":"STRING","widget":{"name":"id"},"link":null},{"localized_name":"description","name":"description","shape":7,"type":"STRING","widget":{"name":"description"},"link":null},{"localized_name":"voice","name":"voice","shape":7,"type":"STRING","widget":{"name":"voice"},"link":null},{"localized_name":"tags","name":"tags","shape":7,"type":"STRING","widget":{"name":"tags"},"link":null}],"outputs":[{"localized_name":"Elements","name":"Elements","type":"RUNWAREVIDEOINFERENCEELEMENTS","links":[10]}],"properties":{"cnr_id":"ComfyUI-Runware","ver":"fb5ec57d989698df6324392fe93591b178ded22a","Node name for S&R":"Runware Video Inference Elements"},"widgets_values":["tagged_character","a young girl wearing green shirt","",""],"bgcolor":"#5345bf"}],"links":[[1,3,0,1,0,"RUNWAREVIDEOMODEL"],[2,1,0,4,0,"VIDEO"],[7,11,0,10,0,"IMAGE"],[8,10,0,1,1,"RUNWAREFRAMEIMAGES"],[9,15,0,1,7,"RUNWAREVIDEOINFERENCEINPUTS"],[10,12,0,13,0,"RUNWAREVIDEOINFERENCEELEMENTS"],[11,13,0,15,10,"RUNWAREVIDEOINFERENCEELEMENTS"],[12,17,0,12,1,"IMAGE"],[13,16,0,12,0,"IMAGE"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.6830134553650705,"offset":[471.90677981523817,-166.79176123385304]}},"version":0.4} \ No newline at end of file From 30d19f263b9150ca22e9e7179d6bd5773da1296a Mon Sep 17 00:00:00 2001 From: Sirshendu Ganguly Date: Fri, 3 Apr 2026 21:27:13 +0530 Subject: [PATCH 2/3] Reverted to original version of mediaUpload --- modules/mediaUpload.py | 17 +++++++++++++---- modules/utils/runwareUtils.py | 30 ------------------------------ modules/videoInferenceElements.py | 23 +++++++++++++---------- 3 files changed, 26 insertions(+), 44 deletions(-) diff --git a/modules/mediaUpload.py b/modules/mediaUpload.py index 2430bd2..1b4f691 100644 --- a/modules/mediaUpload.py +++ b/modules/mediaUpload.py @@ -14,7 +14,6 @@ from .utils.runwareUtils import ( sanitize_for_logging, safe_json_dumps, sendMediaUUID, - convertVideoInputToDataUri, ) @@ -107,9 +106,19 @@ class RunwareMediaUpload: def _convertVideoToBase64(self, videoTensor): """Convert video file to base64 data URI""" print(f"[Debug] VideoFromFile object detected") - out = convertVideoInputToDataUri(videoTensor) - print(f"[Debug] Processing video file (data URI length {len(out) if out else 0})") - return out + + videoFile = self._getVideoFilePath(videoTensor) + videoPath = self._extractVideoPath(videoFile) + + print(f"[Debug] Processing video file: {videoPath}") + + with open(videoPath, 'rb') as f: + videoBytes = f.read() + + mimeType = self._getMimeType(videoPath) + videoBase64 = base64.b64encode(videoBytes).decode('utf-8') + + return f"data:{mimeType};base64,{videoBase64}" def _getVideoFilePath(self, videoTensor): """Extract video file path from VideoFromFile object""" diff --git a/modules/utils/runwareUtils.py b/modules/utils/runwareUtils.py index 2dd6f02..0c97b0f 100644 --- a/modules/utils/runwareUtils.py +++ b/modules/utils/runwareUtils.py @@ -516,36 +516,6 @@ def convertTensor2IMG(tensorImage): return imgDataUri -def convertVideoInputToDataUri(video_tensor): - """Convert ComfyUI VIDEO (file-backed, e.g. Load Video) to data:;base64,... for API payloads.""" - if video_tensor is None: - return None - video_file = getattr(video_tensor, "_VideoFromFile__file", None) - if video_file is None: - for attr in ("video_path", "path", "file_path", "filename", "_file"): - if hasattr(video_tensor, attr): - video_file = getattr(video_tensor, attr) - break - if video_file is None: - raise ValueError( - "Could not resolve video file from VIDEO input. Connect a Load Video (file) output." - ) - path = video_file.name if hasattr(video_file, "name") else video_file - if not isinstance(path, str): - raise ValueError(f"Unexpected video file type: {type(video_file)}") - with open(path, "rb") as f: - video_bytes = f.read() - ext = os.path.splitext(path.lower())[1] - mime = { - ".mp4": "video/mp4", - ".avi": "video/avi", - ".mov": "video/quicktime", - ".webm": "video/webm", - }.get(ext, "video/mp4") - b64 = base64.b64encode(video_bytes).decode("utf-8") - return f"data:{mime};base64,{b64}" - - def convertTensor2IMGBase64Only(tensorImage): """Convert tensor to base64 data URI without caching - for frame images""" imageNP = (tensorImage.squeeze().numpy() * 255).astype(np.uint8) diff --git a/modules/videoInferenceElements.py b/modules/videoInferenceElements.py index e10e12f..f1f0d70 100644 --- a/modules/videoInferenceElements.py +++ b/modules/videoInferenceElements.py @@ -50,7 +50,7 @@ def _build_element_dict( description: str, frontalImage: Any, refer_images: List[Any], - refer_videos: List[Any], + refer_videos: List[str], voice: str, tags: str, ) -> Dict[str, Any]: @@ -70,9 +70,8 @@ def _build_element_dict( imgs = [x for x in refer_images if x is not None] if imgs: el["images"] = imgs - vids = [x for x in refer_videos if x is not None] - if vids: - el["videos"] = vids + if refer_videos: + el["videos"] = refer_videos voices = _split_lines_or_csv(voice) if voices: el["voice"] = voices @@ -116,8 +115,12 @@ class RunwareVideoInferenceElements: "tooltip": f"Optional reference image {i} for image_refer (up to {_REF_IMAGE_SLOTS}).", }) for i in range(1, _REF_VIDEO_SLOTS + 1): - optional[f"videos_{i}"] = ("VIDEO", { - "tooltip": f"Optional reference video {i} for video_refer (file-backed VIDEO, e.g. Load Video).", + optional[f"videos_{i}"] = ("STRING", { + "default": "", + "tooltip": ( + f"Optional reference video {i}: public URL or mediaUUID from Runware Media Upload " + f"(upload a video first, then paste the UUID here)." + ), }) return {"required": {}, "optional": optional} @@ -143,11 +146,11 @@ class RunwareVideoInferenceElements: if slot is not None: refer_images.append(rwUtils.convertTensor2IMG(slot)) - refer_videos: List[Any] = [] + refer_videos: List[str] = [] for i in range(1, _REF_VIDEO_SLOTS + 1): - slot = kwargs.get(f"videos_{i}") - if slot is not None: - refer_videos.append(rwUtils.convertVideoInputToDataUri(slot)) + raw = kwargs.get(f"videos_{i}") + if raw is not None and str(raw).strip() != "": + refer_videos.append(str(raw).strip()) el = _build_element_dict( str(kwargs.get("id") or ""), From 5f118ac471b7ae1464aa70cb6939231d2d6394be Mon Sep 17 00:00:00 2001 From: Sirshendu Ganguly Date: Fri, 3 Apr 2026 21:48:05 +0530 Subject: [PATCH 3/3] Fixed typing nuances --- modules/videoInferenceElements.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/videoInferenceElements.py b/modules/videoInferenceElements.py index f1f0d70..e8f037e 100644 --- a/modules/videoInferenceElements.py +++ b/modules/videoInferenceElements.py @@ -134,7 +134,7 @@ class RunwareVideoInferenceElements: "or use Runware Video Inference Elements Combine for multiple elements." ) - def build_element(self, **kwargs) -> Tuple[List[Dict[str, Any]]]: + def build_element(self, **kwargs) -> Tuple[List[Dict[str, Any]],]: frontal_image = kwargs.get("frontalImage", None) frontal_image_value = None if frontal_image is not None: @@ -202,7 +202,7 @@ class RunwareVideoInferenceElementsCombine: "Connect the result to Runware Video Inference Inputs → Elements." ) - def combine(self, **kwargs) -> Tuple[List[Dict[str, Any]]]: + def combine(self, **kwargs) -> Tuple[List[Dict[str, Any]],]: out: List[Dict[str, Any]] = [] for i in range(1, 5): key = f"Elements {i}"