From acc3ac6456099dc7909166c558504ed3afc2ef1f Mon Sep 17 00:00:00 2001 From: Sirshendu Ganguly Date: Tue, 28 Apr 2026 18:50:07 +0530 Subject: [PATCH] Fixed breaking changes in refImages --- modules/videoInference.py | 91 ++++++++++++++++++- modules/videoInputsReferences.py | 34 ++----- ...re_Video_Inference_xAI_Grok_Ref_Image.json | 1 + 3 files changed, 99 insertions(+), 27 deletions(-) create mode 100644 workflows/Runware_Video_Inference_xAI_Grok_Ref_Image.json diff --git a/modules/videoInference.py b/modules/videoInference.py index a15dddb..544aed9 100644 --- a/modules/videoInference.py +++ b/modules/videoInference.py @@ -4,6 +4,85 @@ import comfy.model_management class txt2vid: + @staticmethod + def _format_reference_images(referenceImages, model): + """ + Shape referenceImages per model requirements: + - skywork:skyreels@v4 -> list of groups: {type, tag, images, audio?} + - others -> flat list[str] (legacy behavior) + """ + if referenceImages is None: + return [] + + if not isinstance(referenceImages, list): + referenceImages = [referenceImages] + + is_skyreels = model == "skywork:skyreels@v4" + + if is_skyreels: + grouped = [] + for idx, ref in enumerate(referenceImages, start=1): + images = [] + ref_type = "image" + ref_tag = f"@image{idx}" + ref_audio = "" + + if isinstance(ref, str): + if ref.strip(): + images = [ref.strip()] + elif isinstance(ref, dict): + ref_type = (str(ref.get("type", "image")).strip() or "image") + raw_tag = str(ref.get("tag", "")).strip() + ref_tag = raw_tag if raw_tag else ref_tag + if not ref_tag.startswith("@"): + ref_tag = f"@{ref_tag}" + + raw_audio = ref.get("audio", "") + if isinstance(raw_audio, str) and raw_audio.strip(): + ref_audio = raw_audio.strip() + + image_value = ref.get("image") + if isinstance(image_value, str) and image_value.strip(): + images.append(image_value.strip()) + + image_group = ref.get("images") + if isinstance(image_group, list): + for item in image_group: + if isinstance(item, str) and item.strip(): + images.append(item.strip()) + + if not images: + continue + + group = { + "type": ref_type, + "tag": ref_tag, + "images": images[:5], # provider limit: max 5 + } + if ref_audio: + group["audio"] = ref_audio + grouped.append(group) + + return grouped + + # Legacy/default path: flatten to list of image strings + flattened = [] + for ref in referenceImages: + if isinstance(ref, str): + if ref.strip(): + flattened.append(ref.strip()) + elif isinstance(ref, dict): + image_value = ref.get("image") + if isinstance(image_value, str) and image_value.strip(): + flattened.append(image_value.strip()) + + image_group = ref.get("images") + if isinstance(image_group, list): + for item in image_group: + if isinstance(item, str) and item.strip(): + flattened.append(item.strip()) + return flattened + @classmethod def INPUT_TYPES(cls): return { @@ -278,8 +357,10 @@ class txt2vid: print(f"[Debugging] Frame images array: {rwUtils.sanitize_for_logging(frameImages)}") # Add referenceImages if provided from Runware Reference Images node if referenceImages is not None and len(referenceImages) > 0: - genConfig[0]["referenceImages"] = referenceImages - print(f"[Debugging] Reference images array: {rwUtils.sanitize_for_logging(referenceImages)}") + formattedReferenceImages = self._format_reference_images(referenceImages, model) + if len(formattedReferenceImages) > 0: + genConfig[0]["referenceImages"] = formattedReferenceImages + print(f"[Debugging] Reference images array: {rwUtils.sanitize_for_logging(formattedReferenceImages)}") # Add inputAudios if provided if inputAudios is not None and len(inputAudios) > 0: genConfig[0]["inputAudios"] = inputAudios @@ -308,6 +389,12 @@ class txt2vid: # Merge each input from inputs (only actual input data, not provider settings) for key, value in inputs.items(): + # Normalize referenceImages so non-SkyReels models keep legacy string[] payload + if key == "referenceImages": + formatted_input_refs = self._format_reference_images(value, model) + if len(formatted_input_refs) > 0: + genConfig[0]["inputs"][key] = formatted_input_refs + continue genConfig[0]["inputs"][key] = value print(f"[Debugging] Video inference inputs merged: {rwUtils.sanitize_for_logging(inputs)}") diff --git a/modules/videoInputsReferences.py b/modules/videoInputsReferences.py index cd7aa56..6f9c013 100644 --- a/modules/videoInputsReferences.py +++ b/modules/videoInputsReferences.py @@ -14,10 +14,7 @@ class videoInputsReferences: for i in range(1, cls.MAX_REFERENCES + 1): ordinal = rwUtils.getOrdinal(i) optionalInputs[f"Image{i}"] = ("IMAGE", { - "tooltip": f"{ordinal.capitalize()} single reference image (mutually exclusive with Images{i}).", - }) - optionalInputs[f"Images{i}"] = ("RUNWAREVIDEOINPUTSREFERENCEMULTIIMAGES", { - "tooltip": f"{ordinal.capitalize()} reference image group from the Multiple Images Connector (mutually exclusive with Image{i}).", + "tooltip": f"{ordinal.capitalize()} reference image.", }) optionalInputs[f"Tag{i}"] = ("STRING", { "tooltip": f"Optional tag for the {ordinal} reference (e.g., @image{i}, @Actor-1).", @@ -38,8 +35,8 @@ class videoInputsReferences: } DESCRIPTION = ( - "Configure multiple reference image entries for video inference inputs. " - "For each slot, provide either ImageN (single image) or ImagesN (connector output), not both." + "Configure multiple reference images for video inference inputs. " + "Use ImageN slots (Image1..Image10)." ) FUNCTION = "createReferences" RETURN_TYPES = ("RUNWAREVIDEOINPUTSREFERENCEIMAGES",) @@ -52,39 +49,26 @@ class videoInputsReferences: for i in range(1, self.MAX_REFERENCES + 1): imageKey = f"Image{i}" - imagesKey = f"Images{i}" tagKey = f"Tag{i}" typeKey = f"Type{i}" audioKey = f"Audio{i}" image = kwargs.get(imageKey) - images = kwargs.get(imagesKey) refTag = (kwargs.get(tagKey) or "").strip() refType = kwargs.get(typeKey, "") refAudio = (kwargs.get(audioKey) or "").strip() - - has_image = image is not None - has_images = isinstance(images, list) and len(images) > 0 - if has_image and has_images: - raise ValueError( - f"Reference {i}: use either {imageKey} or {imagesKey}, not both." - ) - if has_image or has_images: - reference = self._createReference(image, images, refTag, refType, refAudio) + if image is not None: + reference = self._createReference(image, refTag, refType, refAudio) references.append(reference) return (references,) - def _createReference(self, image, images, refTag: str, refType: str, refAudio: str): + def _createReference(self, image, refTag: str, refType: str, refAudio: str): """Create reference entry.""" - entry: Dict[str, Any] = {} - if image is not None: - entry["image"] = rwUtils.convertTensor2IMG(image) - elif isinstance(images, list) and len(images) > 0: - entry["images"] = images - else: - return entry + entry: Dict[str, Any] = { + "image": rwUtils.convertTensor2IMG(image) + } if refTag: entry["tag"] = refTag diff --git a/workflows/Runware_Video_Inference_xAI_Grok_Ref_Image.json b/workflows/Runware_Video_Inference_xAI_Grok_Ref_Image.json new file mode 100644 index 0000000..0ebe311 --- /dev/null +++ b/workflows/Runware_Video_Inference_xAI_Grok_Ref_Image.json @@ -0,0 +1 @@ +{"id":"65d6b0e5-d016-4103-ad00-cfc72a09266d","revision":0,"last_node_id":13,"last_link_id":16,"nodes":[{"id":2,"type":"SaveVideo","pos":[751.237548828125,17.92486000061035],"size":[270,368],"flags":{},"order":5,"mode":0,"inputs":[{"localized_name":"video","name":"video","type":"VIDEO","link":16},{"localized_name":"filename_prefix","name":"filename_prefix","type":"STRING","widget":{"name":"filename_prefix"},"link":null},{"localized_name":"format","name":"format","type":"COMBO","widget":{"name":"format"},"link":null},{"localized_name":"codec","name":"codec","type":"COMBO","widget":{"name":"codec"},"link":null}],"outputs":[],"properties":{"cnr_id":"comfy-core","ver":"0.3.46","Node name for S&R":"SaveVideo"},"widgets_values":["video/ComfyUI","auto","auto"]},{"id":11,"type":"Runware Video Inputs References","pos":[-610.0556635123328,14.091389749887204],"size":[434.710546875,934],"flags":{},"order":2,"mode":0,"inputs":[{"localized_name":"Image1","name":"Image1","shape":7,"type":"IMAGE","link":12},{"localized_name":"Image2","name":"Image2","shape":7,"type":"IMAGE","link":null},{"localized_name":"Image3","name":"Image3","shape":7,"type":"IMAGE","link":null},{"localized_name":"Image4","name":"Image4","shape":7,"type":"IMAGE","link":null},{"localized_name":"Image5","name":"Image5","shape":7,"type":"IMAGE","link":null},{"localized_name":"Image6","name":"Image6","shape":7,"type":"IMAGE","link":null},{"localized_name":"Image7","name":"Image7","shape":7,"type":"IMAGE","link":null},{"localized_name":"Image8","name":"Image8","shape":7,"type":"IMAGE","link":null},{"localized_name":"Image9","name":"Image9","shape":7,"type":"IMAGE","link":null},{"localized_name":"Image10","name":"Image10","shape":7,"type":"IMAGE","link":null},{"localized_name":"Tag1","name":"Tag1","shape":7,"type":"STRING","widget":{"name":"Tag1"},"link":null},{"localized_name":"Type1","name":"Type1","shape":7,"type":"STRING","widget":{"name":"Type1"},"link":null},{"localized_name":"Audio1","name":"Audio1","shape":7,"type":"STRING","widget":{"name":"Audio1"},"link":null},{"localized_name":"Tag2","name":"Tag2","shape":7,"type":"STRING","widget":{"name":"Tag2"},"link":null},{"localized_name":"Type2","name":"Type2","shape":7,"type":"STRING","widget":{"name":"Type2"},"link":null},{"localized_name":"Audio2","name":"Audio2","shape":7,"type":"STRING","widget":{"name":"Audio2"},"link":null},{"localized_name":"Tag3","name":"Tag3","shape":7,"type":"STRING","widget":{"name":"Tag3"},"link":null},{"localized_name":"Type3","name":"Type3","shape":7,"type":"STRING","widget":{"name":"Type3"},"link":null},{"localized_name":"Audio3","name":"Audio3","shape":7,"type":"STRING","widget":{"name":"Audio3"},"link":null},{"localized_name":"Tag4","name":"Tag4","shape":7,"type":"STRING","widget":{"name":"Tag4"},"link":null},{"localized_name":"Type4","name":"Type4","shape":7,"type":"STRING","widget":{"name":"Type4"},"link":null},{"localized_name":"Audio4","name":"Audio4","shape":7,"type":"STRING","widget":{"name":"Audio4"},"link":null},{"localized_name":"Tag5","name":"Tag5","shape":7,"type":"STRING","widget":{"name":"Tag5"},"link":null},{"localized_name":"Type5","name":"Type5","shape":7,"type":"STRING","widget":{"name":"Type5"},"link":null},{"localized_name":"Audio5","name":"Audio5","shape":7,"type":"STRING","widget":{"name":"Audio5"},"link":null},{"localized_name":"Tag6","name":"Tag6","shape":7,"type":"STRING","widget":{"name":"Tag6"},"link":null},{"localized_name":"Type6","name":"Type6","shape":7,"type":"STRING","widget":{"name":"Type6"},"link":null},{"localized_name":"Audio6","name":"Audio6","shape":7,"type":"STRING","widget":{"name":"Audio6"},"link":null},{"localized_name":"Tag7","name":"Tag7","shape":7,"type":"STRING","widget":{"name":"Tag7"},"link":null},{"localized_name":"Type7","name":"Type7","shape":7,"type":"STRING","widget":{"name":"Type7"},"link":null},{"localized_name":"Audio7","name":"Audio7","shape":7,"type":"STRING","widget":{"name":"Audio7"},"link":null},{"localized_name":"Tag8","name":"Tag8","shape":7,"type":"STRING","widget":{"name":"Tag8"},"link":null},{"localized_name":"Type8","name":"Type8","shape":7,"type":"STRING","widget":{"name":"Type8"},"link":null},{"localized_name":"Audio8","name":"Audio8","shape":7,"type":"STRING","widget":{"name":"Audio8"},"link":null},{"localized_name":"Tag9","name":"Tag9","shape":7,"type":"STRING","widget":{"name":"Tag9"},"link":null},{"localized_name":"Type9","name":"Type9","shape":7,"type":"STRING","widget":{"name":"Type9"},"link":null},{"localized_name":"Audio9","name":"Audio9","shape":7,"type":"STRING","widget":{"name":"Audio9"},"link":null},{"localized_name":"Tag10","name":"Tag10","shape":7,"type":"STRING","widget":{"name":"Tag10"},"link":null},{"localized_name":"Type10","name":"Type10","shape":7,"type":"STRING","widget":{"name":"Type10"},"link":null},{"localized_name":"Audio10","name":"Audio10","shape":7,"type":"STRING","widget":{"name":"Audio10"},"link":null}],"outputs":[{"localized_name":"Video Inputs Reference Images","name":"Video Inputs Reference Images","type":"RUNWAREVIDEOINPUTSREFERENCEIMAGES","links":[13]}],"properties":{"cnr_id":"ComfyUI-Runware","ver":"23a176e8d5b5bb377b9c85b4d27a6cd12444b0a7","Node name for S&R":"Runware Video Inputs References"},"widgets_values":["","","","","","","","","","","","","","","","","","","","","","","","","","","","","",""],"bgcolor":"#5345bf"},{"id":13,"type":"Runware Video Inference","pos":[277.15019107295086,-51.563694044433994],"size":[400,818],"flags":{},"order":4,"mode":0,"inputs":[{"localized_name":"Model","name":"Model","type":"RUNWAREVIDEOMODEL","link":15},{"localized_name":"frameImages","name":"frameImages","shape":7,"type":"RUNWAREFRAMEIMAGES","link":null},{"localized_name":"referenceImages","name":"referenceImages","shape":7,"type":"RUNWAREREFERENCEIMAGES","link":null},{"localized_name":"providerSettings","name":"providerSettings","shape":7,"type":"RUNWAREPROVIDERSETTINGS","link":null},{"localized_name":"inputAudios","name":"inputAudios","shape":7,"type":"RUNWAREINPUTAUDIOS","link":null},{"localized_name":"referenceVideos","name":"referenceVideos","shape":7,"type":"RUNWAREREFERENCEVIDEOS","link":null},{"localized_name":"speech","name":"speech","shape":7,"type":"RUNWAREVIDEOINFERENCESPEECHINPUT","link":null},{"localized_name":"inputs","name":"inputs","shape":7,"type":"RUNWAREVIDEOINFERENCEINPUTS","link":14},{"localized_name":"safetyInputs","name":"safetyInputs","shape":7,"type":"RUNWARESAFETYINPUTS","link":null},{"localized_name":"videoAdvancedFeatureInputs","name":"videoAdvancedFeatureInputs","shape":7,"type":"RUNWAREVIDEOADVANCEDFEATUREINPUTS","link":null},{"localized_name":"Accelerator","name":"Accelerator","shape":7,"type":"RUNWAREACCELERATOR","link":null},{"localized_name":"settings","name":"settings","shape":7,"type":"RUNWAREVIDEOSETTINGS","link":null},{"localized_name":"Multi Inference Mode","name":"Multi Inference Mode","type":"BOOLEAN","widget":{"name":"Multi Inference Mode"},"link":null},{"localized_name":"Prompt Weighting","name":"Prompt Weighting","type":"COMBO","widget":{"name":"Prompt Weighting"},"link":null},{"localized_name":"useDuration","name":"useDuration","type":"BOOLEAN","widget":{"name":"useDuration"},"link":null},{"localized_name":"duration","name":"duration","type":"INT","widget":{"name":"duration"},"link":null},{"localized_name":"useFps","name":"useFps","type":"BOOLEAN","widget":{"name":"useFps"},"link":null},{"localized_name":"fps","name":"fps","type":"INT","widget":{"name":"fps"},"link":null},{"localized_name":"outputFormat","name":"outputFormat","type":"COMBO","widget":{"name":"outputFormat"},"link":null},{"localized_name":"useSeed","name":"useSeed","type":"BOOLEAN","widget":{"name":"useSeed"},"link":null},{"localized_name":"seed","name":"seed","type":"INT","widget":{"name":"seed"},"link":null},{"localized_name":"useSteps","name":"useSteps","type":"BOOLEAN","widget":{"name":"useSteps"},"link":null},{"localized_name":"steps","name":"steps","type":"INT","widget":{"name":"steps"},"link":null},{"localized_name":"useBatchSize","name":"useBatchSize","type":"BOOLEAN","widget":{"name":"useBatchSize"},"link":null},{"localized_name":"batchSize","name":"batchSize","type":"INT","widget":{"name":"batchSize"},"link":null},{"localized_name":"useSchedulers","name":"useSchedulers","type":"BOOLEAN","widget":{"name":"useSchedulers"},"link":null},{"localized_name":"scheduler","name":"scheduler","type":"COMBO","widget":{"name":"scheduler"},"link":null},{"localized_name":"useCFGScale","name":"useCFGScale","type":"BOOLEAN","widget":{"name":"useCFGScale"},"link":null},{"localized_name":"cfgScale","name":"cfgScale","type":"FLOAT","widget":{"name":"cfgScale"},"link":null},{"localized_name":"acceleration","name":"acceleration","type":"COMBO","widget":{"name":"acceleration"},"link":null},{"localized_name":"positivePrompt","name":"positivePrompt","shape":7,"type":"STRING","widget":{"name":"positivePrompt"},"link":null},{"localized_name":"negativePrompt","name":"negativePrompt","shape":7,"type":"STRING","widget":{"name":"negativePrompt"},"link":null}],"outputs":[{"localized_name":"VIDEO","name":"VIDEO","type":"VIDEO","links":[16]},{"localized_name":"RW-Task","name":"RW-Task","type":"RUNWARETASK","links":null},{"localized_name":"OUTPUT","name":"OUTPUT","type":"RUNWARETASK","links":null}],"properties":{"cnr_id":"ComfyUI-Runware","ver":"23a176e8d5b5bb377b9c85b4d27a6cd12444b0a7","Node name for S&R":"Runware Video Inference"},"widgets_values":[false,"Disabled",false,5,false,24,"mp4",false,1383176521,"randomize",false,20,false,1,false,"euler",false,5,"none","Girl waving her hand",""],"bgcolor":"#5345bf"},{"id":4,"type":"Runware Video Inference Inputs","pos":[-128.59617614746094,286.5240173339844],"size":[320.0390625,394],"flags":{},"order":3,"mode":0,"inputs":[{"localized_name":"Image","name":"Image","shape":7,"type":"IMAGE","link":null},{"localized_name":"Frame Images","name":"Frame Images","shape":7,"type":"RUNWAREVIDEOINPUTSFRAMEIMAGES","link":null},{"localized_name":"Audio Inputs","name":"Audio Inputs","shape":7,"type":"RUNWAREAUDIOINPUT","link":null},{"localized_name":"Speech Inputs","name":"Speech Inputs","shape":7,"type":"RUNWARESPEECHINPUT","link":null},{"localized_name":"Mask","name":"Mask","shape":7,"type":"IMAGE","link":null},{"localized_name":"Frame","name":"Frame","shape":7,"type":"IMAGE","link":null},{"localized_name":"Background","name":"Background","shape":7,"type":"IMAGE","link":null},{"localized_name":"References Images","name":"References Images","shape":7,"type":"RUNWAREVIDEOINPUTSREFERENCEIMAGES","link":13},{"localized_name":"Reference Videos","name":"Reference Videos","shape":7,"type":"RUNWAREREFERENCEVIDEOS","link":null},{"localized_name":"Reference Voices","name":"Reference Voices","shape":7,"type":"RUNWAREREFERENCEVOICES","link":null},{"localized_name":"Elements","name":"Elements","shape":7,"type":"RUNWAREVIDEOINFERENCEELEMENTS","link":null},{"localized_name":"Reference Audios","name":"Reference Audios","shape":7,"type":"RUNWAREINPUTAUDIOS","link":null},{"localized_name":"Avatar","name":"Avatar","shape":7,"type":"STRING","widget":{"name":"Avatar"},"link":null},{"localized_name":"Audio","name":"Audio","shape":7,"type":"STRING","widget":{"name":"Audio"},"link":null},{"localized_name":"Video","name":"Video","shape":7,"type":"STRING","widget":{"name":"Video"},"link":null},{"localized_name":"draftId","name":"draftId","shape":7,"type":"STRING","widget":{"name":"draftId"},"link":null},{"localized_name":"videoId","name":"videoId","shape":7,"type":"STRING","widget":{"name":"videoId"},"link":null},{"localized_name":"References","name":"References","shape":7,"type":"RUNWAREVIDEOINPUTSREFERENCEIMAGES","link":null}],"outputs":[{"localized_name":"Video Inference Inputs","name":"Video Inference Inputs","type":"RUNWAREVIDEOINFERENCEINPUTS","links":[14]}],"properties":{"cnr_id":"ComfyUI-Runware","ver":"89412ff4b588adf407736eb84e6fb9853b9a033d","Node name for S&R":"Runware Video Inference Inputs"},"widgets_values":["","","","",""],"bgcolor":"#5345bf"},{"id":3,"type":"Runware Video Model Search","pos":[-133.2332000732422,-58.92354965209961],"size":[287.380859375,250],"flags":{},"order":1,"mode":0,"inputs":[{"localized_name":"Model Search","name":"Model Search","type":"STRING","widget":{"name":"Model Search"},"link":null},{"localized_name":"Model Architecture","name":"Model Architecture","type":"COMBO","widget":{"name":"Model Architecture"},"link":null},{"localized_name":"VideoList","name":"VideoList","type":"COMBO","widget":{"name":"VideoList"},"link":null},{"localized_name":"Use Search Value","name":"Use Search Value","type":"BOOLEAN","widget":{"name":"Use Search Value"},"link":null},{"localized_name":"useCustomDimensions","name":"useCustomDimensions","type":"COMBO","widget":{"name":"useCustomDimensions"},"link":null},{"localized_name":"Width","name":"Width","type":"INT","widget":{"name":"Width"},"link":null},{"localized_name":"Height","name":"Height","type":"INT","widget":{"name":"Height"},"link":null},{"localized_name":"useResolution","name":"useResolution","type":"BOOLEAN","widget":{"name":"useResolution"},"link":null},{"localized_name":"resolution","name":"resolution","type":"COMBO","widget":{"name":"resolution"},"link":null}],"outputs":[{"localized_name":"Runware Video Model","name":"Runware Video Model","type":"RUNWAREVIDEOMODEL","links":[15]}],"properties":{"cnr_id":"ComfyUI-Runware","ver":"89412ff4b588adf407736eb84e6fb9853b9a033d","Node name for S&R":"Runware Video Model Search"},"widgets_values":["","xAI","xai:grok-imagine@video (Grok Imagine Video)",false,"Model Default",480,480,false,"1080p"],"bgcolor":"#5345bf"},{"id":12,"type":"LoadImage","pos":[-936.0293716903082,9.23842309768429],"size":[282.798828125,314.00000000000006],"flags":{},"order":0,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[12]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"cnr_id":"comfy-core","ver":"0.19.3","Node name for S&R":"LoadImage"},"widgets_values":[".webp","image"]}],"links":[[12,12,0,11,0,"IMAGE"],[13,11,0,4,7,"RUNWAREVIDEOINPUTSREFERENCEIMAGES"],[14,4,0,13,7,"RUNWAREVIDEOINFERENCEINPUTS"],[15,3,0,13,0,"RUNWAREVIDEOMODEL"],[16,13,0,2,0,"VIDEO"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.5,"offset":[1479.322771320789,489.04543497267605]}},"version":0.4} \ No newline at end of file