diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..efed26e Binary files /dev/null and b/.DS_Store differ diff --git a/clientlibs/utils.js b/clientlibs/utils.js index 5db4e12..db579f4 100644 --- a/clientlibs/utils.js +++ b/clientlibs/utils.js @@ -44,11 +44,15 @@ function captionNodeHandler(msgEvent) { const captionData = msgEvent.detail; const captionText = captionData.captionText; const captionNodeID = parseInt(captionData.nodeID); + if(captionData.success) { const captionNode = app.graph.getNodeById(captionNodeID); - if(captionNode !== null || captionNode !== undefined) { - const captionInput = captionNode.widgets[1].inputEl; - captionInput.value = captionText; + if(captionNode !== null && captionNode !== undefined) { + // Find the imageCaption widget specifically + const imageCaptionWidget = captionNode.widgets.find(widget => widget.name === "imageCaption"); + if(imageCaptionWidget && imageCaptionWidget.inputEl) { + imageCaptionWidget.inputEl.value = captionText; + } } } return false; diff --git a/modules/imageCaptioning.py b/modules/imageCaptioning.py index 931c292..69caca5 100644 --- a/modules/imageCaptioning.py +++ b/modules/imageCaptioning.py @@ -11,20 +11,40 @@ class imageCaptioning: "label_on": "Enabled", "label_off": "Disabled", }), - "Image": ("IMAGE", { - "tooltip": "Specify The Image to be Captioned." + "prompt": ("STRING", { + "multiline": True, + "default": "Describe this image in detail", + "placeholder": "Enter the prompt for image captioning. You can provide multiple prompts separated by newlines.", + "tooltip": "The prompt(s) to guide the image captioning. Multiple prompts can be provided, each on a new line." }), }, "optional": { - "Image Description": ("STRING", { + "inputImages": ("IMAGE", { + "tooltip": "Connect multiple Load Image nodes here to caption multiple images at once." + }), + "inputImage": ("IMAGE", { + "tooltip": "Connect a single Load Image node here for convenience. If inputImages is provided, this will be ignored." + }), + "model": ("STRING", { + "default": "", + "placeholder": "Leave empty for default model (runware:150@1)", + "tooltip": "Optional: Specify the AIR model ID (e.g., runware:150@1, runware:150@2). Leave empty to use the default model." + }), + "template": ("STRING", { + "default": "", + "placeholder": "Optional template for structured output", + "tooltip": "Optional template to structure the caption output." + }), + "imageCaption": ("STRING", { "multiline": True, - "placeholder": "You Don't Have to write Anything here.\nThe Image Description Will Be Generated Automatically.", + "placeholder": "Generated image caption will appear here automatically.", + "tooltip": "This field will be automatically populated with the generated image caption." }), }, "hidden": { "node_id": "UNIQUE_ID" } } - DESCRIPTION = "Image to text, also known as image captioning, allows you to obtain descriptive text prompts based on uploaded or previously generated images. This process is instrumental in generating textual descriptions that can be used to create additional images or provide detailed insights into visual content." + DESCRIPTION = "Advanced image captioning with support for multiple images, custom prompts, and model selection. Generate descriptive text from single or multiple images using customizable prompts and optional AIR models. Perfect for batch processing and structured output generation." FUNCTION = "imageCaptioning" RETURN_TYPES = ("STRING",) @@ -40,16 +60,75 @@ class imageCaptioning: return True def imageCaptioning(self, **kwargs): - image = kwargs.get("Image") - genConfig = [ - { - "taskType": "imageCaption", - "taskUUID": rwUtils.genRandUUID(), - "inputImage": rwUtils.convertTensor2IMG(image) - } - ] + inputImages = kwargs.get("inputImages", None) + inputImage = kwargs.get("inputImage", None) + prompt = kwargs.get("prompt", "Describe this image in detail") + model = kwargs.get("model", "") + template = kwargs.get("template", "") + + # Handle multiple prompts - split by newlines + prompt_list = [p.strip() for p in prompt.split('\n') if p.strip()] + if not prompt_list: + prompt_list = ["Describe this image in detail"] + + # Determine which images to use (following SDK logic) + if inputImages is not None: + images_to_process = inputImages + elif inputImage is not None: + # Single image provided via inputImage - convert to array + images_to_process = [inputImage] + else: + raise Exception("Either inputImages or inputImage must be provided") + + # Set inputImage to inputImages[0] if not already provided + actual_input_image = inputImage + if actual_input_image is None and len(images_to_process) > 0: + actual_input_image = images_to_process[0] + + # Convert images to base64 format + converted_images = [] + for img in images_to_process: + converted_images.append(rwUtils.convertTensor2IMG(img)) + + # Create a dictionary with mandatory parameters + task_params = { + "taskType": "imageCaption", + "taskUUID": rwUtils.genRandUUID(), + } + + # Add either inputImage or inputImages, but not both (API requirement) + if len(converted_images) == 1: + # Single image - use inputImage parameter + task_params["inputImage"] = converted_images[0] + else: + # Multiple images - use inputImages parameter + task_params["inputImages"] = converted_images + # Add model parameter only if specified - backend handles default + if model and model.strip(): + task_params["model"] = model.strip() + + # Add template parameter if specified + if template and template.strip(): + task_params["template"] = template.strip() + # When using template, do NOT include prompt parameter + else: + # Use the provided prompt when no template + task_params["prompt"] = prompt_list + + # Send the task with all applicable parameters + genConfig = [task_params] genResult = rwUtils.inferenecRequest(genConfig) - genText = genResult["data"][0]["text"] + print(genResult) + + # Handle multiple results if multiple images were processed + if len(genResult["data"]) > 1: + # Multiple images - combine results + combined_text = "\n\n".join([item["text"] for item in genResult["data"]]) + genText = combined_text + else: + # Single image + genText = genResult["data"][0]["text"] + rwUtils.sendImageCaption(genText, kwargs.get("node_id")) return (genText, ) \ No newline at end of file diff --git a/modules/utils/runwareUtils.py b/modules/utils/runwareUtils.py index fc03839..ebf3150 100644 --- a/modules/utils/runwareUtils.py +++ b/modules/utils/runwareUtils.py @@ -240,6 +240,7 @@ def sendImageCaption(captionText, nodeID): "success": True, "captionText": captionText, "nodeID": nodeID, + "widgetName": "imageCaption", # Target the new imageCaption widget }, ) diff --git a/workflows/Runware_Image_Inference_Captioning.json b/workflows/Runware_Image_Inference_Captioning.json index d677cd2..6da9e7c 100644 --- a/workflows/Runware_Image_Inference_Captioning.json +++ b/workflows/Runware_Image_Inference_Captioning.json @@ -1 +1 @@ -{"id":"fd6e0d9a-f881-402b-90a5-8abe2c06994c","revision":0,"last_node_id":19,"last_link_id":25,"nodes":[{"id":12,"type":"PreviewImage","pos":[1269.0423583984375,236.072021484375],"size":[515.3893432617188,538.380126953125],"flags":{},"order":4,"mode":0,"inputs":[{"localized_name":"images","name":"images","type":"IMAGE","link":25}],"outputs":[],"properties":{"cnr_id":"comfy-core","ver":"0.3.46","Node name for S&R":"PreviewImage"},"widgets_values":[]},{"id":10,"type":"LoadImage","pos":[241.08689880371094,350.3967590332031],"size":[236.30767822265625,325.9602966308594],"flags":{},"order":0,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","slot_index":0,"links":[12]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"cnr_id":"comfy-core","ver":"0.3.46","Node name for S&R":"LoadImage"},"widgets_values":["car.jpg","image"]},{"id":13,"type":"Runware Image Caption","pos":[508.9840393066406,446.2952575683594],"size":[278.0941162109375,190.92190551757812],"flags":{},"order":2,"mode":0,"inputs":[{"localized_name":"Image","name":"Image","type":"IMAGE","link":12},{"localized_name":"Always Recaption","name":"Always Recaption","type":"BOOLEAN","widget":{"name":"Always Recaption"},"link":null},{"localized_name":"Image Description","name":"Image Description","shape":7,"type":"STRING","widget":{"name":"Image Description"},"link":null}],"outputs":[{"localized_name":"IMAGE PROMPT","name":"IMAGE PROMPT","type":"STRING","slot_index":0,"links":[23]}],"properties":{"cnr_id":"ComfyUI-Runware","ver":"de0a4c2e258a1f3a104d0f9994f9b7ea66e56736","Node name for S&R":"Runware Image Caption"},"widgets_values":[true,"car driving down a road next to the ocean, still from a 2015 pixar movie, kauai, panoramic, :: nixri, hills and mountains, hyperrealistic matte painting, connectivity, anamorphic cinematography, thumbnail, scenery artwork, imagery"],"bgcolor":"#6c5ce7"},{"id":14,"type":"Runware Model Search","pos":[497.6798095703125,175.9790802001953],"size":[315,154],"flags":{"collapsed":false},"order":1,"mode":0,"inputs":[{"localized_name":"Model Search","name":"Model Search","type":"STRING","widget":{"name":"Model Search"},"link":null},{"localized_name":"Model Architecture","name":"Model Architecture","type":"COMBO","widget":{"name":"Model Architecture"},"link":null},{"localized_name":"ModelType","name":"ModelType","type":"COMBO","widget":{"name":"ModelType"},"link":null},{"localized_name":"ModelList","name":"ModelList","type":"COMBO","widget":{"name":"ModelList"},"link":null},{"localized_name":"Use Search Value","name":"Use Search Value","type":"BOOLEAN","widget":{"name":"Use Search Value"},"link":null}],"outputs":[{"localized_name":"Runware Model","name":"Runware Model","type":"RUNWAREMODEL","slot_index":0,"links":[24]}],"properties":{"cnr_id":"ComfyUI-Runware","ver":"de0a4c2e258a1f3a104d0f9994f9b7ea66e56736","Node name for S&R":"Runware Model Search"},"widgets_values":["","All","Base Model","runware:100@1 (Flux Schnell)",false],"bgcolor":"#6c5ce7"},{"id":19,"type":"Runware Image Inference","pos":[836.716552734375,172.55825805664062],"size":[400,862],"flags":{},"order":3,"mode":0,"inputs":[{"localized_name":"Model","name":"Model","type":"RUNWAREMODEL","link":24},{"localized_name":"Accelerator","name":"Accelerator","shape":7,"type":"RUNWAREACCELERATOR","link":null},{"localized_name":"Lora","name":"Lora","shape":7,"type":"RUNWARELORA","link":null},{"localized_name":"Outpainting","name":"Outpainting","shape":7,"type":"RUNWAREOUTPAINT","link":null},{"localized_name":"IPAdapters","name":"IPAdapters","shape":7,"type":"RUNWAREIPAdapter","link":null},{"localized_name":"ControlNet","name":"ControlNet","shape":7,"type":"RUNWARECONTROLNET","link":null},{"localized_name":"Refiner","name":"Refiner","shape":7,"type":"RUNWAREREFINER","link":null},{"localized_name":"seedImage","name":"seedImage","shape":7,"type":"IMAGE","link":null},{"localized_name":"maskImage","name":"maskImage","shape":7,"type":"MASK","link":null},{"localized_name":"Embeddings","name":"Embeddings","shape":7,"type":"RUNWAREEMBEDDING","link":null},{"localized_name":"VAE","name":"VAE","shape":7,"type":"RUNWAREVAE","link":null},{"localized_name":"referenceImages","name":"referenceImages","shape":7,"type":"RUNWAREREFERENCEIMAGES","link":null},{"localized_name":"providerSettings","name":"providerSettings","shape":7,"type":"RUNWAREPROVIDERSETTINGS","link":null},{"localized_name":"positivePrompt","name":"positivePrompt","type":"STRING","widget":{"name":"positivePrompt"},"link":23},{"localized_name":"negativePrompt","name":"negativePrompt","type":"STRING","widget":{"name":"negativePrompt"},"link":null},{"localized_name":"Multi Inference Mode","name":"Multi Inference Mode","type":"BOOLEAN","widget":{"name":"Multi Inference Mode"},"link":null},{"localized_name":"Prompt Weighting","name":"Prompt Weighting","type":"COMBO","widget":{"name":"Prompt Weighting"},"link":null},{"localized_name":"dimensions","name":"dimensions","type":"COMBO","widget":{"name":"dimensions"},"link":null},{"localized_name":"width","name":"width","type":"INT","widget":{"name":"width"},"link":null},{"localized_name":"height","name":"height","type":"INT","widget":{"name":"height"},"link":null},{"localized_name":"useSteps","name":"useSteps","type":"BOOLEAN","widget":{"name":"useSteps"},"link":null},{"localized_name":"steps","name":"steps","type":"INT","widget":{"name":"steps"},"link":null},{"localized_name":"useScheduler","name":"useScheduler","type":"BOOLEAN","widget":{"name":"useScheduler"},"link":null},{"localized_name":"scheduler","name":"scheduler","type":"COMBO","widget":{"name":"scheduler"},"link":null},{"localized_name":"useCFGScale","name":"useCFGScale","type":"BOOLEAN","widget":{"name":"useCFGScale"},"link":null},{"localized_name":"cfgScale","name":"cfgScale","type":"FLOAT","widget":{"name":"cfgScale"},"link":null},{"localized_name":"useSeed","name":"useSeed","type":"BOOLEAN","widget":{"name":"useSeed"},"link":null},{"localized_name":"seed","name":"seed","type":"INT","widget":{"name":"seed"},"link":null},{"localized_name":"useClipSkip","name":"useClipSkip","type":"BOOLEAN","widget":{"name":"useClipSkip"},"link":null},{"localized_name":"clipSkip","name":"clipSkip","type":"INT","widget":{"name":"clipSkip"},"link":null},{"localized_name":"strength","name":"strength","type":"FLOAT","widget":{"name":"strength"},"link":null},{"localized_name":"Mask Margin","name":"Mask Margin","type":"BOOLEAN","widget":{"name":"Mask Margin"},"link":null},{"localized_name":"maskMargin","name":"maskMargin","type":"INT","widget":{"name":"maskMargin"},"link":null},{"localized_name":"batchSize","name":"batchSize","type":"INT","widget":{"name":"batchSize"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[25]},{"localized_name":"RW-Task","name":"RW-Task","type":"RUNWARETASK","links":null}],"properties":{"cnr_id":"ComfyUI-Runware","ver":"5e5e2b3fac2c984692b057e4e607d778a8c73e40","Node name for S&R":"Runware Image Inference"},"widgets_values":["","",false,"Disabled","Square (512x512)",512,512,true,4,true,"Default",true,6.5,true,64635316412137,"randomize",true,0,0.8,false,32,1],"bgcolor":"#5345bf"}],"links":[[12,10,0,13,0,"IMAGE"],[23,13,0,19,13,"STRING"],[24,14,0,19,0,"RUNWAREMODEL"],[25,19,0,12,0,"IMAGE"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.6070968866929469,"offset":[220.1318740324629,16.745718810171816]}},"version":0.4} \ No newline at end of file +{"id":"fd6e0d9a-f881-402b-90a5-8abe2c06994c","revision":0,"last_node_id":22,"last_link_id":33,"nodes":[{"id":12,"type":"PreviewImage","pos":[1269.0423583984375,236.072021484375],"size":[515.3893432617188,538.380126953125],"flags":{},"order":4,"mode":0,"inputs":[{"localized_name":"images","name":"images","type":"IMAGE","link":25}],"outputs":[],"properties":{"cnr_id":"comfy-core","ver":"0.3.46","Node name for S&R":"PreviewImage"},"widgets_values":[]},{"id":14,"type":"Runware Model Search","pos":[497.6798095703125,175.9790802001953],"size":[315,154],"flags":{"collapsed":false},"order":0,"mode":0,"inputs":[{"localized_name":"Model Search","name":"Model Search","type":"STRING","widget":{"name":"Model Search"},"link":null},{"localized_name":"Model Architecture","name":"Model Architecture","type":"COMBO","widget":{"name":"Model Architecture"},"link":null},{"localized_name":"ModelType","name":"ModelType","type":"COMBO","widget":{"name":"ModelType"},"link":null},{"localized_name":"ModelList","name":"ModelList","type":"COMBO","widget":{"name":"ModelList"},"link":null},{"localized_name":"Use Search Value","name":"Use Search Value","type":"BOOLEAN","widget":{"name":"Use Search Value"},"link":null}],"outputs":[{"localized_name":"Runware Model","name":"Runware Model","type":"RUNWAREMODEL","slot_index":0,"links":[24]}],"properties":{"cnr_id":"ComfyUI-Runware","ver":"de0a4c2e258a1f3a104d0f9994f9b7ea66e56736","Node name for S&R":"Runware Model Search"},"widgets_values":["","All","Base Model","runware:100@1 (Flux Schnell)",false],"bgcolor":"#6c5ce7"},{"id":19,"type":"Runware Image Inference","pos":[836.716552734375,172.55825805664062],"size":[400,862],"flags":{},"order":3,"mode":0,"inputs":[{"localized_name":"Model","name":"Model","type":"RUNWAREMODEL","link":24},{"localized_name":"Accelerator","name":"Accelerator","shape":7,"type":"RUNWAREACCELERATOR","link":null},{"localized_name":"Lora","name":"Lora","shape":7,"type":"RUNWARELORA","link":null},{"localized_name":"Outpainting","name":"Outpainting","shape":7,"type":"RUNWAREOUTPAINT","link":null},{"localized_name":"IPAdapters","name":"IPAdapters","shape":7,"type":"RUNWAREIPAdapter","link":null},{"localized_name":"ControlNet","name":"ControlNet","shape":7,"type":"RUNWARECONTROLNET","link":null},{"localized_name":"Refiner","name":"Refiner","shape":7,"type":"RUNWAREREFINER","link":null},{"localized_name":"seedImage","name":"seedImage","shape":7,"type":"IMAGE","link":null},{"localized_name":"maskImage","name":"maskImage","shape":7,"type":"MASK","link":null},{"localized_name":"Embeddings","name":"Embeddings","shape":7,"type":"RUNWAREEMBEDDING","link":null},{"localized_name":"VAE","name":"VAE","shape":7,"type":"RUNWAREVAE","link":null},{"localized_name":"referenceImages","name":"referenceImages","shape":7,"type":"RUNWAREREFERENCEIMAGES","link":null},{"localized_name":"providerSettings","name":"providerSettings","shape":7,"type":"RUNWAREPROVIDERSETTINGS","link":null},{"localized_name":"positivePrompt","name":"positivePrompt","type":"STRING","widget":{"name":"positivePrompt"},"link":33},{"localized_name":"negativePrompt","name":"negativePrompt","type":"STRING","widget":{"name":"negativePrompt"},"link":null},{"localized_name":"Multi Inference Mode","name":"Multi Inference Mode","type":"BOOLEAN","widget":{"name":"Multi Inference Mode"},"link":null},{"localized_name":"Prompt Weighting","name":"Prompt Weighting","type":"COMBO","widget":{"name":"Prompt Weighting"},"link":null},{"localized_name":"dimensions","name":"dimensions","type":"COMBO","widget":{"name":"dimensions"},"link":null},{"localized_name":"width","name":"width","type":"INT","widget":{"name":"width"},"link":null},{"localized_name":"height","name":"height","type":"INT","widget":{"name":"height"},"link":null},{"localized_name":"useSteps","name":"useSteps","type":"BOOLEAN","widget":{"name":"useSteps"},"link":null},{"localized_name":"steps","name":"steps","type":"INT","widget":{"name":"steps"},"link":null},{"localized_name":"useScheduler","name":"useScheduler","type":"BOOLEAN","widget":{"name":"useScheduler"},"link":null},{"localized_name":"scheduler","name":"scheduler","type":"COMBO","widget":{"name":"scheduler"},"link":null},{"localized_name":"useCFGScale","name":"useCFGScale","type":"BOOLEAN","widget":{"name":"useCFGScale"},"link":null},{"localized_name":"cfgScale","name":"cfgScale","type":"FLOAT","widget":{"name":"cfgScale"},"link":null},{"localized_name":"useSeed","name":"useSeed","type":"BOOLEAN","widget":{"name":"useSeed"},"link":null},{"localized_name":"seed","name":"seed","type":"INT","widget":{"name":"seed"},"link":null},{"localized_name":"useClipSkip","name":"useClipSkip","type":"BOOLEAN","widget":{"name":"useClipSkip"},"link":null},{"localized_name":"clipSkip","name":"clipSkip","type":"INT","widget":{"name":"clipSkip"},"link":null},{"localized_name":"strength","name":"strength","type":"FLOAT","widget":{"name":"strength"},"link":null},{"localized_name":"Mask Margin","name":"Mask Margin","type":"BOOLEAN","widget":{"name":"Mask Margin"},"link":null},{"localized_name":"maskMargin","name":"maskMargin","type":"INT","widget":{"name":"maskMargin"},"link":null},{"localized_name":"batchSize","name":"batchSize","type":"INT","widget":{"name":"batchSize"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[25]},{"localized_name":"RW-Task","name":"RW-Task","type":"RUNWARETASK","links":null}],"properties":{"cnr_id":"ComfyUI-Runware","ver":"5e5e2b3fac2c984692b057e4e607d778a8c73e40","Node name for S&R":"Runware Image Inference"},"widgets_values":["","",false,"Disabled","Square (512x512)",512,512,true,4,true,"Default",true,6.5,true,1042637731176015,"randomize",true,0,0.8,false,32,1],"bgcolor":"#5345bf"},{"id":10,"type":"LoadImage","pos":[158.67625427246094,332.9147644042969],"size":[236.30767822265625,325.9602966308594],"flags":{},"order":1,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","slot_index":0,"links":[32]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"cnr_id":"comfy-core","ver":"0.3.46","Node name for S&R":"LoadImage"},"widgets_values":["car.jpg","image"]},{"id":22,"type":"Runware Image Caption","pos":[423.45361328125,454.9598693847656],"size":[400,234],"flags":{},"order":2,"mode":0,"inputs":[{"localized_name":"inputImages","name":"inputImages","shape":7,"type":"IMAGE","link":null},{"localized_name":"inputImage","name":"inputImage","shape":7,"type":"IMAGE","link":32},{"localized_name":"Always Recaption","name":"Always Recaption","type":"BOOLEAN","widget":{"name":"Always Recaption"},"link":null},{"localized_name":"prompt","name":"prompt","type":"STRING","widget":{"name":"prompt"},"link":null},{"localized_name":"model","name":"model","shape":7,"type":"STRING","widget":{"name":"model"},"link":null},{"localized_name":"template","name":"template","shape":7,"type":"STRING","widget":{"name":"template"},"link":null},{"localized_name":"Image Description","name":"Image Description","shape":7,"type":"STRING","widget":{"name":"Image Description"},"link":null}],"outputs":[{"localized_name":"IMAGE PROMPT","name":"IMAGE PROMPT","type":"STRING","links":[33]}],"properties":{"cnr_id":"ComfyUI-Runware","ver":"c2e04ddc9a206c519399f21c3d86bc49d1f7e33c","Node name for S&R":"Runware Image Caption"},"widgets_values":[false,"Describe this image in detail","","",""],"bgcolor":"#5345bf"}],"links":[[24,14,0,19,0,"RUNWAREMODEL"],[25,19,0,12,0,"IMAGE"],[32,10,0,22,1,"IMAGE"],[33,22,0,19,13,"STRING"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.6070968866929469,"offset":[237.94204607748716,-64.64187735944965]}},"version":0.4} \ No newline at end of file