Merge pull request #5 from JosephThomasParker/feature/output_json

Add pipeline version

CHANGELOG.md

@@ -1,3 +1,7 @@
+0.3
+
+* Added img2img node 
+
 0.2 
 
 * Added textbox to change model (must match downloaded model) and dropdown list to choose sampler

README.md

@@ -4,9 +4,9 @@ These nodes provide a wrapper for calling [Draw Things](https://drawthings.ai/)
 
 **Wait, why?** The Draw Things app has been optimized for Apple hardware and runs roughly x3 faster than ComfyUI generations. But ComfyUI is a flexible and powerful tools, and has some features - like queuing and face swapping - that haven't been implemented in Draw Things.
 
-This simple node calls a local instance of Draw Things through its API and returns the resulting image to ComfyUI.
+These simple nodes for txt2img and img2img call a local instance of Draw Things through its API and return the resulting image to ComfyUI.
 
-![](basic_example.png)
+![](examples/basic_example.png)
 
 # Set up
 
@@ -21,9 +21,38 @@ cd ComfyUI-DrawThingsWrapper
 pip install -r requirements.txt
 ```
 
+# Usage
+
+## Basic usage
+
+For basic usage, use the Txt2Img or Img2Img nodes as in the picture above. Other options like lora or controlnets can be used by setting these directly in the Draw Things app. Values set in Draw Things will be respected, unless overridden by values set in these nodes.
+
+## Advanced usage
+
+For more control over option setting from within ComfyUI, the **pipeline** mode can be used. In this mode, the nodes **Txt2Img Pipeline** or **Img2Img Pipeline** create the settings ("pipeline") for image generation, but unlike their basic counterparts, do not call the image generation. The image generation is done by the **Generate from Pipeline** mode, which takes a pipeline and calls the Draw Things API. Between these steps, the pipeline can be modified to set advanced options, and add lora and controlnets.
+
+![](examples/advanced_example.png)
+
 # Limitations
 
-This node treats Draw Things as a black box, and can only change the settings available through the [Draw Things API](https://docs.drawthings.ai/documentation/documentation/8.scripts). Settings changed through the API automatically change the values in the Draw Things UI. Other settings can be set manually inside the UI. These manual settings _do_ take effect for jobs initiated by ComfyUI, but are _not_ recorded by ComfyUI, so such ComfyUI workflows are not reproducible (without also ensuring the Draw Things settings are the same). This also means that if changes are made _only_ in the Draw Things UI, ComfyUI will not recognise that the workflow is different, and will _not_ reexecute the run.
+**Model names** must be given to the nodes exactly as they appear in the Draw Things filename. The easiest way to ensure the file names are correct is to select the desired model in Draw Things, and then click the "..." at the top by "Settings", and select "Copy configuration". Paste that text into a text editor, and the correct model and lora names will appear in the configuration. 
+
+**Inpainting** cannot be called through these nodes.
+
+**Controlnets, PuLID** and similar _do_ work, but image controls (e.g. openpose, reference images) must be set manually in the Draw Things app.
+
+For **reproducibility** both the ComfyUI settings **and** the manually-set Draw Things settings must be the same
+
+**Run triggering**. ComfyUI only triggers runs if settings in ComfyUI have changed. Changing settings only in the Draw Things app will not trigger changes.
+
+# Implementation details
+
+This node basically does two things: it makes a python dict of Draw Things configuration parameters, and passes that configuration to the [Draw Things API](https://docs.drawthings.ai/documentation/documentation/8.scripts) for execution.
+
+The basic nodes do both these steps. The pipeline nodes split this into two steps, allowing for the python dict to be edited in between.
+
+The functionality of these nodes is limited by what can be passed through the API. As I understand it, it's not possible to pass masks or control images through the API. This is what prevents inpainting from working and means that control images must be set manually in Draw Things.
+
 
 # Disclaimer
 

__init__.py

@@ -1,3 +1,3 @@
 from .nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS
 
-__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"]
+__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"]

examples/advanced_workflow.json

@@ -0,0 +1,287 @@
+{
+  "last_node_id": 10,
+  "last_link_id": 8,
+  "nodes": [
+    {
+      "id": 9,
+      "type": "DrawThingsGenerateFromPipeline",
+      "pos": {
+        "0": 491,
+        "1": 275
+      },
+      "size": [
+        285.6000061035156,
+        26
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "pipeline",
+          "type": "dict",
+          "link": 7
+        }
+      ],
+      "outputs": [
+        {
+          "name": "generated_image",
+          "type": "IMAGE",
+          "links": [
+            8
+          ],
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "DrawThingsGenerateFromPipeline"
+      }
+    },
+    {
+      "id": 10,
+      "type": "PreviewImage",
+      "pos": {
+        "0": 551,
+        "1": 496
+      },
+      "size": [
+        210,
+        246
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "images",
+          "type": "IMAGE",
+          "link": 8
+        }
+      ],
+      "outputs": [],
+      "properties": {
+        "Node name for S&R": "PreviewImage"
+      }
+    },
+    {
+      "id": 6,
+      "type": "DrawThingsPipelineAddLora",
+      "pos": {
+        "0": 55,
+        "1": 275
+      },
+      "size": {
+        "0": 365.4000244140625,
+        "1": 82
+      },
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "pipeline",
+          "type": "dict",
+          "link": 4
+        }
+      ],
+      "outputs": [
+        {
+          "name": "pipeline",
+          "type": "dict",
+          "links": [
+            5
+          ],
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "DrawThingsPipelineAddLora"
+      },
+      "widgets_values": [
+        "flux.1__dev__to__schnell__4_step_lora_f16.ckpt",
+        1
+      ]
+    },
+    {
+      "id": 7,
+      "type": "DrawThingsPipelineAddCustom",
+      "pos": {
+        "0": 58,
+        "1": 437
+      },
+      "size": [
+        376.08315404579366,
+        106
+      ],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "pipeline",
+          "type": "dict",
+          "link": 5
+        }
+      ],
+      "outputs": [
+        {
+          "name": "pipeline",
+          "type": "dict",
+          "links": [
+            6
+          ],
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "DrawThingsPipelineAddCustom"
+      },
+      "widgets_values": [
+        "shift",
+        "FLOAT",
+        "4.0"
+      ]
+    },
+    {
+      "id": 8,
+      "type": "DrawThingsPipelineAddControl",
+      "pos": {
+        "0": 50,
+        "1": 634
+      },
+      "size": {
+        "0": 403.1999816894531,
+        "1": 274
+      },
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "pipeline",
+          "type": "dict",
+          "link": 6
+        }
+      ],
+      "outputs": [
+        {
+          "name": "pipeline",
+          "type": "dict",
+          "links": [
+            7
+          ],
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "DrawThingsPipelineAddControl"
+      },
+      "widgets_values": [
+        "pulid_0.9_eva02_clip_l14_336_f16.ckpt",
+        1,
+        0,
+        0.5,
+        false,
+        false,
+        0,
+        "balanced",
+        "",
+        ""
+      ]
+    },
+    {
+      "id": 5,
+      "type": "DrawThingsTxt2ImgPipeline",
+      "pos": {
+        "0": -369,
+        "1": 276
+      },
+      "size": {
+        "0": 352.79998779296875,
+        "1": 250
+      },
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "pipeline",
+          "type": "dict",
+          "links": [
+            4
+          ],
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "DrawThingsTxt2ImgPipeline"
+      },
+      "widgets_values": [
+        "flux_1_dev_q8p.ckpt",
+        "A face in the clouds",
+        240,
+        "randomize",
+        512,
+        512,
+        3.5,
+        "Euler A Trailing",
+        4
+      ]
+    }
+  ],
+  "links": [
+    [
+      4,
+      5,
+      0,
+      6,
+      0,
+      "dict"
+    ],
+    [
+      5,
+      6,
+      0,
+      7,
+      0,
+      "dict"
+    ],
+    [
+      6,
+      7,
+      0,
+      8,
+      0,
+      "dict"
+    ],
+    [
+      7,
+      8,
+      0,
+      9,
+      0,
+      "dict"
+    ],
+    [
+      8,
+      9,
+      0,
+      10,
+      0,
+      "IMAGE"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {
+    "ds": {
+      "scale": 0.9229599817706451,
+      "offset": [
+        493.04065648453127,
+        -149.20627556050917
+      ]
+    }
+  },
+  "version": 0.4
+}

examples/basic_workflow.json

@@ -0,0 +1,184 @@
+{
+  "last_node_id": 4,
+  "last_link_id": 3,
+  "nodes": [
+    {
+      "id": 2,
+      "type": "PreviewImage",
+      "pos": {
+        "0": 401,
+        "1": 700
+      },
+      "size": {
+        "0": 210,
+        "1": 26
+      },
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "images",
+          "type": "IMAGE",
+          "link": 1
+        }
+      ],
+      "outputs": [],
+      "properties": {
+        "Node name for S&R": "PreviewImage"
+      },
+      "widgets_values": []
+    },
+    {
+      "id": 1,
+      "type": "DrawThingsTxt2Img",
+      "pos": {
+        "0": -8.88882064819336,
+        "1": 700.3402099609375
+      },
+      "size": {
+        "0": 315,
+        "1": 250
+      },
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "generated_image",
+          "type": "IMAGE",
+          "links": [
+            1,
+            2
+          ],
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "DrawThingsTxt2Img"
+      },
+      "widgets_values": [
+        "flux_1_dev_q8p.ckpt",
+        "The Cloud of Unknowing",
+        42,
+        "randomize",
+        512,
+        512,
+        3.5,
+        "Euler A Trailing",
+        20
+      ]
+    },
+    {
+      "id": 4,
+      "type": "PreviewImage",
+      "pos": {
+        "0": 410,
+        "1": 1019
+      },
+      "size": {
+        "0": 210,
+        "1": 26
+      },
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "images",
+          "type": "IMAGE",
+          "link": 3
+        }
+      ],
+      "outputs": [],
+      "properties": {
+        "Node name for S&R": "PreviewImage"
+      },
+      "widgets_values": []
+    },
+    {
+      "id": 3,
+      "type": "DrawThingsImg2Img",
+      "pos": {
+        "0": -6,
+        "1": 1015
+      },
+      "size": {
+        "0": 315,
+        "1": 226
+      },
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "images",
+          "type": "IMAGE",
+          "link": 2
+        }
+      ],
+      "outputs": [
+        {
+          "name": "generated_image",
+          "type": "IMAGE",
+          "links": [
+            3
+          ],
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "DrawThingsImg2Img"
+      },
+      "widgets_values": [
+        "flux_1_dev_q8p.ckpt",
+        "A black cloud with lightening",
+        43,
+        "randomize",
+        3.5,
+        "Euler A Trailing",
+        20,
+        0.9
+      ]
+    }
+  ],
+  "links": [
+    [
+      1,
+      1,
+      0,
+      2,
+      0,
+      "IMAGE"
+    ],
+    [
+      2,
+      1,
+      0,
+      3,
+      0,
+      "IMAGE"
+    ],
+    [
+      3,
+      3,
+      0,
+      4,
+      0,
+      "IMAGE"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {
+    "ds": {
+      "scale": 0.8390545288824038,
+      "offset": [
+        545.9536592330822,
+        -550.971947599925
+      ]
+    }
+  },
+  "version": 0.4
+}

nodes.py

@@ -8,6 +8,7 @@ import numpy as np
 import requests
 from PIL import Image
 import io
+from io import BytesIO
 import torch
 
 
@@ -27,7 +28,28 @@ class DrawThingsTxt2Img:
                 "width": ("INT", {"default": 512}),
                 "height": ("INT", {"default": 512}),
                 "guidance_scale": ("FLOAT", {"default": 3.5}),
-                "sampler": (["UniPC","DPM++ 2M Karras","Euler Ancestral", "DPM++ SDE Karras", "PLMS", "DDIM", "LCM", "Euler A Substep", "DPM++ SDE Substep", "TCD", "DPM++ 2M Trailing", "Euler A Trailing", "DPM++ SDE Trailing", "DDIM Trailing", "DPM++ 2M AYS", "Euler A AYS", "DPM++ SDE AYS"], {"default": "Euler A Trailing"}),
+                "sampler": (
+                    [
+                        "UniPC",
+                        "DPM++ 2M Karras",
+                        "Euler Ancestral",
+                        "DPM++ SDE Karras",
+                        "PLMS",
+                        "DDIM",
+                        "LCM",
+                        "Euler A Substep",
+                        "DPM++ SDE Substep",
+                        "TCD",
+                        "DPM++ 2M Trailing",
+                        "Euler A Trailing",
+                        "DPM++ SDE Trailing",
+                        "DDIM Trailing",
+                        "DPM++ 2M AYS",
+                        "Euler A AYS",
+                        "DPM++ SDE AYS",
+                    ],
+                    {"default": "Euler A Trailing"},
+                ),
                 "steps": ("INT", {"default": 20}),
             }
         }
@@ -36,7 +58,9 @@ class DrawThingsTxt2Img:
     RETURN_NAMES = ("generated_image",)
     FUNCTION = "generate_image"
 
-    def generate_image(self, model, prompt, seed, width, height, guidance_scale, sampler, steps):
+    def generate_image(
+        self, model, prompt, seed, width, height, guidance_scale, sampler, steps
+    ):
         # Call the Draw Things API
         api_url = "http://127.0.0.1:7860/sdapi/v1/txt2img"
 
@@ -72,6 +96,555 @@ class DrawThingsTxt2Img:
         return (torch.stack(images),)
 
 
-NODE_CLASS_MAPPINGS = {"DrawThingsTxt2Img": DrawThingsTxt2Img}
+def image_to_base64(image_tensor):
+    # Convert the image tensor to a NumPy array and scale it to the range 0-255
+    i = 255.0 * image_tensor.cpu().numpy()
+    img = Image.fromarray(np.clip(i, 0, 255).astype(np.uint8))
 
-NODE_DISPLAY_NAME_MAPPINGS = {"DrawThingsTxt2Img": "Draw Things Txt2Img"}
+    # Save the image to a BytesIO object (in memory) rather than to a file
+    buffered = BytesIO()
+    img.save(buffered, format="PNG")
+
+    # Encode the image as base64
+    encoded_string = base64.b64encode(buffered.getvalue()).decode("utf-8")
+    return encoded_string
+
+
+def resize_for_inpainting(pixels, mask=None):
+    x = (pixels.shape[1] // 64) * 64
+    y = (pixels.shape[2] // 64) * 64
+    # mask = torch.nn.functional.interpolate(mask.reshape((-1, 1, mask.shape[-2], mask.shape[-1])), size=(pixels.shape[1], pixels.shape[2]), mode="bilinear")
+
+    orig_pixels = pixels
+    pixels = orig_pixels.clone()
+    if pixels.shape[1] != x or pixels.shape[2] != y:
+        x_offset = (pixels.shape[1] % 64) // 2
+        y_offset = (pixels.shape[2] % 64) // 2
+        pixels = pixels[:, x_offset : x + x_offset, y_offset : y + y_offset, :]
+        # pixels = pixels[:,x_offset:x + x_offset, y_offset:y + y_offset]
+        # mask = mask[:,:,x_offset:x + x_offset, y_offset:y + y_offset]
+
+    # m = (1.0 - mask.round()).squeeze(1)
+    # for i in range(3):
+    #    pixels[:,:,:,i] -= 0.5
+    #    pixels[:,:,:,i] *= m
+    #    pixels[:,:,:,i] += 0.5
+    return pixels
+
+
+def get_image_size(pixels):
+    """
+    Get image size from a size image, i.e. assumed input size is [H, W, C]
+    """
+    x = (pixels.shape[0] // 64) * 64
+    y = (pixels.shape[1] // 64) * 64
+    return x, y
+
+
+class DrawThingsImg2Img:
+    def __init__(self):
+        pass
+
+    CATEGORY = "DrawThingsWrapper"
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "images": ("IMAGE", {"tooltip": "input image"}),
+                "model": ("STRING", {"default": "flux_1_dev_q8p.ckpt"}),
+                "prompt": ("STRING", {"default": ""}),
+                "seed": ("INT", {"default": 42}),
+                "guidance_scale": (
+                    "FLOAT",
+                    {"default": 3.5, "min": 0, "max": 25, "step": 0.1},
+                ),
+                "sampler": (
+                    [
+                        "UniPC",
+                        "DPM++ 2M Karras",
+                        "Euler Ancestral",
+                        "DPM++ SDE Karras",
+                        "PLMS",
+                        "DDIM",
+                        "LCM",
+                        "Euler A Substep",
+                        "DPM++ SDE Substep",
+                        "TCD",
+                        "DPM++ 2M Trailing",
+                        "Euler A Trailing",
+                        "DPM++ SDE Trailing",
+                        "DDIM Trailing",
+                        "DPM++ 2M AYS",
+                        "Euler A AYS",
+                        "DPM++ SDE AYS",
+                    ],
+                    {"default": "Euler A Trailing"},
+                ),
+                "steps": ("INT", {"default": 20, "min": 1, "max": 150, "step": 1}),
+                "denoise": (
+                    "FLOAT",
+                    {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01},
+                ),
+            }
+        }
+
+    RETURN_TYPES = ("IMAGE",)
+    RETURN_NAMES = ("generated_image",)
+    FUNCTION = "generate_image"
+
+    def generate_image(
+        self, images, model, prompt, seed, guidance_scale, sampler, steps, denoise
+    ):
+        # Call the Draw Things API
+        api_url = "http://127.0.0.1:7860/sdapi/v1/img2img"
+
+        encoded_images = []
+        images_resized = resize_for_inpainting(images)
+        for image_tensor in images_resized:
+            encoded_images.append(image_to_base64(image_tensor))
+
+        height, width = get_image_size(images_resized[0])
+
+        payload = {
+            "model": model,
+            "prompt": prompt,
+            "seed": seed,
+            "width": width,
+            "height": height,
+            "guidance_scale": guidance_scale,
+            "sampler": sampler,
+            "steps": steps,
+            "init_images": encoded_images,
+            "strength": denoise,
+        }
+
+        response = requests.post(api_url, json=payload)
+
+        # Raise an error if the request failed
+        response.raise_for_status()
+
+        # Parse the JSON response
+        data = response.json()
+
+        # Process the images (assuming they are base64 encoded or raw binary data)
+        images = []
+        for img_data in data["images"]:
+            image_bytes = base64.b64decode(img_data)
+            # Convert the image data to a Pillow Image object
+            image = Image.open(io.BytesIO(image_bytes))
+            image_np = np.array(image)
+            # Convert to float32 tensor and normalize
+            tensor_image = torch.from_numpy(image_np.astype(np.float32) / 255.0)
+            images.append(tensor_image)
+        return (torch.stack(images),)
+
+
+class DrawThingsTxt2ImgPipeline:
+    def __init__(self):
+        pass
+
+    CATEGORY = "DrawThingsWrapper"
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "model": ("STRING", {"default": "flux_1_dev_q8p.ckpt"}),
+                "prompt": ("STRING", {"default": ""}),
+                "seed": ("INT", {"default": 42}),
+                "width": ("INT", {"default": 512}),
+                "height": ("INT", {"default": 512}),
+                "guidance_scale": ("FLOAT", {"default": 3.5}),
+                "sampler": (
+                    [
+                        "UniPC",
+                        "DPM++ 2M Karras",
+                        "Euler Ancestral",
+                        "DPM++ SDE Karras",
+                        "PLMS",
+                        "DDIM",
+                        "LCM",
+                        "Euler A Substep",
+                        "DPM++ SDE Substep",
+                        "TCD",
+                        "DPM++ 2M Trailing",
+                        "Euler A Trailing",
+                        "DPM++ SDE Trailing",
+                        "DDIM Trailing",
+                        "DPM++ 2M AYS",
+                        "Euler A AYS",
+                        "DPM++ SDE AYS",
+                    ],
+                    {"default": "Euler A Trailing"},
+                ),
+                "steps": ("INT", {"default": 20}),
+            }
+        }
+
+    RETURN_TYPES = ("dict",)
+    RETURN_NAMES = ("pipeline",)
+    FUNCTION = "generate_pipeline"
+
+    def generate_pipeline(
+        self, model, prompt, seed, width, height, guidance_scale, sampler, steps
+    ):
+
+        payload = {
+            "generation_mode": "txt2img",
+            "model": model,
+            "prompt": prompt,
+            "seed": seed,
+            "width": width,
+            "height": height,
+            "guidance_scale": guidance_scale,
+            "sampler": sampler,
+            "steps": steps,
+        }
+
+        return (payload,)
+
+
+class DrawThingsImg2ImgPipeline:
+    def __init__(self):
+        pass
+
+    CATEGORY = "DrawThingsWrapper"
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "images": ("IMAGE", {"tooltip": "input image"}),
+                "model": ("STRING", {"default": "flux_1_dev_q8p.ckpt"}),
+                "prompt": ("STRING", {"default": ""}),
+                "seed": ("INT", {"default": 42}),
+                "guidance_scale": (
+                    "FLOAT",
+                    {"default": 3.5, "min": 0, "max": 25, "step": 0.1},
+                ),
+                "sampler": (
+                    [
+                        "UniPC",
+                        "DPM++ 2M Karras",
+                        "Euler Ancestral",
+                        "DPM++ SDE Karras",
+                        "PLMS",
+                        "DDIM",
+                        "LCM",
+                        "Euler A Substep",
+                        "DPM++ SDE Substep",
+                        "TCD",
+                        "DPM++ 2M Trailing",
+                        "Euler A Trailing",
+                        "DPM++ SDE Trailing",
+                        "DDIM Trailing",
+                        "DPM++ 2M AYS",
+                        "Euler A AYS",
+                        "DPM++ SDE AYS",
+                    ],
+                    {"default": "Euler A Trailing"},
+                ),
+                "steps": ("INT", {"default": 20, "min": 1, "max": 150, "step": 1}),
+                "denoise": (
+                    "FLOAT",
+                    {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01},
+                ),
+            }
+        }
+
+    RETURN_TYPES = ("dict",)
+    RETURN_NAMES = ("pipeline",)
+    FUNCTION = "generate_pipeline"
+
+    def generate_pipeline(
+        self, images, model, prompt, seed, guidance_scale, sampler, steps, denoise
+    ):
+
+        encoded_images = []
+        images_resized = resize_for_inpainting(images)
+        for image_tensor in images_resized:
+            encoded_images.append(image_to_base64(image_tensor))
+
+        height, width = get_image_size(images_resized[0])
+
+        payload = {
+            "generation_mode": "img2img",
+            "model": model,
+            "prompt": prompt,
+            "seed": seed,
+            "width": width,
+            "height": height,
+            "guidance_scale": guidance_scale,
+            "sampler": sampler,
+            "steps": steps,
+            "init_images": encoded_images,
+            "strength": denoise,
+        }
+
+        return (payload,)
+
+
+class DrawThingsPipelineAddCustom:
+    def __init__(self):
+        pass
+
+    CATEGORY = "DrawThingsWrapper"
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "pipeline": ("dict", {"tooltip": "Draw Things pipeline"}),
+                "field": ("STRING",),
+                "value_type": (
+                    ["STRING", "INT", "FLOAT"],
+                    {"tooltip": "Choose the type of the value"},
+                ),
+                "value": ("STRING",),
+            }
+        }
+
+    RETURN_TYPES = ("dict",)
+    RETURN_NAMES = ("pipeline",)
+    FUNCTION = "add_to_pipeline"
+
+    def add_to_pipeline(self, pipeline, field, value, value_type):
+
+        if value_type == "INT":
+            value = int(value)
+        elif value_type == "FLOAT":
+            value = float(value)
+        elif value_type == "STRING":
+            value = str(value)
+
+        pipeline[field] = value
+
+        return (pipeline,)
+
+
+class DrawThingsPipelineAddLora:
+    def __init__(self):
+        pass
+
+    CATEGORY = "DrawThingsWrapper"
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "pipeline": ("dict", {"tooltip": "Draw Things pipeline"}),
+                "lora": ("STRING",),
+                "weight": (
+                    "FLOAT",
+                    {"default": 1.0, "min": 0.0, "max": 2.5, "step": 0.1},
+                ),
+            }
+        }
+
+    RETURN_TYPES = ("dict",)
+    RETURN_NAMES = ("pipeline",)
+    FUNCTION = "add_to_pipeline"
+
+    def add_to_pipeline(self, pipeline, lora, weight):
+
+        # Check if 'loras' exists in the pipeline
+        if "loras" not in pipeline:
+            # Create 'loras' as an empty list
+            pipeline["loras"] = []
+
+        # Append the new entry as a dictionary to the list
+        pipeline["loras"].append({"file": lora, "weight": weight})
+
+        return (pipeline,)
+
+
+class DrawThingsPipelineAddControl:
+    def __init__(self):
+        pass
+
+    CATEGORY = "DrawThingsWrapper"
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "pipeline": ("dict", {"tooltip": "Draw Things pipeline"}),
+                "control": ("STRING",),
+                "weight": (
+                    "FLOAT",
+                    {"default": 1.0, "min": 0.0, "max": 2.5, "step": 0.1},
+                ),
+                "guidanceStart": (
+                    "FLOAT",
+                    {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01},
+                ),
+                "guidanceEnd": (
+                    "FLOAT",
+                    {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01},
+                ),
+                "noPrompt": (
+                    "BOOLEAN",
+                    {"default": False},
+                ),
+                "globalAveragePooling": (
+                    "BOOLEAN",
+                    {"default": False},
+                ),
+                "downSamplingRate": (
+                    "FLOAT",
+                    {"default": 0.0},
+                ),
+                "controlImportance": (
+                    "STRING",
+                    {"default": 1.0},
+                ),
+                "controlImportance": (
+                    [
+                        "balanced",
+                        "prompt",
+                        "control",
+                    ],
+                    {"default": "balanced"},
+                ),
+                "inputOverride": ("STRING",),
+                "targetBlocks": ("STRING",),
+            }
+        }
+
+    RETURN_TYPES = ("dict",)
+    RETURN_NAMES = ("pipeline",)
+    FUNCTION = "add_to_pipeline"
+
+    def add_to_pipeline(
+        self,
+        pipeline,
+        control,
+        weight,
+        guidanceStart,
+        guidanceEnd,
+        noPrompt,
+        globalAveragePooling,
+        downSamplingRate,
+        controlImportance,
+        inputOverride,
+        targetBlocks,
+    ):
+
+        # file_path = ""
+        # with open(file_path, "rb") as png_file:
+        #    # Read the file contents
+        #    png_data = png_file.read()
+        #
+        #    # Encode the binary data to base64
+        #    encoded_data = base64.b64encode(png_data)
+        #
+        #    # Convert bytes to string for easier handling
+        #    base64_string = encoded_data.decode('utf-8')
+
+        # Check if 'controls' exists in the pipeline
+        if "controls" not in pipeline:
+            # Create 'controls' as an empty list
+            pipeline["controls"] = []
+
+        # Append the new entry as a dictionary to the list
+        pipeline["controls"].append(
+            {
+                "file": control,
+                "weight": weight,
+                "guidanceStart": guidanceStart,
+                "guidanceEnd": guidanceEnd,
+                "noPrompt": noPrompt,
+                "globalAveragePooling": globalAveragePooling,
+                "downSamplingRate": downSamplingRate,
+                "controlImportance": controlImportance,
+                "inputOverride": inputOverride,  # in eg union controlnets, select type
+                "targetBlocks": [],
+                "enabled": True,
+                # "image": {
+                #  "image": base64_string
+                # }
+            }
+        )
+
+        return (pipeline,)
+
+
+class DrawThingsGenerateFromPipeline:
+    def __init__(self):
+        pass
+
+    CATEGORY = "DrawThingsWrapper"
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "pipeline": ("dict", {"tooltip": "Draw Things pipeline"}),
+            }
+        }
+
+    RETURN_TYPES = ("IMAGE",)
+    RETURN_NAMES = ("generated_image",)
+    FUNCTION = "generate_image"
+
+    def generate_image(self, pipeline):
+
+        # Cannot include generation_mode in payload, but need its value
+        gen_mode = pipeline["generation_mode"]
+        # Call the Draw Things API
+        if gen_mode == "txt2img":
+            api_url = "http://127.0.0.1:7860/sdapi/v1/txt2img"
+        elif gen_mode == "img2img":
+            api_url = "http://127.0.0.1:7860/sdapi/v1/img2img"
+
+        payload = {
+            key: value for key, value in pipeline.items() if key != "generation_mode"
+        }
+
+        #print(payload)
+        response = requests.post(api_url, json=payload)
+
+        #data = response.json()
+        # print(data)
+
+        # Raise an error if the request failed
+        response.raise_for_status()
+
+        # Parse the JSON response
+        data = response.json()
+
+        # Process the images (assuming they are base64 encoded or raw binary data)
+        images = []
+        for img_data in data["images"]:
+            image_bytes = base64.b64decode(img_data)
+            # Convert the image data to a Pillow Image object
+            image = Image.open(io.BytesIO(image_bytes))
+            image_np = np.array(image)
+            # Convert to float32 tensor and normalize
+            tensor_image = torch.from_numpy(image_np.astype(np.float32) / 255.0)
+            images.append(tensor_image)
+        return (torch.stack(images),)
+
+
+NODE_CLASS_MAPPINGS = {
+    "DrawThingsTxt2Img": DrawThingsTxt2Img,
+    "DrawThingsImg2Img": DrawThingsImg2Img,
+    "DrawThingsTxt2ImgPipeline": DrawThingsTxt2ImgPipeline,
+    "DrawThingsImg2ImgPipeline": DrawThingsImg2ImgPipeline,
+    "DrawThingsPipelineAddCustom": DrawThingsPipelineAddCustom,
+    "DrawThingsPipelineAddLora": DrawThingsPipelineAddLora,
+    "DrawThingsPipelineAddControl": DrawThingsPipelineAddControl,
+    "DrawThingsGenerateFromPipeline": DrawThingsGenerateFromPipeline,
+}
+
+NODE_DISPLAY_NAME_MAPPINGS = {
+    "DrawThingsTxt2Img": "Draw Things Txt2Img",
+    "DrawThingsImg2Img": "Draw Things Img2Img",
+    "DrawThingsTxt2ImgPipeline": "Draw Things Txt2Img Pipeline",
+    "DrawThingsImg2ImgPipeline": "Draw Things Img2Img Pipeline",
+    "DrawThingsPipelineAddCustom": "Draw Things Pipeline Add Custom Field",
+    "DrawThingsPipelineAddLora": "Draw Things Pipeline Add Lora",
+    "DrawThingsPipelineAddControl": "Draw Things Pipeline Add Control",
+    "DrawThingsGenerateFromPipeline": "Draw Things Generate from Pipeline",
+}

pyproject.toml

@@ -1,15 +0,0 @@
-[project]
-name = "comfyui-drawthingswrapper"
-description = "These nodes provide a wrapper for calling Draw Things image generations from ComfyUI. The Draw Things app has been optimized for Apple hardware and runs roughly x3 faster than ComfyUI generations. But ComfyUI is a flexible and powerful tool, and has some features - like queuing and face swapping - that haven't been implemented in Draw Things."
-version = "1.0.0"
-license = {file = "LICENSE"}
-dependencies = ["numpy", "PIL", "requests", "torch"]
-
-[project.urls]
-Repository = "https://github.com/JosephThomasParker/ComfyUI-DrawThingsWrapper"
-#  Used by Comfy Registry https://comfyregistry.org
-
-[tool.comfy]
-PublisherId = "seosamh"
-DisplayName = "ComfyUI-DrawThingsWrapper"
-Icon = ""