Postprocess upscale by: 2.0, Postprocess upscaler: Lanczos
", +# "image": "iVBORw0KGgoAAAANSUhEUgAACAAJggg==" +# } +# only "image" is needed + +class OpenOutpainterServingInputUpscale: + CLASSNAME = "OpenOutpainterServingInputUpscaleV1" + NAME = "OpenOutpainter Serving Input Upscale" + CATEGORY = get_category() + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "oop_request": ("OOP_REQUEST", {"lazy": True}), + } + } + + RETURN_TYPES = ("IMAGE", "FLOAT") + RETURN_NAMES = ("image", "scale_by") + FUNCTION = "out" + + def __init__(self): + self.command_name = POSTPATHS.PATH_UPSCALE + pass + + def check_lazy_status(self, oop_request): + if oop_request is None: + return ["oop_request"] + if oop_request.is_command(self.command_name): + return ["oop_request"] + return None + + def out(self, oop_request): + # lazy eval does not work for some reason + if not oop_request.is_command(self.command_name): + return (ExecutionBlocker(None), ExecutionBlocker(None)) + return ( + base64_to_image(oop_request.request_data["image"]), + float(oop_request.request_data["upscaling_resize"]), + ) + + +class OpenOutpainterServingOutputUpscale: + CLASSNAME = "OpenOutpainterServingOutputUpscaleV1" + NAME = "OpenOutpainter Serving Output Upscale" + CATEGORY = get_category() + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "oop_request": ("OOP_REQUEST", {"lazy": True}), + "image": ("IMAGE", {"lazy": True}), + }, + } + + FUNCTION = "out" + RETURN_TYPES = () + OUTPUT_NODE = True + + def __init__(self): + self.command_name = POSTPATHS.PATH_UPSCALE + pass + + def check_lazy_status(self, oop_request, image=None): + if oop_request is None: + return ["oop_request"] + if oop_request.is_command(self.command_name): + return ["image", "oop_request"] + return None + + def out(self, oop_request, image=None): + print(f"{self.NAME} out '{self.command_name}' image: {bool(image is not None)}") + if image is not None and oop_request.is_command(self.command_name): + response = {"image": image_to_base64(image)} + oop_request.finalize(response) + return {} + + +############################ +# Interrogate Nodes # +############################ +# caption image: '/sdapi/v1/interrogate': +# request +# {"image":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAABggg==","model":"clip"} +# model is hardcoded to always return "clip" +# response +# {"caption":"caption text here"} + +class OpenOutpainterServingInputInterrogate: + CLASSNAME = "OpenOutpainterServingInputInterrogateV1" + NAME = "OpenOutpainter Serving Input Interrogate" + CATEGORY = get_category() + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "oop_request": ("OOP_REQUEST", {"lazy": True}), + } + } + + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES = ("image",) + FUNCTION = "out" + + def __init__(self): + self.command_name = POSTPATHS.PATH_INTERROGATE + pass + + def check_lazy_status(self, oop_request): + if oop_request is None: + return ["oop_request"] + if oop_request.is_command(self.command_name): + return ["oop_request"] + return None + + def out(self, oop_request): + # lazy eval does not work for some reason + if not oop_request.is_command(self.command_name): + return (ExecutionBlocker(None),) + return (base64_to_image(oop_request.request_data["image"]),) + + +class OpenOutpainterServingOutputInterrogate: + CLASSNAME = "OpenOutpainterServingOutputInterrogateV1" + NAME = "OpenOutpainter Serving Output Interrogate" + CATEGORY = get_category() + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "oop_request": ("OOP_REQUEST", {"lazy": True}), + "caption": ("STRING", {"lazy": True, "forceInput": True}), + }, + } + + FUNCTION = "out" + RETURN_TYPES = () + OUTPUT_NODE = True + + def __init__(self): + self.command_name = POSTPATHS.PATH_INTERROGATE + pass + + def check_lazy_status(self, oop_request, caption=None): + if oop_request is None: + return ["oop_request"] + if oop_request.is_command(self.command_name): + return ["caption", "oop_request"] + return None + + def out(self, oop_request, caption=None): + print(f"{self.NAME} out '{self.command_name}' caption: {caption}") + if caption is not None and oop_request.is_command(self.command_name): + response = {"caption": caption} + oop_request.finalize(response) + return {} + + +######################## +# TXT2IMG Nodes # +######################## +# txt2img: '/sdapi/v1/txt2img': +# request +# { +# "prompt": "This is a prompt", +# "negative_prompt": "This is a negative prompt", +# "seed": "-1", +# "cfg_scale": 7, +# "sampler_index": "DDIM", +# "steps": 30, +# "denoising_strength": 1, +# "mask_blur": 8, +# "batch_size": 1, +# "width": 128, +# "height": 128, +# "n_iter": 3, +# "mask": "", +# "init_images": [], +# "inpaint_full_res": false, +# "inpainting_fill": 1, +# "outpainting_fill": 2, +# "enable_hr": false, +# "restore_faces": false, +# "hr_scale": 2, +# "hr_upscaler": "None", +# "hr_second_pass_steps": 0, +# "hr_resize_x": 0, +# "hr_resize_y": 0, +# "hr_square_aspect": false, +# "styles": [], +# "upscale_x": 2, +# "hr_denoising_strength": 0.7, +# "hr_fix_lock_px": 0, +# "enable_refiner": false, +# "alwayson_scripts": {} +# } + +# response +# { +# "images": [ +# "iVBORw0KGgoAAAANSUhEUgAAAIAAAACAC5DU117AAAAAElFTkSuQmCC", +# "iVBORw0KGgoAAAANSUhEUgAAAIAAAACAC5DU117AAAAAElFTkSuQmCC", +# "iVBORw0KGgoAAAANSUhEUgAAAIAAAACAC5DU117AAAAAElFTkSuQmCC" +# ], +# "parameters": {}, # not used +# "info": { +# "all_seeds": [ +# 2742961909, +# 2742961910, +# 2742961911 +# ], +# } +# } + +class OpenOutpainterServingInputTXT2IMG: + CLASSNAME = "OpenOutpainterServingInputTXT2IMGV1" + NAME = "OpenOutpainter Serving Input TXT2IMG" + CATEGORY = get_category() + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "oop_request": ("OOP_REQUEST", {"lazy": True}), + } + } + + OUTPUTS = { + "prompt": "STRING", + "negative_prompt": "STRING", + "width": "INT", + "height": "INT", + "SEED": "INT", + "steps": "INT", + "cfg_scale": "FLOAT", + "batch_size": "INT", + "n_iter": "INT", + } + RETURN_TYPES = tuple(OUTPUTS.values()) + RETURN_NAMES = tuple(OUTPUTS.keys()) + FUNCTION = "out" + + def __init__(self): + self.command_name = POSTPATHS.PATH_TXT2IMG + pass + + def check_lazy_status(self, oop_request): + if oop_request is None: + return ["oop_request"] + if oop_request.is_command(self.command_name): + return ["oop_request"] + return None + + def out(self, oop_request): + # lazy eval does not work for some reason + if not oop_request.is_command(self.command_name): + return (ExecutionBlocker(None),) * len(self.OUTPUTS) + request_data = oop_request.request_data + return ( + request_data["prompt"], + request_data["negative_prompt"], + int(request_data["width"]), + int(request_data["height"]), + int(request_data["seed"]), + int(request_data["steps"]), + float(request_data["cfg_scale"]), + int(request_data["batch_size"]), + int(request_data["n_iter"]), + ) + + +class OpenOutpainterServingOutputTXT2IMG: + CLASSNAME = "OpenOutpainterServingOutputTXT2IMGV1" + NAME = "OpenOutpainter Serving Output TXT2IMG" + CATEGORY = get_category() + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "oop_request": ("OOP_REQUEST", {"lazy": True}), + "images": ("IMAGE", {"lazy": True}), + "SEEDS": ("INT", {"lazy": True, "forceInput": True}), + }, + } + + FUNCTION = "out" + RETURN_TYPES = () + OUTPUT_NODE = True + INPUT_IS_LIST = True + + def __init__(self): + self.command_name = POSTPATHS.PATH_TXT2IMG + pass + + def check_lazy_status(self, oop_request, images=None, SEEDS=None): + if oop_request is None: + return ["oop_request"] + if oop_request[0] is None: + return ["oop_request"] + if oop_request[0].is_command(self.command_name): + return ["SEEDS", "images", "oop_request"] + return None + + def out(self, oop_request, images=None, SEEDS=None): + print(f"{self.NAME} out '{self.command_name}' images: {bool(images is not None)}") + if images is not None and SEEDS is not None and oop_request[0].is_command(self.command_name): + response = { + "images": images_to_base64(images), + "info": json.dumps({"all_seeds": SEEDS}, default=lambda o: None) + } + oop_request[0].finalize(response) + return {} + + + +######################## +# IMG2IMG Nodes # +######################## +# txt2img: '/sdapi/v1/img2img': +# request +# { +# "prompt": "This is a prompt", +# "negative_prompt": "people, person, humans, human, divers, diver, glitch, error, text, watermark, bad quality, blurry", +# "seed": "-1", +# "cfg_scale": 7, +# "steps": 30, +# "denoising_strength": 0.5, +# "mask_blur": 8, +# "batch_size": 1, +# "width": 512, +# "height": 512, +# "n_iter": 3, +# "mask": "data:image/png;base64,iVBORw0KGgoAAPQAAAABJRU5ErkJggg==", +# "init_images": ["data:image/png;base64,iVBORwo9jTl4TD/Er4/f5by6KwHISuQmCC"], +# "image_cfg_scale": 8, +# } + +# response +# { +# "images": ["iVBORw0KGgoAAAANSUhEUgAAAIAAAACACAIAAABAAElFTkSuQmCC"], +# "parameters": {}, # not used +# "info": { +# "all_subseeds": [ +# 3242546751, +# 3242546752, +# 3242546753 +# ], +# } +# } + +class OpenOutpainterServingInputIMG2IMG: + CLASSNAME = "OpenOutpainterServingInputIMG2IMGV1" + NAME = "OpenOutpainter Serving Input IMG2IMG" + CATEGORY = get_category() + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "oop_request": ("OOP_REQUEST", {"lazy": True}), + } + } + + OUTPUTS = { + "init_images": "IMAGE", + "mask": "MASK", + "prompt": "STRING", + "negative_prompt": "STRING", + "width": "INT", + "height": "INT", + "SEED": "INT", # seed + "steps": "INT", + "cfg_scale": "FLOAT", + "denoising_strength": "FLOAT", + "mask_blur": "INT", + "ip2p_image_cfg_scale": "FLOAT", # image_cfg_scale + "batch_size": "INT", + "n_iter": "INT", + } + RETURN_TYPES = tuple(OUTPUTS.values()) + RETURN_NAMES = tuple(OUTPUTS.keys()) + FUNCTION = "out" + + def __init__(self): + self.command_name = POSTPATHS.PATH_IMG2IMG + pass + + def check_lazy_status(self, oop_request): + if oop_request is None: + return ["oop_request"] + if oop_request.is_command(self.command_name): + return ["oop_request"] + return None + + def out(self, oop_request): + # lazy eval does not work for some reason + if not oop_request.is_command(self.command_name): + return (ExecutionBlocker(None),) * len(self.OUTPUTS) + request_data = oop_request.request_data + return ( + base64_to_image(request_data["init_images"][0]), + base64_to_mask(request_data["mask"]), + request_data["prompt"], + request_data["negative_prompt"], + int(request_data["width"]), + int(request_data["height"]), + int(request_data["seed"]), + int(request_data["steps"]), + float(request_data["cfg_scale"]), + float(request_data["denoising_strength"]), + int(request_data["mask_blur"]), + float(request_data.get("image_cfg_scale", 0)), # may be missing if txt2img tool calls img2img instead + int(request_data["batch_size"]), + int(request_data["n_iter"]), + ) + + +class OpenOutpainterServingOutputIMG2IMG: + CLASSNAME = "OpenOutpainterServingOutputIMG2IMGV1" + NAME = "OpenOutpainter Serving Output IMG2IMG" + CATEGORY = get_category() + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "oop_request": ("OOP_REQUEST", {"lazy": True}), + "images": ("IMAGE", {"lazy": True}), + "SEEDS": ("INT", {"lazy": True, "forceInput": True}), + }, + } + + FUNCTION = "out" + RETURN_TYPES = () + OUTPUT_NODE = True + INPUT_IS_LIST = True + + def __init__(self): + self.command_name = POSTPATHS.PATH_IMG2IMG + pass + + def check_lazy_status(self, oop_request, images=None, SEEDS=None): + if oop_request is None: + return ["oop_request"] + if oop_request[0] is None: + return ["oop_request"] + if oop_request[0].is_command(self.command_name): + return ["SEEDS", "images", "oop_request"] + return None + + def out(self, oop_request, images=None, SEEDS=None): + print(f"{self.NAME} out '{self.command_name}' images: {bool(images is not None)}") + if images is not None and SEEDS is not None and oop_request[0].is_command(self.command_name): + response = { + "images": images_to_base64(images), + "info": json.dumps({"all_seeds": SEEDS}, default=lambda o: None) + } + oop_request[0].finalize(response) + return {} + + +######################## + + + +def register_nodes(node_class_mappings=None, node_display_name_mappings=None): + if node_display_name_mappings is None: + node_display_name_mappings = {} + if node_class_mappings is None: + node_class_mappings = {} + + node_classes = [ + OpenOutpainterServing, + + OpenOutpainterServingInputUpscale, + OpenOutpainterServingOutputUpscale, + + OpenOutpainterServingInputInterrogate, + OpenOutpainterServingOutputInterrogate, + + OpenOutpainterServingInputTXT2IMG, + OpenOutpainterServingOutputTXT2IMG, + + OpenOutpainterServingInputIMG2IMG, + OpenOutpainterServingOutputIMG2IMG, + ] + + for node_class in node_classes: + node_class_mappings[node_class.CLASSNAME] = node_class + node_display_name_mappings[node_class.CLASSNAME] = node_class.NAME + + return node_class_mappings, node_display_name_mappings + diff --git a/py/utils.py b/py/utils.py new file mode 100644 index 0000000..46cdb60 --- /dev/null +++ b/py/utils.py @@ -0,0 +1,64 @@ +import base64 +from io import BytesIO +from PIL import Image +import numpy as np +import torch +import cv2 + +CATEGORYNAMESPACE = 'OpenOutpaint-Serving' + +def get_category(sub_dirs = None): + if sub_dirs is None: + return CATEGORYNAMESPACE + else: + return "{}/utils".format(CATEGORYNAMESPACE) + +def _strip_prefix(s: str, prefix: str) -> str: + if s.startswith(prefix): + return s[len(prefix) :] + return s + + +def convert_color(image): + if len(image.shape) > 2 and image.shape[2] >= 4: + return cv2.cvtColor(image, cv2.COLOR_BGRA2RGB) + return cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + +def base64_to_image(base64_str): + base64_str = _strip_prefix(base64_str, "data:image/png;base64,") + nparr = np.frombuffer(base64.b64decode(base64_str), np.uint8) + result = cv2.imdecode(nparr, cv2.IMREAD_UNCHANGED) + result = convert_color(result) + result = result.astype(np.float32) / 255.0 + image = torch.from_numpy(result)[None,] + return image + +def base64_to_mask(base64_str): + base64_str = _strip_prefix(base64_str, "data:image/png;base64,") + nparr = np.frombuffer(base64.b64decode(base64_str), np.uint8) + result = cv2.imdecode(nparr, cv2.IMREAD_UNCHANGED) + result = result.astype(np.float32) / 255.0 + image = torch.from_numpy(result) + if image.dim() == 3: # RGB(A) input, use red channel + image = image[:, :, 0] + return image.unsqueeze(0) + +def image_to_base64(image): + img_np = (image.cpu().numpy() * 255).astype('uint8') + img_bytes = BytesIO() + Image.fromarray(img_np.squeeze()).save(img_bytes, format='PNG') + base64_image = base64.b64encode(img_bytes.getvalue()).decode('utf-8') + return base64_image + +def images_to_base64(images): + base64_images = [] + for image in images: + base64_image = image_to_base64(image) + base64_images.append(base64_image) + return base64_images + +def preview_to_base64(image): + img_bytes = BytesIO() + image.save(img_bytes, format="JPEG", quality=80) + base64_image = base64.b64encode(img_bytes.getvalue()).decode('utf-8') + return base64_image diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..dab6d29 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,19 @@ +[project] +name = "openoutpaint_comfyui_interface" +version = "0.0.1" +description = "An API interface for OpenOutpaint to work with ComfyUI workflows" +authors = [{name = "Without Ordinary", email = "without-ordinary@users.noreply.github.com"}] +readme = "README.md" +license = {text = "GNU General Public License v3"} +classifiers = [] +dependencies = ["opencv-python"] + +[project.urls] +bugs = "https://github.com/without-ordinary/openoutpaint_comfyui_interface/issues" +homepage = "https://github.com/without-ordinary/openoutpaint_comfyui_interface" + +[tool.comfy] +PublisherId = "without-ordinary" +DisplayName = "OpenOutpaint ComfyUI Interface" +Icon = "" + diff --git a/web/httpserving.js b/web/httpserving.js new file mode 100644 index 0000000..3c6ac03 --- /dev/null +++ b/web/httpserving.js @@ -0,0 +1,26 @@ +import { app } from "../../../scripts/app.js"; +import { api } from "../../scripts/api.js"; + +function QueuePrompt_MessageHandler(event) { + let nodes = app.graph._nodes_by_id; + let node = nodes[event.detail.node_id]; + console.debug(node); + if(node && node.type == event.detail.node_type) { + const port_widget = node.widgets.find((w) => w.name === "port"); + const request_id_widget = node.widgets.find((w) => w.name === "request_id"); + if(request_id_widget && port_widget && port_widget.value == event.detail.port) { + request_id_widget.value = event.detail.request_id; + console.debug("wo_QueuePrompt app.queuePrompt"); + app.queuePrompt(0, 1); + return; + } + } +} + +app.registerExtension({ + name: "wo.OpenOutpainterServing", + async setup() { + api.addEventListener("wo_QueuePrompt", QueuePrompt_MessageHandler); + }, +}) +