diff --git a/README.md b/README.md index 2a9b70c..c6afd18 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,7 @@ Please try downgrading the ```protobuf``` dependency package to 3.20.3, or set e **If the dependency package error after updating, please double clicking ```repair_dependency.bat``` (for Official ComfyUI Protable) or ```repair_dependency_aki.bat``` (for ComfyUI-aki-v1.x) in the plugin folder to reinstall the dependency packages. +* Commit [Collage](#Collage) node to collage images into one. * Commit [DeepSeekAPI](DeepSeekAPI) node, Use DeepSeek API for text inference. * Commit [SegmentAnythingUltraV3](#SegmentAnythingUltraV3) and [LoadSegmentAnythingModels](#LoadSegmentAnythingModels) nodes, Avoid duplicating model loading when using multiple SAM nodes. * Commit [ZhipuGLM4](#ZhipuGLM4) and [ZhipuGLM4V](#ZhipuGLM4V) nodes, Use the Zhipu API for textual and visual inference. Among the current Zhipu models, GLM-4-Flash and glm-4v-flash models are free. @@ -162,6 +163,27 @@ download Florence-2-Flux-Large and Florence-2-Flux folder from [BaiduNetdisk](ht ## Description +### Collage +Randomly collage the input images into one large image. + +![image](image/collage_example.jpg) + +Node Options: +![image](image/collage_node.jpg) + +* images: The input images. +* florence2_model: Optional input for object recognition and cropping. +* canvas_width: Output the width of the image. +* canvas_height: Output the height of the image. +* border_width: The border width. +* rounded_rect_radius: The border fillet radius. +* uniformity: The randomness of image stitching size. The value range is 0-1, and the larger the value, the greater the randomness of the size. +* background_color: The background color. +* seed: The seed of random number. +* control_after_generate: Seed change options. If this option is fixed, the generated random number will always be the same. +* object_prompt: When connecting to florence2_model, fill in the prompt words for object recognition here. + + ### QWenImage2Prompt Inference the prompts based on the image. this node is repackage of the [ComfyUI_VLM_nodes](https://github.com/gokayfem/ComfyUI_VLM_nodes)'s ```UForm-Gen2 Qwen Node```, thanks to the original author. diff --git a/README_CN.MD b/README_CN.MD index 856b316..8e3c20f 100644 --- a/README_CN.MD +++ b/README_CN.MD @@ -121,6 +121,7 @@ If this call came from a _pb2.py file, your generated code is out of date and mu ## 更新说明 **如果本插件更新后出现依赖包错误，请双击运行插件目录下的```install_requirements.bat```(官方便携包)，或 ```install_requirements_aki.bat```(秋叶整合包) 重新安装依赖包。 +* 添加 [Collage](#Collage) 节点，将多张图片拼合为一张大图。 * 添加 [DeepSeekAPI](DeepSeekAPI) 节点，使用DeepSeek API进行文本推理。 * 添加 [SegmentAnythingUltraV3](#SegmentAnythingUltraV3) 和 [LoadSegmentAnythingModels](#LoadSegmentAnythingModels)节点， 在使用多个SAM节点时避免重复加载模型。 * 添加 [ZhipuGLM4](#ZhipuGLM4) 和 [ZhipuGLM4V](#ZhipuGLM4V)节点，使用智谱API进行文本和视觉推理。目前的智谱模型中，GLM-4-Flash和glm-4v-flash模型是免费的。 @@ -138,6 +139,28 @@ If this call came from a _pb2.py file, your generated code is out of date and mu ## 节点说明 + +### Collage +将输入的批量图片随机拼合为一张大图。 + +![image](image/collage_example.jpg) + +节点选项说明: +![image](image/collage_node.jpg) + +* images: 图片输入。 +* florence2_model: 可选输入，用于物体识别裁切。 +* canvas_width: 输出图片的宽度。 +* canvas_height: 输出图片的高度。 +* border_width: 边框宽度。 +* rounded_rect_radius: 边框圆角半径。 +* uniformity: 图片拼合大小的随机性。取值范围为0-1，值越大，大小随机性越大。 +* background_color: 背景色。 +* seed: 随机种子。 +* control_after_generate: 设置每次执行时种子值的变化。 +* object_prompt: 当接入florence2_model时，此处填写物体识别的提示词。 + + ### QWenImage2Prompt 根据图片反推提示词。这个节点是[ComfyUI_VLM_nodes](https://github.com/gokayfem/ComfyUI_VLM_nodes)中的```UForm-Gen2 Qwen Node```节点的重新封装，感谢原作者。 从[huggingface](https://huggingface.co/unum-cloud/uform-gen2-qwen-500m)或者[百度网盘](https://pan.baidu.com/s/1oRkUoOKWaxGod_XTJ8NiTA?pwd=d5d2)下载模型到```ComfyUI/models/LLavacheckpoints/files_for_uform_gen2_qwen```文件夹。 diff --git a/image/collage_example.jpg b/image/collage_example.jpg new file mode 100644 index 0000000..32425a1 Binary files /dev/null and b/image/collage_example.jpg differ diff --git a/image/collage_node.jpg b/image/collage_node.jpg new file mode 100644 index 0000000..9920c37 Binary files /dev/null and b/image/collage_node.jpg differ diff --git a/py/collage.py b/py/collage.py new file mode 100644 index 0000000..bbb2d62 --- /dev/null +++ b/py/collage.py @@ -0,0 +1,251 @@ +import torch +import random +import cv2 +import numpy as np +from tqdm import tqdm +from PIL import Image, ImageDraw, ImageFilter +import copy +from .imagefunc import log, tensor2pil, pil2tensor, image2mask +from .imagefunc import fit_resize_image, extract_numbers, gaussian_blur, mask_area, draw_rounded_rectangle + +class LS_CollageGenerator: + """ + 随机分割生成指定数量的不规则小矩形。 + """ + def __init__(self, width, height, num, border_width, r, uniformity, seed ): + self.width = width + self.height = height + self.num = num + self.border_width = int((self.width + self.height) * border_width / 200) + self.r = r + self.seed = seed + self.split_num = int(1e18) + self.uniformity = uniformity + self.rectangles = self.adjust_bboxes_with_gaps(self.split_rec()) + + def split_rec(self): + random.seed(self.seed) + if self.num <= 0 or self.width <= 0 or self.height <= 0: + raise ValueError("Value mast be positive integer") + + current_rectangles = [(0, 0, self.width, self.height, 0)] + + while len(current_rectangles) < self.num: + split_counts = [rect[4] for rect in current_rectangles] + min_splits = min(split_counts) + max_splits = max(split_counts) + probabilities = [] + + for rect in current_rectangles: + split_count = rect[4] + normalized_splits = (split_count - min_splits) / ( + max_splits - min_splits if max_splits > min_splits else 1) + probability = 1 - (normalized_splits * (1 - self.uniformity)) + probabilities.append(probability) + if sum(probabilities) > 0: + probabilities = [p / sum(probabilities) for p in probabilities] + else: + probabilities = [1.0 / len(probabilities)] * len(probabilities) + + rect_index = random.choices(range(len(current_rectangles)), + weights=probabilities, k=1)[0] + + x, y, w, h, split_count = current_rectangles.pop(rect_index) + + if w > h or (w == h and random.choice([True, False])): + split = random.uniform(0.3, 0.7) * w + rect1 = (x, y, split, h, split_count + 1) + rect2 = (x + split, y, w - split, h, split_count + 1) + else: + split = random.uniform(0.3, 0.7) * h + rect1 = (x, y, w, split, split_count + 1) + rect2 = (x, y + split, w, h - split, split_count + 1) + + current_rectangles.extend([rect1, rect2]) + + rectangles = [(int(x), int(y), int(w), int(h)) + for x, y, w, h, _ in current_rectangles] + + return rectangles + + def adjust_bboxes_with_gaps(self, rectangles): + MIN_SIZE = 1 + adjusted_bboxes = [] + + for x, y, w, h in rectangles: + new_x = min(x + self.border_width, self.width - MIN_SIZE) + new_y = min(y + self.border_width, self.height - MIN_SIZE) + new_w = max(MIN_SIZE, w - 2 * self.border_width) + new_h = max(MIN_SIZE, h - 2 * self.border_width) + + if new_x + new_w > self.width: + new_x = max(0, self.width - new_w) + if new_y + new_h > self.height: + new_y = max(0, self.height - new_h) + + adjusted_bboxes.append((new_x, new_y, new_w, new_h)) + + return adjusted_bboxes + + def draw_mask(self): + bboxes = [] + + for bbox in self.rectangles: + bboxes.append((bbox[0], bbox[1], bbox[0] + bbox[2], bbox[1] + bbox[3])) + scale_factor = 2 + + img = Image.new('RGB', (self.width, self.height), color='white') + img = draw_rounded_rectangle(img, self.r, bboxes, scale_factor, color='black') + + return img + + +class LS_Collage: + def __init__(self): + self.NODE_NAME = 'Collage' + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "images": ("IMAGE",), + "canvas_width": ("INT", {"default": 2048, "min": 512, "max": 8192, "step": 16}), + "canvas_height": ("INT", {"default": 2048, "min": 512, "max": 8192, "step": 16}), + "border_width": ("FLOAT", {"default": 2, "min": 0, "max": 20, "step": 0.1}), + "rounded_rect_radius": ("INT", {"default": 8, "min": 0, "max": 100, "step": 1}), + "uniformity": ("FLOAT", {"default": 0.5, "min": 0, "max": 1, "step": 0.1}), # 分割均匀权重 0均匀分割，1不均匀分割 + "background_color": ("STRING", {"default": "#FFFFFF"}), + "seed": ("INT", {"default": 0, "min": 0, "max": 1e18, "step": 1}), + }, + "optional": { + "florence2_model": ("FLORENCE2",), + "object_prompt": ("STRING", {"default": "face"}), + } + } + + RETURN_TYPES = ("IMAGE", "MASK",) + RETURN_NAMES = ("image", "mask",) + FUNCTION = "collage" + CATEGORY = '😺dzNodes/LayerUtility' + + def collage(self, images, canvas_width, canvas_height, border_width, rounded_rect_radius, + uniformity, background_color, seed, florence2_model=None, object_prompt="face"): + + batch_size = images.shape[0] + + rects = LS_CollageGenerator(width=canvas_width, + height=canvas_height, + num=batch_size, + border_width=border_width, + r=rounded_rect_radius, + uniformity=uniformity, + seed=seed) + + rects_border_image = rects.draw_mask() + canvas = Image.new("RGB", (canvas_width, canvas_height), color=background_color) + color_image = copy.deepcopy(canvas) + from .object_detector import LS_OBJECT_DETECTOR_FL2 + + for i in tqdm(range(batch_size)): + img = tensor2pil(images[i]).convert("RGB") + img_x = rects.rectangles[i][0] + img_y = rects.rectangles[i][1] + img_target_width = rects.rectangles[i][2] + img_target_height = rects.rectangles[i][3] + + od = LS_OBJECT_DETECTOR_FL2() + if florence2_model is not None: + bboxes = od.object_detector_fl2(image=[images[i]], prompt=object_prompt, florence2_model=florence2_model, + sort_method="confidence", bbox_select="first", select_index="0")[0] + bbox_mask = self.draw_bbox_mask(img, bboxes, 0, 0, 0, 0) + resized_img = self.image_auto_crop_v3(img, img_target_width, img_target_height, bbox_mask) + else: + resized_img = fit_resize_image(img, img_target_width, img_target_height, fit="crop", resize_sampler=Image.LANCZOS) + + canvas.paste(resized_img, box=(img_x, img_y)) + canvas.paste(color_image, box=(0, 0), mask=rects_border_image.convert("L")) + + return (pil2tensor(canvas), 1 - image2mask(rects_border_image),) + + def draw_bbox_mask(self, image, bboxes, grow_top, grow_bottom, grow_left, grow_right + ): + + mask = Image.new("L", image.size, color='black') + for bbox in bboxes: + try: + if len(bbox) == 0: + continue + else: + x1, y1, x2, y2 = bbox + except ValueError: + if len(bbox) == 0: + continue + else: + x1, y1, x2, y2 = bbox[0] + w = x2 - x1 + h = y2 - y1 + if grow_top: + y1 = int(y1 - h * grow_top) + if grow_bottom: + y2 = int(y2 + h * grow_bottom) + if grow_left: + x1 = int(x1 - w * grow_left) + if grow_right: + x2 = int(x2 + w * grow_right) + if y1 > y2 or x1 > x2: + continue + draw = ImageDraw.Draw(mask) + draw.rectangle([x1, y1, x2, y2], fill='white', outline='white', width=0) + + return mask + + def image_auto_crop_v3(self, image, proportional_width, proportional_height, mask, + ): + + scale_to_length = proportional_width + _image = image + ratio = proportional_width / proportional_height + resize_sampler = Image.LANCZOS + # calculate target width and height + if ratio > 1: + target_width = scale_to_length + target_height = int(target_width / ratio) + else: + target_width = scale_to_length + target_height = int(target_width / ratio) + + _mask = mask + bluredmask = gaussian_blur(_mask, 20).convert('L') + (mask_x, mask_y, mask_w, mask_h) = mask_area(bluredmask) + orig_ratio = _image.width / _image.height + target_ratio = target_width / target_height + # crop image to target ratio + if orig_ratio > target_ratio: # crop LiftRight side + crop_w = int(_image.height * target_ratio) + crop_h = _image.height + else: # crop TopBottom side + crop_w = _image.width + crop_h = int(_image.width / target_ratio) + crop_x = mask_w // 2 + mask_x - crop_w // 2 + if crop_x < 0: + crop_x = 0 + if crop_x + crop_w > _image.width: + crop_x = _image.width - crop_w + crop_y = mask_h // 2 + mask_y - crop_h // 2 + if crop_y < 0: + crop_y = 0 + if crop_y + crop_h > _image.height: + crop_y = _image.height - crop_h + crop_image = _image.crop((crop_x, crop_y, crop_x + crop_w, crop_y + crop_h)) + ret_image = crop_image.resize((target_width, target_height), resize_sampler) + + return ret_image + + +NODE_CLASS_MAPPINGS = { + "LayerUtility: Collage": LS_Collage, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "LayerUtility: Collage": "LayerUtility: Collage(Advance)", +} \ No newline at end of file diff --git a/py/object_detector.py b/py/object_detector.py index 96aa91b..d7ad6cd 100644 --- a/py/object_detector.py +++ b/py/object_detector.py @@ -124,10 +124,11 @@ class LS_OBJECT_DETECTOR_FL2: preview = draw_bounding_boxes(img, bboxes, color="random", line_width=-1) ret_previews.append(pil2tensor(preview)) ret_bboxes.append(standardize_bbox(bboxes)) - if len(bboxes) == 0: - log(f"{self.NODE_NAME} no object found", message_type='warning') - else: - log(f"{self.NODE_NAME} found {len(bboxes)} object(s)", message_type='info') + + # if len(bboxes) == 0: + # log(f"{self.NODE_NAME} no object found", message_type='warning') + # else: + # log(f"{self.NODE_NAME} found {len(bboxes)} object(s)", message_type='info') return (ret_bboxes, torch.cat(ret_previews, dim=0)) @@ -175,7 +176,9 @@ class LS_OBJECT_DETECTOR_FL2: x2_c = max(x2_c, int(max(polygon[0::2]))) y1_c = min(y1_c, int(min(polygon[1::2]))) y2_c = max(y2_c, int(max(polygon[1::2]))) - ret_bboxes.append(x1_c, y1_c, x2_c, y2_c) + ret_bboxes.append([x1_c, y1_c, x2_c, y2_c]) + if len(ret_bboxes) == 0: + ret_bboxes.append([x1_c, y1_c, x2_c, y2_c]) return ret_bboxes class LS_OBJECT_DETECTOR_MASK: @@ -226,10 +229,11 @@ class LS_OBJECT_DETECTOR_MASK: preview = draw_bounding_boxes(tensor2pil(msk).convert("RGB"), bboxes, color="random", line_width=-1) ret_previews.append(pil2tensor(preview)) - if len(bboxes) == 0: - log(f"{self.NODE_NAME} no object found", message_type='warning') - else: - log(f"{self.NODE_NAME} found {len(bboxes)} object(s)", message_type='info') + # if len(bboxes) == 0: + # log(f"{self.NODE_NAME} no object found", message_type='warning') + # else: + # log(f"{self.NODE_NAME} found {len(bboxes)} object(s)", message_type='info') + ret_bboxes.append(standardize_bbox(bboxes)) return (ret_bboxes, torch.cat(ret_previews, dim=0)) @@ -290,10 +294,11 @@ class LS_OBJECT_DETECTOR_YOLO8: preview = draw_bounding_boxes(_image.convert("RGB"), bboxes, color="random", line_width=-1) ret_previews.append(pil2tensor(preview)) - if len(bboxes) == 0: - log(f"{self.NODE_NAME} no object found", message_type='warning') - else: - log(f"{self.NODE_NAME} found {len(bboxes)} object(s)", message_type='info') + # if len(bboxes) == 0: + # log(f"{self.NODE_NAME} no object found", message_type='warning') + # else: + # log(f"{self.NODE_NAME} found {len(bboxes)} object(s)", message_type='info') + ret_bboxes.append(standardize_bbox(bboxes)) return (ret_bboxes, torch.cat(ret_previews, dim=0),) @@ -373,10 +378,11 @@ class LS_OBJECT_DETECTOR_YOLOWORLD: preview = draw_bounding_boxes(tensor2pil(i.unsqueeze(0)).convert('RGB'), bboxes, color="random", line_width=-1) ret_previews.append(pil2tensor(preview)) - if len(bboxes) == 0: - log(f"{self.NODE_NAME} no object found", message_type='warning') - else: - log(f"{self.NODE_NAME} found {len(bboxes)} object(s)", message_type='info') + # if len(bboxes) == 0: + # log(f"{self.NODE_NAME} no object found", message_type='warning') + # else: + # log(f"{self.NODE_NAME} found {len(bboxes)} object(s)", message_type='info') + ret_bboxes.append(standardize_bbox(bboxes)) return (ret_bboxes, torch.cat(ret_previews, dim=0)) diff --git a/pyproject.toml b/pyproject.toml index fa80a50..93e35a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "comfyui_layerstyle_advance" description = "The nodes detached from ComfyUI Layer Style are mainly those with complex requirements for dependency packages." -version = "2.0.12" +version = "2.0.13" license = "MIT" dependencies = ["numpy", "matplotlib", "scikit_image", "scikit_learn", "opencv-contrib-python", "pymatting", "timm", "blend_modes", "transformers", "diffusers", "loguru", "colour-science", "huggingface_hub", "segment_anything", "addict", "omegaconf", "yapf", "wget", "iopath", "mediapipe", "typer_config", "fastapi", "rich", "google-generativeai", "ultralytics", "transparent-background", "accelerate", "onnxruntime", "bitsandbytes", "peft", "protobuf", "hydra-core", "blind-watermark", "qrcode", "pyzbar", "psd-tools", "wandb", "zhipuai", "openai"] diff --git a/workflow/collage_example.json b/workflow/collage_example.json new file mode 100644 index 0000000..d5139f4 --- /dev/null +++ b/workflow/collage_example.json @@ -0,0 +1,275 @@ +{ + "last_node_id": 14, + "last_link_id": 14, + "nodes": [ + { + "id": 6, + "type": "LayerMask: LoadFlorence2Model", + "pos": [ + 702.1195068359375, + 804.4620971679688 + ], + "size": [ + 378.0740966796875, + 62.4444580078125 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "florence2_model", + "type": "FLORENCE2", + "links": [ + 14 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LayerMask: LoadFlorence2Model" + }, + "widgets_values": [ + "base" + ], + "color": "rgba(27, 80, 119, 0.7)" + }, + { + "id": 1, + "type": "LayerUtility: Collage", + "pos": [ + 699.5286254882812, + 926.8685913085938 + ], + "size": [ + 378, + 270 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 12 + }, + { + "name": "florence2_model", + "type": "FLORENCE2", + "link": 14, + "shape": 7 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 7 + ], + "slot_index": 0 + }, + { + "name": "mask", + "type": "MASK", + "links": [ + 8 + ], + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "LayerUtility: Collage" + }, + "widgets_values": [ + 2048, + 2048, + 1, + 20, + 1, + "#FFFFFF", + 1078302422640976, + "randomize", + "subject" + ], + "color": "rgba(38, 73, 116, 0.7)" + }, + { + "id": 9, + "type": "PreviewImage", + "pos": [ + 1151.0084228515625, + 604.7581787109375 + ], + "size": [ + 598.259765625, + 317.0512390136719 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 7 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 10, + "type": "LayerMask: MaskPreview", + "pos": [ + 1152.8602294921875, + 972.933349609375 + ], + "size": [ + 593.96630859375, + 310.0398864746094 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "mask", + "type": "MASK", + "link": 8 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "LayerMask: MaskPreview" + }, + "widgets_values": [], + "color": "rgba(27, 80, 119, 0.7)" + }, + { + "id": 13, + "type": "VHS_LoadImagesPath", + "pos": [ + 397.5517883300781, + 821.9283447265625 + ], + "size": [ + 242.24609375, + 194 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 12 + ], + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null + }, + { + "name": "frame_count", + "type": "INT", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_LoadImagesPath" + }, + "widgets_values": { + "directory": "c:\\images", + "image_load_cap": 0, + "skip_first_images": 0, + "select_every_nth": 1, + "choose folder to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "filename": "c:\\images", + "type": "path", + "format": "folder" + }, + "muted": false + } + } + } + ], + "links": [ + [ + 7, + 1, + 0, + 9, + 0, + "IMAGE" + ], + [ + 8, + 1, + 1, + 10, + 0, + "MASK" + ], + [ + 12, + 13, + 0, + 1, + 0, + "IMAGE" + ], + [ + 14, + 6, + 0, + 1, + 1, + "FLORENCE2" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 1.2100000000000002, + "offset": [ + -33.25166101546846, + -396.5863340766634 + ] + }, + "node_versions": { + "ComfyUI_LayerStyle_Advance": "7fdcbce0727a541efcd3ff393a099b3f0fa52d33", + "comfy-core": "0.3.12", + "ComfyUI_LayerStyle": "3bf7244fa652322c3307609b928f7aed8c3b3707", + "ComfyUI-VideoHelperSuite": "c47b10ca1798b4925ff5a5f07d80c51ca80a837d" + }, + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0 + }, + "version": 0.4 +} \ No newline at end of file