diff --git a/README.md b/README.md index c5dd88f..0ff8274 100644 --- a/README.md +++ b/README.md @@ -4,3 +4,54 @@ Jannchie's ComfyUI custom nodes. This is a completely different set of nodes than Comfy's own KSampler series. This set of nodes is based on Diffusers, which makes it easier to import models, apply prompts with weights, inpaint, reference only, controlnet, etc. + +## Installation + +In the `custom_nodes` directory, run + +```bash +git clone github.com/Jannchie/ComfyUI-J +cd ComfyUI-J +pip install -r requirements.txt +``` + +## Examples + +### Base Usage of Jannchie's Diffusers Pipeline + +![Base Usage](./examples/base.png) + +### Reference Only with Jannchie's Diffusers Pipeline + +![Reference only](./examples/reference_only.png) + +### ControlNet with Jannchie's Diffusers Pipeline + +![ControlNet](./examples/controlnet.png) + +## Inpainting with Jannchie's Diffusers Pipeline + +![Inpainting](./examples/inpainting.png) + +## Remove something with Jannchie's Diffusers Pipeline + +![Remove something](./examples/remove_something.png) + +## Change Clothes with Jannchie's Diffusers Pipeline + +This is a composite application of diffusers pipeline custom node. Includes: + +- Reference only +- ControlNet +- Inpainting +- Texture Inversion + +This is a demonstration of a simple workflow for properly dressing a character. + +A checkpoint for stablediffusion 1.5 is all your need. But for full automation, I use the Comfyui_segformer_b2_clothes custom node for generating masks. you can draw your own masks without it. + +![Change Clothes](./examples/change_clothes.png) + +## TODO + +- [ ] Add LoRA support diff --git a/examples/Inpainting.png b/examples/Inpainting.png new file mode 100644 index 0000000..9a789c7 Binary files /dev/null and b/examples/Inpainting.png differ diff --git a/examples/base.png b/examples/base.png new file mode 100644 index 0000000..e26cb86 Binary files /dev/null and b/examples/base.png differ diff --git a/examples/change_clothes.png b/examples/change_clothes.png new file mode 100644 index 0000000..24db7f9 Binary files /dev/null and b/examples/change_clothes.png differ diff --git a/examples/controlnet.json b/examples/controlnet.json deleted file mode 100644 index c86e08c..0000000 --- a/examples/controlnet.json +++ /dev/null @@ -1,475 +0,0 @@ -{ - "last_node_id": 14, - "last_link_id": 33, - "nodes": [ - { - "id": 1, - "type": "DiffusersPipeline", - "pos": [ - 110, - 310 - ], - "size": { - "0": 370, - "1": 110 - }, - "flags": {}, - "order": 0, - "mode": 0, - "outputs": [ - { - "name": "pipeline", - "type": "DIFFUSERS_PIPELINE", - "links": [ - 2, - 29 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DiffusersPipeline" - }, - "widgets_values": [ - "picxReal_10.safetensors", - "vae-ft-mse-840000-ema-pruned.safetensors", - "-" - ], - "shape": 1 - }, - { - "id": 3, - "type": "DiffusersCompelPromptEmbedding", - "pos": [ - 720, - 80 - ], - "size": { - "0": 430.8000183105469, - "1": 200 - }, - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [ - { - "name": "pipeline", - "type": "DIFFUSERS_PIPELINE", - "link": 2 - } - ], - "outputs": [ - { - "name": "positive prompt embedding", - "type": "DIFFUSERS_PROMPT_EMBEDDING", - "links": [ - 30 - ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "negative prompt embedding", - "type": "DIFFUSERS_PROMPT_EMBEDDING", - "links": [ - 31 - ], - "shape": 3, - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "DiffusersCompelPromptEmbedding" - }, - "widgets_values": [ - "(masterpiece)1.2, (best quality)1.4, ", - "" - ], - "shape": 1 - }, - { - "id": 10, - "type": "DiffusersControlnetLoader", - "pos": [ - 510, - 480 - ], - "size": { - "0": 340, - "1": 60 - }, - "flags": {}, - "order": 1, - "mode": 0, - "outputs": [ - { - "name": "controlnet", - "type": "DIFFUSERS_CONTROLNET", - "links": [ - 20 - ], - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "DiffusersControlnetLoader" - }, - "widgets_values": [ - "control_v11p_sd15_openpose_fp16.safetensors" - ], - "shape": 1 - }, - { - "id": 9, - "type": "DiffusersControlnetUnit", - "pos": [ - 960, - 480 - ], - "size": { - "0": 330, - "1": 126 - }, - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [ - { - "name": "controlnet", - "type": "DIFFUSERS_CONTROLNET", - "link": 20, - "slot_index": 0 - }, - { - "name": "image", - "type": "IMAGE", - "link": 18 - } - ], - "outputs": [ - { - "name": "controlnet unit", - "type": "CONTROLNET_UNIT", - "links": [ - 32 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DiffusersControlnetUnit" - }, - "widgets_values": [ - 1, - 0, - 1 - ], - "shape": 1 - }, - { - "id": 8, - "type": "OpenposePreprocessor", - "pos": [ - 510, - 620 - ], - "size": { - "0": 360, - "1": 150 - }, - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [ - { - "name": "image", - "type": "IMAGE", - "link": 17 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 18, - 28 - ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "POSE_KEYPOINT", - "type": "POSE_KEYPOINT", - "links": null, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "OpenposePreprocessor" - }, - "widgets_values": [ - "enable", - "enable", - "enable", - 512 - ], - "shape": 1 - }, - { - "id": 13, - "type": "Image Comparer (rgthree)", - "pos": { - "0": 1880, - "1": 600, - "2": 0, - "3": 0, - "4": 0, - "5": 0, - "6": 0, - "7": 0, - "8": 0, - "9": 0 - }, - "size": { - "0": 603.9534301757812, - "1": 646.658935546875 - }, - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [ - { - "name": "image_a", - "type": "IMAGE", - "link": 33, - "dir": 3 - }, - { - "name": "image_b", - "type": "IMAGE", - "link": 28, - "dir": 3 - } - ], - "outputs": [], - "properties": { - "comparer_mode": "Slide" - }, - "widgets_values": [ - [ - "/view?filename=rgthree.compare._temp_pknnj_00003_.png&type=temp&subfolder=&rand=0.7251475711869597", - "/view?filename=rgthree.compare._temp_pknnj_00004_.png&type=temp&subfolder=&rand=0.5900207825118742" - ] - ], - "shape": 1 - }, - { - "id": 14, - "type": "DiffusersGenerator", - "pos": [ - 1380, - 310 - ], - "size": { - "0": 405.5999755859375, - "1": 418 - }, - "flags": {}, - "order": 6, - "mode": 0, - "inputs": [ - { - "name": "pipeline", - "type": "DIFFUSERS_PIPELINE", - "link": 29 - }, - { - "name": "positive_prompt_embedding", - "type": "DIFFUSERS_PROMPT_EMBEDDING", - "link": 30 - }, - { - "name": "negative_prompt_embedding", - "type": "DIFFUSERS_PROMPT_EMBEDDING", - "link": 31 - }, - { - "name": "images", - "type": "IMAGE", - "link": null - }, - { - "name": "mask", - "type": "MASK", - "link": null - }, - { - "name": "controlnet_units", - "type": "CONTROLNET_UNIT", - "link": 32 - }, - { - "name": "reference_image", - "type": "IMAGE", - "link": null - } - ], - "outputs": [ - { - "name": "images", - "type": "IMAGE", - "links": [ - 33 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DiffusersGenerator" - }, - "widgets_values": [ - 1, - 30, - 7, - 127993911908, - "randomize", - 1, - 512, - 512, - 0.5, - "disable", - "disable" - ] - }, - { - "id": 7, - "type": "LoadImage", - "pos": [ - 110, - 470 - ], - "size": { - "0": 370.4840393066406, - "1": 537.7867431640625 - }, - "flags": {}, - "order": 2, - "mode": 0, - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 17 - ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": null, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "image_jjZSuko8_1701021290044_raw.webp", - "image" - ], - "shape": 1 - } - ], - "links": [ - [ - 2, - 1, - 0, - 3, - 0, - "DIFFUSERS_PIPELINE" - ], - [ - 17, - 7, - 0, - 8, - 0, - "IMAGE" - ], - [ - 18, - 8, - 0, - 9, - 1, - "IMAGE" - ], - [ - 20, - 10, - 0, - 9, - 0, - "DIFFUSERS_CONTROLNET" - ], - [ - 28, - 8, - 0, - 13, - 1, - "IMAGE" - ], - [ - 29, - 1, - 0, - 14, - 0, - "DIFFUSERS_PIPELINE" - ], - [ - 30, - 3, - 0, - 14, - 1, - "DIFFUSERS_PROMPT_EMBEDDING" - ], - [ - 31, - 3, - 1, - 14, - 2, - "DIFFUSERS_PROMPT_EMBEDDING" - ], - [ - 32, - 9, - 0, - 14, - 5, - "CONTROLNET_UNIT" - ], - [ - 33, - 14, - 0, - 13, - 0, - "IMAGE" - ] - ], - "groups": [], - "config": {}, - "extra": {}, - "version": 0.4 -} \ No newline at end of file diff --git a/examples/controlnet.png b/examples/controlnet.png new file mode 100644 index 0000000..ed69835 Binary files /dev/null and b/examples/controlnet.png differ diff --git a/examples/reference_only.png b/examples/reference_only.png new file mode 100644 index 0000000..bc61560 Binary files /dev/null and b/examples/reference_only.png differ diff --git a/examples/remove_something.png b/examples/remove_something.png new file mode 100644 index 0000000..d7e0aef Binary files /dev/null and b/examples/remove_something.png differ diff --git a/pipelines/__init__.py b/pipelines/__init__.py index 2a09260..0437237 100644 --- a/pipelines/__init__.py +++ b/pipelines/__init__.py @@ -1,3 +1,5 @@ +import contextlib + from diffusers import AutoencoderKL, StableDiffusionImg2ImgPipeline from diffusers.schedulers import ( DEISMultistepScheduler, @@ -38,6 +40,7 @@ schedulers = { "UniPC": UniPCMultistepScheduler(), } + class PipelineWrapper: def __init__( @@ -71,3 +74,5 @@ class PipelineWrapper: self.pipeline.scheduler = scheduler self.pipeline.to(device) self.pipeline.safety_checker = None + with contextlib.suppress(Exception): + self.pipeline.enable_xformers_memory_efficient_attention() diff --git a/pipelines/jannchie.py b/pipelines/jannchie.py index 994c8c8..e9b89a4 100644 --- a/pipelines/jannchie.py +++ b/pipelines/jannchie.py @@ -1212,8 +1212,6 @@ class BasicTransformerBlockReferenceOnly(BasicTransformerBlock): assert isinstance(bank, list) uc_mask = ref_data.uc_mask - bool_mask = ref_data.bool_mask - ref_mask_dict = ref_data.ref_mask_dict if self.use_ada_layer_norm: norm_hidden_states = self.norm1(hidden_states, timestep)