nodes/Jannchie-ComfyUI-J
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

✨ feat(nodes): add inpaint, controlnet, reference only support

Browse Source
This commit is contained in:
Jianqi Pan
2024-03-21 22:47:24 +09:00
parent c1f09d5a9a
commit 57a2772909
4 changed files with 2189 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+3
View File
@@ -1,3 +1,6 @@
# ComfyUI-J
Jannchie's ComfyUI custom nodes.
This is a completely different set of nodes than Comfy's own KSampler series.
This set of nodes is based on Diffusers, which makes it easier to import models, apply prompts with weights, inpaint, reference only, controlnet, etc.
__init__.py
+25 -11
View File
@@ -112,7 +112,6 @@ def latents_to_mask_tensor(pipeline, latents):
def prepare_latents(
pipe: StableDiffusionPipeline,
batch_size: int,
num_channels_latents: int,
height: int,
width: int,
dtype: torch.dtype,
@@ -122,7 +121,7 @@ def prepare_latents(
):
shape = (
batch_size,
num_channels_latents,
pipe.unet.config.in_channels,
height // pipe.vae_scale_factor,
width // pipe.vae_scale_factor,
)
@@ -146,7 +145,6 @@ def prepare_image(
pipeline: StableDiffusionPipeline,
seed=47,
batch_size=1,
num_channels_latents=4,
height=512,
width=512,
):
@@ -155,7 +153,6 @@ def prepare_image(
latents = prepare_latents(
pipe=pipeline,
batch_size=batch_size,
num_channels_latents=num_channels_latents,
height=height,
width=width,
generator=generator,
@@ -432,7 +429,6 @@ class DiffusersPrepareLatents:
latents = prepare_latents(
pipe=pipeline,
batch_size=batch_size,
num_channels_latents=4,
height=height,
width=width,
dtype=comfy.model_management.VAE_DTYPE,
@@ -627,15 +623,27 @@ class DiffusersGenerator:
"step": 64,
},
),
"num_channels_latents": (
"INT",
{"default": 4, "min": 1, "max": 4, "step": 1},
),
},
"optional": {
"images": ("IMAGE",),
"mask": ("MASK",),
"controlnet_units": ("CONTROLNET_UNIT",),
"reference_image": (
"IMAGE",
{"default": None},
),
"reference_only": (
["disable", "enable"],
{
"default": "disable",
},
),
"reference_only_adain": (
["disable", "enable"],
{
"default": "disable",
},
),
},
}
@@ -650,12 +658,16 @@ class DiffusersGenerator:
images: torch.Tensor | None = None,
num_inference_steps: int = 30,
strength: float = 1.0,
num_channels_latents: int = 4,
guidance_scale: float = 7.0,
controlnet_units: tuple[ControlNetUnit] = None,
seed=None,
mask: torch.Tensor | None = None,
reference_only: str = "disable",
reference_only_adain: str = "disable",
reference_image: torch.Tensor | None = None,
):
reference_only = reference_only == "enable"
reference_only_adain = reference_only_adain == "enable"
latents = None
pbar = ProgressBar(int(num_inference_steps * strength))
device = comfy.model_management.get_torch_device()
@@ -668,7 +680,6 @@ class DiffusersGenerator:
latents = prepare_latents(
pipe=pipeline,
batch_size=batch_size,
num_channels_latents=num_channels_latents,
height=height,
width=width,
generator=generator,
@@ -697,6 +708,7 @@ class DiffusersGenerator:
result = pipeline(
image=images,
mask_image=mask,
ref_image=reference_image if reference_image is not None else images,
generator=generator,
width=width,
height=height,
@@ -708,6 +720,8 @@ class DiffusersGenerator:
strength=strength,
controlnet_units=controlnet_units,
callback=callback,
reference_attn=reference_only,
reference_adain=reference_only_adain,
return_dict=True,
)
# image = result["images"][0]
examples/controlnet.json
+475
View File
@@ -0,0 +1,475 @@
{
"last_node_id": 13,
"last_link_id": 28,
"nodes": [
{
"id": 7,
"type": "LoadImage",
"pos": [
110,
470
],
"size": {
"0": 370.4840393066406,
"1": 537.7867431640625
},
"flags": {},
"order": 0,
"mode": 0,
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
17
],
"shape": 3,
"slot_index": 0
},
{
"name": "MASK",
"type": "MASK",
"links": null,
"shape": 3
}
],
"properties": {
"Node name for S&R": "LoadImage"
},
"widgets_values": [
"image.png",
"image"
],
"shape": 1
},
{
"id": 1,
"type": "DiffusersPipeline",
"pos": [
110,
310
],
"size": {
"0": 370,
"1": 110
},
"flags": {},
"order": 1,
"mode": 0,
"outputs": [
{
"name": "pipeline",
"type": "DIFFUSERS_PIPELINE",
"links": [
2,
24
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "DiffusersPipeline"
},
"widgets_values": [
"picxReal_10.safetensors",
"vae-ft-mse-840000-ema-pruned.safetensors",
"-"
],
"shape": 1
},
{
"id": 3,
"type": "DiffusersCompelPromptEmbedding",
"pos": [
720,
80
],
"size": {
"0": 430.8000183105469,
"1": 200
},
"flags": {},
"order": 4,
"mode": 0,
"inputs": [
{
"name": "pipeline",
"type": "DIFFUSERS_PIPELINE",
"link": 2
}
],
"outputs": [
{
"name": "positive prompt embedding",
"type": "DIFFUSERS_PROMPT_EMBEDDING",
"links": [
22
],
"shape": 3,
"slot_index": 0
},
{
"name": "negative prompt embedding",
"type": "DIFFUSERS_PROMPT_EMBEDDING",
"links": [
23
],
"shape": 3,
"slot_index": 1
}
],
"properties": {
"Node name for S&R": "DiffusersCompelPromptEmbedding"
},
"widgets_values": [
"(masterpiece)1.2, (best quality)1.4, ",
""
],
"shape": 1
},
{
"id": 10,
"type": "DiffusersControlnetLoader",
"pos": [
510,
480
],
"size": {
"0": 340,
"1": 60
},
"flags": {},
"order": 2,
"mode": 0,
"outputs": [
{
"name": "controlnet",
"type": "DIFFUSERS_CONTROLNET",
"links": [
20
],
"shape": 3
}
],
"properties": {
"Node name for S&R": "DiffusersControlnetLoader"
},
"widgets_values": [
"control_v11p_sd15_openpose_fp16.safetensors"
],
"shape": 1
},
{
"id": 9,
"type": "DiffusersControlnetUnit",
"pos": [
960,
480
],
"size": {
"0": 330,
"1": 126
},
"flags": {},
"order": 5,
"mode": 0,
"inputs": [
{
"name": "controlnet",
"type": "DIFFUSERS_CONTROLNET",
"link": 20,
"slot_index": 0
},
{
"name": "image",
"type": "IMAGE",
"link": 18
}
],
"outputs": [
{
"name": "controlnet unit",
"type": "CONTROLNET_UNIT",
"links": [
25
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "DiffusersControlnetUnit"
},
"widgets_values": [
1,
0,
1
],
"shape": 1
},
{
"id": 8,
"type": "OpenposePreprocessor",
"pos": [
510,
620
],
"size": {
"0": 360,
"1": 150
},
"flags": {},
"order": 3,
"mode": 0,
"inputs": [
{
"name": "image",
"type": "IMAGE",
"link": 17
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
18,
28
],
"shape": 3,
"slot_index": 0
},
{
"name": "POSE_KEYPOINT",
"type": "POSE_KEYPOINT",
"links": null,
"shape": 3
}
],
"properties": {
"Node name for S&R": "OpenposePreprocessor"
},
"widgets_values": [
"enable",
"enable",
"enable",
512
],
"shape": 1
},
{
"id": 13,
"type": "Image Comparer (rgthree)",
"pos": {
"0": 1880,
"1": 600,
"2": 0,
"3": 0,
"4": 0,
"5": 0,
"6": 0,
"7": 0,
"8": 0,
"9": 0
},
"size": {
"0": 603.9534301757812,
"1": 646.658935546875
},
"flags": {},
"order": 7,
"mode": 0,
"inputs": [
{
"name": "image_a",
"type": "IMAGE",
"link": 27,
"dir": 3
},
{
"name": "image_b",
"type": "IMAGE",
"link": 28,
"dir": 3
}
],
"outputs": [],
"properties": {
"comparer_mode": "Slide"
},
"widgets_values": [
[
"/view?filename=rgthree.compare._temp_dbajh_00019_.png&type=temp&subfolder=&rand=0.6031372112098188",
"/view?filename=rgthree.compare._temp_dbajh_00020_.png&type=temp&subfolder=&rand=0.1277051032478802"
]
],
"shape": 1
},
{
"id": 12,
"type": "DiffusersGenerator",
"pos": [
1380,
310
],
"size": {
"0": 405.5999755859375,
"1": 394
},
"flags": {},
"order": 6,
"mode": 0,
"inputs": [
{
"name": "pipeline",
"type": "DIFFUSERS_PIPELINE",
"link": 24
},
{
"name": "positive_prompt_embedding",
"type": "DIFFUSERS_PROMPT_EMBEDDING",
"link": 22
},
{
"name": "negative_prompt_embedding",
"type": "DIFFUSERS_PROMPT_EMBEDDING",
"link": 23
},
{
"name": "images",
"type": "IMAGE",
"link": null
},
{
"name": "mask",
"type": "MASK",
"link": null
},
{
"name": "controlnet_units",
"type": "CONTROLNET_UNIT",
"link": 25
},
{
"name": "reference_image",
"type": "IMAGE",
"link": null
}
],
"outputs": [
{
"name": "images",
"type": "IMAGE",
"links": [
27
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "DiffusersGenerator"
},
"widgets_values": [
1,
30,
7,
711606850977,
"randomize",
1,
512,
512,
"disable",
"disable"
],
"shape": 1
}
],
"links": [
[
2,
1,
0,
3,
0,
"DIFFUSERS_PIPELINE"
],
[
17,
7,
0,
8,
0,
"IMAGE"
],
[
18,
8,
0,
9,
1,
"IMAGE"
],
[
20,
10,
0,
9,
0,
"DIFFUSERS_CONTROLNET"
],
[
22,
3,
0,
12,
1,
"DIFFUSERS_PROMPT_EMBEDDING"
],
[
23,
3,
1,
12,
2,
"DIFFUSERS_PROMPT_EMBEDDING"
],
[
24,
1,
0,
12,
0,
"DIFFUSERS_PIPELINE"
],
[
25,
9,
0,
12,
5,
"CONTROLNET_UNIT"
],
[
27,
12,
0,
13,
0,
"IMAGE"
],
[
28,
8,
0,
13,
1,
"IMAGE"
]
],
"groups": [],
"config": {},
"extra": {},
"version": 0.4
}
pipelines/jannchie.py
+1686
View File
File diff suppressed because it is too large Load Diff