nodes/brianfitzgerald-style_aligned_comfy
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

docs update

Browse Source
This commit is contained in:
brianf
2023-12-13 01:19:10 +00:00
parent c4f912b310
commit 7b2eba49e3
6 changed files with 1201 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+28 -18
View File
@@ -2,27 +2,37 @@
Implementation of the [StyleAligned](https://style-aligned-gen.github.io/) paper for ComfyUI.
This node allows you to apply a consistent style to all images in a batch; by default it will use the first image in the batch as the style reference, forcing all other images to be consistent with it.
This implementation is split into two different nodes:
![](resources/header.jpg)
#### StyleAligned Reference Sampler
_A batch of generations with the same parameters, with the node applied (left) and without (right)._
This node replaces the KSampler, and lets you reference an existing latent as a style reference. In order to retrieve the latent, you will need to perform DDIM inversion; an example workflow for this is provided [here](resources/style_aligned_inversion.json).
In the next few days I plan on implementing the second feature of the paper - the ability to use another image as the style reference.
![](resources/reference_sampler.png)
_Above, a reference image, and a batch of images generated using the prompt 'a robot' and the reference image shown as style input._
##### Parameters
- `model`: The base model to patch.
- `share_norm`: Whether to share normalization across the batch. Defaults to `both`. Set to `group` or `layer` to only share group or layer normalization, respectively.
- `scale`: The scale at which to apply the style-alignment effect. Defaults to `1`.
#### StyleAligned Batch Align
Instead of referencing a single latent, this node aligns the style of the entire batch with the first image generated in the batch, effectively causing all images in the batch to be generated with the same style.
![](resources/batch_align.jpg)
_A batch of generations with the same parameters and the Batch Align node applied (left) and disabled (right)._
##### Parameters
- `model`: The base model to patch.
- `share_norm`: Whether to share normalization across the batch. Defaults to `both`. Set to `group` or `layer` to only share group or layer normalization, respectively.
- `scale`: The scale at which to apply the style-alignment effect. Defaults to `1`.
- `batch_size`, `noise_seed`, `control_after_generate`, `cfg`: Identical to the standard `KSampler` parameters.
### Installation
Simply download or git clone this repository in `ComfyUI/custom_nodes/`.
### Usage
Use the example workflow from [here](resources/example_workflow.json).
Or, simply add the `StyleAlignedPatch` node after `LoadCheckpoint`.
### Parameters
- `model`: Required, the base model to patch.
- `style_image`: (**not implemented yet!**) Optional, path to the latent to use as a style inference. If left blank, the first latent in the batch will be used, effectively making the output of the batch consistent.
- `share_norm`: Whether to share normalization across the batch. Defaults to `both`. Set to `group` or `layer` to only share group or layer normalization, respectively.
- `scale`: The scale at which to apply the style-alignment effect. Defaults to `1`.
Simply download or git clone this repository in `ComfyUI/custom_nodes/`.
__init__.py
+6 -6
View File
@@ -144,7 +144,7 @@ def register_shared_norm(
SHARE_NORM_OPTIONS = ["both", "group", "layer", "disabled"]
class StyleAlignWithRefSampler:
class StyleAlignedReferenceSampler:
@classmethod
def INPUT_TYPES(cls):
return {
@@ -254,7 +254,7 @@ class StyleAlignWithRefSampler:
return (out, out_denoised)
class StyleAlignWithinBatch:
class StyleAlignedBatchAlign:
@classmethod
def INPUT_TYPES(cls):
return {
@@ -287,12 +287,12 @@ class StyleAlignWithinBatch:
NODE_CLASS_MAPPINGS = {
"StyleAlignWithRefSampler": StyleAlignWithRefSampler,
"StyleAlignWithinBatch": StyleAlignWithinBatch,
"StyleAlignedReferenceSampler": StyleAlignedReferenceSampler,
"StyleAlignedBatchAlign": StyleAlignedBatchAlign,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"StyleAlignWithRefSampler": "StyleAligned Reference Sampler",
"StyleAlignWithinBatch": "StyleAligned Within Batch",
"StyleAlignedReferenceSampler": "StyleAligned Reference Sampler",
"StyleAlignedBatchAlign": "StyleAligned Batch Align",
}
resources/header.jpg → resources/batch_align.jpg
View File
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 174 KiB

After

Width:  |  Height:  |  Size: 174 KiB

resources/reference_sampler.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 3.7 MiB

resources/style_aligned_inversion.json
+980
View File
@@ -0,0 +1,980 @@
{
"last_node_id": 45,
"last_link_id": 148,
"nodes": [
{
"id": 13,
"type": "FlipSigmas",
"pos": [
81,
907
],
"size": {
"0": 210,
"1": 26
},
"flags": {},
"order": 9,
"mode": 0,
"inputs": [
{
"name": "sigmas",
"type": "SIGMAS",
"link": 14,
"slot_index": 0
}
],
"outputs": [
{
"name": "SIGMAS",
"type": "SIGMAS",
"links": [
13
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "FlipSigmas"
}
},
{
"id": 20,
"type": "VAEEncode",
"pos": [
424,
1105
],
"size": {
"0": 210,
"1": 46
},
"flags": {},
"order": 11,
"mode": 0,
"inputs": [
{
"name": "pixels",
"type": "IMAGE",
"link": 43,
"slot_index": 0
},
{
"name": "vae",
"type": "VAE",
"link": 22
}
],
"outputs": [
{
"name": "LATENT",
"type": "LATENT",
"links": [
25
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "VAEEncode"
}
},
{
"id": 23,
"type": "ImageScale",
"pos": [
-128,
1321
],
"size": {
"0": 315,
"1": 130
},
"flags": {},
"order": 8,
"mode": 0,
"inputs": [
{
"name": "image",
"type": "IMAGE",
"link": 42
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
43
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "ImageScale"
},
"widgets_values": [
"nearest-exact",
512,
512,
"center"
]
},
{
"id": 38,
"type": "EmptyLatentImage",
"pos": [
988,
145
],
"size": {
"0": 315,
"1": 106
},
"flags": {},
"order": 0,
"mode": 0,
"outputs": [
{
"name": "LATENT",
"type": "LATENT",
"links": [
122
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "EmptyLatentImage"
},
"widgets_values": [
512,
512,
4
]
},
{
"id": 4,
"type": "CheckpointLoaderSimple",
"pos": [
-305,
251
],
"size": {
"0": 315,
"1": 98
},
"flags": {},
"order": 1,
"mode": 0,
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
20,
24
],
"slot_index": 0
},
{
"name": "CLIP",
"type": "CLIP",
"links": [
3,
5
],
"slot_index": 1
},
{
"name": "VAE",
"type": "VAE",
"links": [
8,
16,
22,
117
],
"slot_index": 2
}
],
"properties": {
"Node name for S&R": "CheckpointLoaderSimple"
},
"widgets_values": [
"sd_xl_base_1.0.safetensors"
]
},
{
"id": 36,
"type": "VAEDecode",
"pos": [
1446,
84
],
"size": {
"0": 210,
"1": 46
},
"flags": {},
"order": 7,
"mode": 0,
"inputs": [
{
"name": "samples",
"type": "LATENT",
"link": 122,
"slot_index": 0
},
{
"name": "vae",
"type": "VAE",
"link": 117
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
118
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "VAEDecode"
}
},
{
"id": 37,
"type": "PreviewImage",
"pos": [
1683,
45
],
"size": {
"0": 210,
"1": 246
},
"flags": {},
"order": 10,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 118
}
],
"properties": {
"Node name for S&R": "PreviewImage"
}
},
{
"id": 17,
"type": "VAEDecode",
"pos": [
1313,
1013
],
"size": {
"0": 210,
"1": 46
},
"flags": {},
"order": 14,
"mode": 0,
"inputs": [
{
"name": "samples",
"type": "LATENT",
"link": 136,
"slot_index": 0
},
{
"name": "vae",
"type": "VAE",
"link": 16
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
135
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "VAEDecode"
}
},
{
"id": 43,
"type": "PreviewImage",
"pos": [
1604,
1039
],
"size": {
"0": 210,
"1": 246
},
"flags": {},
"order": 16,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 135
}
],
"properties": {
"Node name for S&R": "PreviewImage"
}
},
{
"id": 8,
"type": "VAEDecode",
"pos": [
1436,
449
],
"size": {
"0": 210,
"1": 46
},
"flags": {},
"order": 15,
"mode": 0,
"inputs": [
{
"name": "samples",
"type": "LATENT",
"link": 144
},
{
"name": "vae",
"type": "VAE",
"link": 8
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
10
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "VAEDecode"
}
},
{
"id": 15,
"type": "SamplerCustom",
"pos": [
655,
846
],
"size": {
"0": 355.20001220703125,
"1": 442
},
"flags": {},
"order": 12,
"mode": 0,
"inputs": [
{
"name": "model",
"type": "MODEL",
"link": 20
},
{
"name": "positive",
"type": "CONDITIONING",
"link": 60
},
{
"name": "negative",
"type": "CONDITIONING",
"link": 65,
"slot_index": 2
},
{
"name": "sampler",
"type": "SAMPLER",
"link": 21,
"slot_index": 3
},
{
"name": "sigmas",
"type": "SIGMAS",
"link": 13
},
{
"name": "latent_image",
"type": "LATENT",
"link": 25
}
],
"outputs": [
{
"name": "output",
"type": "LATENT",
"links": [
143
],
"shape": 3,
"slot_index": 0
},
{
"name": "denoised_output",
"type": "LATENT",
"links": [
136
],
"shape": 3,
"slot_index": 1
}
],
"properties": {
"Node name for S&R": "SamplerCustom"
},
"widgets_values": [
false,
156680208709678,
"fixed",
1.1
]
},
{
"id": 21,
"type": "LoadImage",
"pos": [
1891,
1018
],
"size": {
"0": 480.172607421875,
"1": 588.647705078125
},
"flags": {},
"order": 2,
"mode": 0,
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
42
],
"shape": 3,
"slot_index": 0
},
{
"name": "MASK",
"type": "MASK",
"links": null,
"shape": 3
}
],
"properties": {
"Node name for S&R": "LoadImage"
},
"widgets_values": [
"0_1-1.png",
"image"
]
},
{
"id": 10,
"type": "PreviewImage",
"pos": [
1893,
452
],
"size": {
"0": 485.727294921875,
"1": 517.061767578125
},
"flags": {},
"order": 17,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 10
}
],
"properties": {
"Node name for S&R": "PreviewImage"
}
},
{
"id": 45,
"type": "StyleAlignedReferenceSampler",
"pos": [
1058,
514
],
"size": {
"0": 330,
"1": 278
},
"flags": {},
"order": 13,
"mode": 0,
"inputs": [
{
"name": "model",
"type": "MODEL",
"link": null
},
{
"name": "positive",
"type": "CONDITIONING",
"link": 145
},
{
"name": "negative",
"type": "CONDITIONING",
"link": 146
},
{
"name": "sampler",
"type": "SAMPLER",
"link": 148
},
{
"name": "sigmas",
"type": "SIGMAS",
"link": 147
},
{
"name": "ref_latent",
"type": "LATENT",
"link": 143,
"slot_index": 5
}
],
"outputs": [
{
"name": "output",
"type": "LATENT",
"links": null,
"shape": 3
},
{
"name": "denoised_output",
"type": "LATENT",
"links": [
144
],
"shape": 3,
"slot_index": 1
}
],
"properties": {
"Node name for S&R": "StyleAlignedReferenceSampler"
},
"widgets_values": [
"both",
1,
2,
0,
"randomize",
8
]
},
{
"id": 7,
"type": "CLIPTextEncode",
"pos": [
168,
643
],
"size": {
"0": 425.27801513671875,
"1": 180.6060791015625
},
"flags": {},
"order": 6,
"mode": 0,
"inputs": [
{
"name": "clip",
"type": "CLIP",
"link": 5
}
],
"outputs": [
{
"name": "CONDITIONING",
"type": "CONDITIONING",
"links": [
65,
146
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "CLIPTextEncode"
},
"widgets_values": [
""
]
},
{
"id": 6,
"type": "CLIPTextEncode",
"pos": [
368,
159
],
"size": {
"0": 422.84503173828125,
"1": 164.31304931640625
},
"flags": {},
"order": 5,
"mode": 0,
"inputs": [
{
"name": "clip",
"type": "CLIP",
"link": 3
}
],
"outputs": [
{
"name": "CONDITIONING",
"type": "CONDITIONING",
"links": [
60,
145
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "CLIPTextEncode"
},
"widgets_values": [
"Scene from le heine, in full colour, Kodak, soho, London, night, Neo - noir, noir, mist, moody\n\n"
]
},
{
"id": 19,
"type": "KSamplerSelect",
"pos": [
471,
387
],
"size": {
"0": 315,
"1": 58
},
"flags": {},
"order": 3,
"mode": 0,
"outputs": [
{
"name": "SAMPLER",
"type": "SAMPLER",
"links": [
21,
148
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "KSamplerSelect"
},
"widgets_values": [
"ddim"
]
},
{
"id": 16,
"type": "BasicScheduler",
"pos": [
471,
524
],
"size": {
"0": 315,
"1": 82
},
"flags": {},
"order": 4,
"mode": 0,
"inputs": [
{
"name": "model",
"type": "MODEL",
"link": 24,
"slot_index": 0
}
],
"outputs": [
{
"name": "SIGMAS",
"type": "SIGMAS",
"links": [
14,
147
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "BasicScheduler"
},
"widgets_values": [
"ddim_uniform",
32
]
}
],
"links": [
[
3,
4,
1,
6,
0,
"CLIP"
],
[
5,
4,
1,
7,
0,
"CLIP"
],
[
8,
4,
2,
8,
1,
"VAE"
],
[
10,
8,
0,
10,
0,
"IMAGE"
],
[
13,
13,
0,
15,
4,
"SIGMAS"
],
[
14,
16,
0,
13,
0,
"SIGMAS"
],
[
16,
4,
2,
17,
1,
"VAE"
],
[
20,
4,
0,
15,
0,
"MODEL"
],
[
21,
19,
0,
15,
3,
"SAMPLER"
],
[
22,
4,
2,
20,
1,
"VAE"
],
[
24,
4,
0,
16,
0,
"MODEL"
],
[
25,
20,
0,
15,
5,
"LATENT"
],
[
42,
21,
0,
23,
0,
"IMAGE"
],
[
43,
23,
0,
20,
0,
"IMAGE"
],
[
60,
6,
0,
15,
1,
"CONDITIONING"
],
[
65,
7,
0,
15,
2,
"CONDITIONING"
],
[
117,
4,
2,
36,
1,
"VAE"
],
[
118,
36,
0,
37,
0,
"IMAGE"
],
[
122,
38,
0,
36,
0,
"LATENT"
],
[
135,
17,
0,
43,
0,
"IMAGE"
],
[
136,
15,
1,
17,
0,
"LATENT"
],
[
143,
15,
0,
45,
5,
"LATENT"
],
[
144,
45,
1,
8,
0,
"LATENT"
],
[
145,
6,
0,
45,
1,
"CONDITIONING"
],
[
146,
7,
0,
45,
2,
"CONDITIONING"
],
[
147,
16,
0,
45,
4,
"SIGMAS"
],
[
148,
19,
0,
45,
3,
"SAMPLER"
]
],
"groups": [],
"config": {},
"extra": {},
"version": 0.4
}
resources/example_workflow.json → resources/style_aligned_within_batch.json
+187 -191
View File
@@ -1,159 +1,13 @@
{
"last_node_id": 12,
"last_node_id": 10,
"last_link_id": 12,
"nodes": [
{
"id": 4,
"type": "CheckpointLoaderSimple",
"pos": [
26,
474
],
"size": {
"0": 315,
"1": 98
},
"flags": {},
"order": 0,
"mode": 0,
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
11
],
"slot_index": 0
},
{
"name": "CLIP",
"type": "CLIP",
"links": [
3,
5
],
"slot_index": 1
},
{
"name": "VAE",
"type": "VAE",
"links": [
8
],
"slot_index": 2
}
],
"properties": {
"Node name for S&R": "CheckpointLoaderSimple"
},
"widgets_values": [
"sd_xl_base_1.0.safetensors"
]
},
{
"id": 5,
"type": "EmptyLatentImage",
"pos": [
455,
645
],
"size": {
"0": 315,
"1": 106
},
"flags": {},
"order": 1,
"mode": 0,
"outputs": [
{
"name": "LATENT",
"type": "LATENT",
"links": [
2
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "EmptyLatentImage"
},
"widgets_values": [
512,
512,
4
]
},
{
"id": 8,
"type": "VAEDecode",
"pos": [
1247,
7
],
"size": {
"0": 210,
"1": 46
},
"flags": {},
"order": 6,
"mode": 0,
"inputs": [
{
"name": "samples",
"type": "LATENT",
"link": 7
},
{
"name": "vae",
"type": "VAE",
"link": 8
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
10
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "VAEDecode"
}
},
{
"id": 10,
"type": "PreviewImage",
"pos": [
1509,
110
],
"size": {
"0": 481.1595764160156,
"1": 565.72705078125
},
"flags": {},
"order": 7,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 10
}
],
"properties": {
"Node name for S&R": "PreviewImage"
}
},
{
"id": 7,
"type": "CLIPTextEncode",
"pos": [
427,
426
413,
389
],
"size": {
"0": 425.27801513671875,
@@ -183,15 +37,15 @@
"Node name for S&R": "CLIPTextEncode"
},
"widgets_values": [
"text, watermark, low quality, ugly"
"text, watermark"
]
},
{
"id": 6,
"type": "CLIPTextEncode",
"pos": [
410,
-74
415,
186
],
"size": {
"0": 422.84503173828125,
@@ -221,19 +75,166 @@
"Node name for S&R": "CLIPTextEncode"
},
"widgets_values": [
"a toy train, macro photo, high quality, centered, zoom lens"
"beautiful scenery nature glass bottle landscape, , purple galaxy bottle,"
]
},
{
"id": 4,
"type": "CheckpointLoaderSimple",
"pos": [
26,
474
],
"size": {
"0": 315,
"1": 98
},
"flags": {},
"order": 0,
"mode": 0,
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
10
],
"slot_index": 0
},
{
"name": "CLIP",
"type": "CLIP",
"links": [
3,
5
],
"slot_index": 1
},
{
"name": "VAE",
"type": "VAE",
"links": [
8
],
"slot_index": 2
}
],
"properties": {
"Node name for S&R": "CheckpointLoaderSimple"
},
"widgets_values": [
"sd_xl_base_1.0.safetensors"
]
},
{
"id": 9,
"type": "SaveImage",
"pos": [
1896,
346
],
"size": [
334.20353016404806,
384.790212276488
],
"flags": {},
"order": 7,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 9
}
],
"properties": {},
"widgets_values": [
"ComfyUI"
]
},
{
"id": 5,
"type": "EmptyLatentImage",
"pos": [
606,
681
],
"size": {
"0": 315,
"1": 106
},
"flags": {},
"order": 1,
"mode": 0,
"outputs": [
{
"name": "LATENT",
"type": "LATENT",
"links": [
12
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "EmptyLatentImage"
},
"widgets_values": [
512,
512,
4
]
},
{
"id": 8,
"type": "VAEDecode",
"pos": [
1696,
182
],
"size": {
"0": 210,
"1": 46
},
"flags": {},
"order": 6,
"mode": 0,
"inputs": [
{
"name": "samples",
"type": "LATENT",
"link": 7
},
{
"name": "vae",
"type": "VAE",
"link": 8
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
9
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "VAEDecode"
}
},
{
"id": 3,
"type": "KSampler",
"pos": [
955,
340
1376,
411
],
"size": {
"0": 315,
"1": 474
"1": 262
},
"flags": {},
"order": 5,
@@ -242,7 +243,7 @@
{
"name": "model",
"type": "MODEL",
"link": 12
"link": 11
},
{
"name": "positive",
@@ -257,7 +258,7 @@
{
"name": "latent_image",
"type": "LATENT",
"link": 2
"link": 12
}
],
"outputs": [
@@ -274,25 +275,25 @@
"Node name for S&R": "KSampler"
},
"widgets_values": [
156680208709677,
122740602030855,
"fixed",
32,
7,
"uni_pc",
"sgm_uniform",
20,
8,
"euler",
"normal",
1
]
},
{
"id": 11,
"type": "StyleAlignedPatch",
"id": 10,
"type": "StyleAlignedBatchAlign",
"pos": [
476,
191
923,
317
],
"size": {
"0": 216.59999084472656,
"1": 102
"0": 315,
"1": 82
},
"flags": {},
"order": 2,
@@ -301,12 +302,7 @@
{
"name": "model",
"type": "MODEL",
"link": 11
},
{
"name": "style_image",
"type": "IMAGE",
"link": null
"link": 10
}
],
"outputs": [
@@ -314,14 +310,14 @@
"name": "MODEL",
"type": "MODEL",
"links": [
12
11
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "StyleAlignedPatch"
"Node name for S&R": "StyleAlignedBatchAlign"
},
"widgets_values": [
"both",
@@ -330,14 +326,6 @@
}
],
"links": [
[
2,
5,
0,
3,
3,
"LATENT"
],
[
3,
4,
@@ -387,28 +375,36 @@
"VAE"
],
[
10,
9,
8,
0,
10,
9,
0,
"IMAGE"
],
[
11,
10,
4,
0,
10,
0,
"MODEL"
],
[
11,
10,
0,
3,
0,
"MODEL"
],
[
12,
11,
5,
0,
3,
0,
"MODEL"
3,
"LATENT"
]
],
"groups": [],