diff --git a/README.md b/README.md index 481c8fb..0544b99 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,19 @@ # ComfyUI_VisualStylePrompting ComfyUI Version of "Visual Style Prompting with Swapping Self-Attention" -![image](https://github.com/ExponentialML/ComfyUI_VisualStylePrompting/assets/59846140/a4272a7a-5f9a-4e34-a1b7-9cb92cf7ef71) +![image](workflows/workflow_simple.png) *credits to @pamparamm* -> [!NOTE] +> [!NOTE] > This is WIP. -> +> > Major changes were made. Please make sure to update your workflows. An updated workflow can be found in the `workflows` directory. -Implements the very basics of [Visual Style Prompting](https://github.com/naver-ai/Visual-Style-Prompting)https://github.com/naver-ai/Visual-Style-Prompting by Naver AI. +Implements the very basics of [Visual Style Prompting](https://github.com/naver-ai/Visual-Style-Prompting) by Naver AI. ## Getting Started -Clone the reposlitory into your `custom_nodes` folder, and you'll see the node. It should be placed between your sampler and inputs like the example image. +Clone the repository into your `custom_nodes` folder, and you'll see `Apply Visual Style Prompting` node. It should be placed between your sampler and inputs like the example image. This has currently only been tested with 1.5 based models. - `reference_image`: The image you wish to reference, @@ -25,18 +25,19 @@ This has currently only been tested with 1.5 based models. - `middle_block`: Focuses attention on the middle layers. - `output_blocks`: Focuses attention on the decoder layers. -> [!TIP] -> In order to get the best results, you must engineer both the positive and reference image prompts correctly. Focus on the details you want to derive from the image reference, and the details you wish to see in the output. -> -> Based on the hero image above, the positive and prompts were as follows: +> [!NOTE] +> `positive` and `reference_cond` must have equal amount of chunks. Mismatch will cause `RuntimeError: Sizes of tensors must match except in dimension 0.` + +> [!TIP] +> In order to get the best results (matching legacy functionality), you must use the same prompt for `positive` and `reference_cond`. > -> `kitten art, lines and colors, 3d art, unreal engine 5 render, colors, deep colors, shading, canon 60d`, `lines and colors`. -> -> For this generation, I only wanted the lines and colors of the reference. +> The example workflow uses the following for both conds: +> +> `orange fox, origami, deep colors, shading, canon 60d`. ## Notes - Currently, this method utilized the VAE Encode & Inpaint method as it needs to iteralively denoise on each step. Due to how this method works, you'll always get two outputs. To remove the reference latent from the output, simple use a Batch Index Select node. -- For legacy funcionality, please pull this [PR](https://github.com/ExponentialML/ComfyUI_VisualStylePrompting/tree/712a41579e730c5db87d68c2dcde4e92c48ab2cc). +- For legacy functionality, please pull this [PR](https://github.com/ExponentialML/ComfyUI_VisualStylePrompting/tree/712a41579e730c5db87d68c2dcde4e92c48ab2cc). diff --git a/workflows/workflow_simple.json b/workflows/workflow_simple.json deleted file mode 100644 index 285a149..0000000 --- a/workflows/workflow_simple.json +++ /dev/null @@ -1,568 +0,0 @@ -{ - "last_node_id": 47, - "last_link_id": 149, - "nodes": [ - { - "id": 42, - "type": "CLIPTextEncode", - "pos": [ - 683, - 798 - ], - "size": { - "0": 400, - "1": 200 - }, - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 133 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 136, - 143 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "" - ], - "color": "#322", - "bgcolor": "#533" - }, - { - "id": 38, - "type": "CheckpointLoaderSimple", - "pos": [ - 240, - 640 - ], - "size": { - "0": 315, - "1": 98 - }, - "flags": {}, - "order": 0, - "mode": 0, - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 139 - ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [ - 133, - 144 - ], - "shape": 3, - "slot_index": 1 - }, - { - "name": "VAE", - "type": "VAE", - "links": [ - 131, - 140 - ], - "shape": 3, - "slot_index": 2 - } - ], - "properties": { - "Node name for S&R": "CheckpointLoaderSimple" - }, - "widgets_values": [ - "realisticVisionV60B1_v51VAE.safetensors" - ] - }, - { - "id": 40, - "type": "VAEDecode", - "pos": [ - 1898, - 578 - ], - "size": { - "0": 210, - "1": 46 - }, - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 130 - }, - { - "name": "vae", - "type": "VAE", - "link": 131 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 132 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "VAEDecode" - } - }, - { - "id": 46, - "type": "LoadImage", - "pos": [ - 243, - 252 - ], - "size": [ - 315, - 314 - ], - "flags": {}, - "order": 1, - "mode": 0, - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 147 - ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": null, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "pasted/image (149).png", - "image" - ] - }, - { - "id": 47, - "type": "ImageScale", - "pos": [ - 682, - 257 - ], - "size": { - "0": 315, - "1": 130 - }, - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [ - { - "name": "image", - "type": "IMAGE", - "link": 147 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 148 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "ImageScale" - }, - "widgets_values": [ - "nearest-exact", - 768, - 512, - "center" - ] - }, - { - "id": 44, - "type": "ApplyVisualStyle", - "pos": [ - 1186, - 571 - ], - "size": { - "0": 355.20001220703125, - "1": 254 - }, - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 139 - }, - { - "name": "vae", - "type": "VAE", - "link": 140 - }, - { - "name": "reference_image", - "type": "IMAGE", - "link": 148, - "slot_index": 2 - }, - { - "name": "visual_style_prompt", - "type": "CONDITIONING", - "link": 142, - "slot_index": 3 - }, - { - "name": "negative_prompt", - "type": "CONDITIONING", - "link": 143 - }, - { - "name": "init_image", - "type": "IMAGE", - "link": null - } - ], - "outputs": [ - { - "name": "model", - "type": "MODEL", - "links": [ - 138 - ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "positive", - "type": "CONDITIONING", - "links": [ - 149 - ], - "shape": 3, - "slot_index": 1 - }, - { - "name": "negative", - "type": "CONDITIONING", - "links": null, - "shape": 3 - }, - { - "name": "latents", - "type": "LATENT", - "links": [ - 146 - ], - "shape": 3, - "slot_index": 3 - } - ], - "properties": { - "Node name for S&R": "ApplyVisualStyle" - }, - "widgets_values": [ - true, - 1, - false, - true, - true - ] - }, - { - "id": 39, - "type": "KSampler", - "pos": [ - 1557, - 572 - ], - "size": [ - 315, - 474 - ], - "flags": {}, - "order": 6, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 138 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 149 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 136 - }, - { - "name": "latent_image", - "type": "LATENT", - "link": 146 - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 130 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "KSampler" - }, - "widgets_values": [ - 656380172428958, - "randomize", - 20, - 8, - "dpmpp_2m", - "sgm_uniform", - 1 - ] - }, - { - "id": 45, - "type": "VisualStylePrompt", - "pos": [ - 681, - 545 - ], - "size": { - "0": 400, - "1": 200 - }, - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 144 - } - ], - "outputs": [ - { - "name": "visual_style_prompt", - "type": "CONDITIONING", - "links": [ - 142 - ], - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "VisualStylePrompt" - }, - "widgets_values": [ - "a cute kitten made out of clouds and cotton candy, colorful, beautiful, unreal engine 5", - "a photo of clouds in the sky" - ], - "color": "#232", - "bgcolor": "#353" - }, - { - "id": 41, - "type": "PreviewImage", - "pos": [ - 1108, - 16 - ], - "size": [ - 1331.5175122643755, - 486.63612424492214 - ], - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 132 - } - ], - "properties": { - "Node name for S&R": "PreviewImage" - } - } - ], - "links": [ - [ - 130, - 39, - 0, - 40, - 0, - "LATENT" - ], - [ - 131, - 38, - 2, - 40, - 1, - "VAE" - ], - [ - 132, - 40, - 0, - 41, - 0, - "IMAGE" - ], - [ - 133, - 38, - 1, - 42, - 0, - "CLIP" - ], - [ - 136, - 42, - 0, - 39, - 2, - "CONDITIONING" - ], - [ - 138, - 44, - 0, - 39, - 0, - "MODEL" - ], - [ - 139, - 38, - 0, - 44, - 0, - "MODEL" - ], - [ - 140, - 38, - 2, - 44, - 1, - "VAE" - ], - [ - 142, - 45, - 0, - 44, - 3, - "CONDITIONING" - ], - [ - 143, - 42, - 0, - 44, - 4, - "CONDITIONING" - ], - [ - 144, - 38, - 1, - 45, - 0, - "CLIP" - ], - [ - 146, - 44, - 3, - 39, - 3, - "LATENT" - ], - [ - 147, - 46, - 0, - 47, - 0, - "IMAGE" - ], - [ - 148, - 47, - 0, - 44, - 2, - "IMAGE" - ], - [ - 149, - 44, - 1, - 39, - 1, - "CONDITIONING" - ] - ], - "groups": [], - "config": {}, - "extra": {}, - "version": 0.4 -} \ No newline at end of file diff --git a/workflows/workflow_simple.png b/workflows/workflow_simple.png new file mode 100644 index 0000000..6de266a Binary files /dev/null and b/workflows/workflow_simple.png differ