nodes/ExponentialML-ComfyUI_VisualStylePrompting
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add lazy block selection to style prompting

Browse Source 
Adds the feature recommended from this issue: https://github.com/ExponentialML/ComfyUI_VisualStylePrompting/issues/9#issue-2199077769.

Also fixes bugs that wasn't allowing style prompting to work correctly.
This commit is contained in:
ExponentialML
2024-03-21 15:14:52 -07:00
committed by GitHub
parent 05e5b2dbf6
commit af6fe36f17
2 changed files with 107 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
utils/attention_functions.py
+64 -3
View File
@@ -1,11 +1,47 @@
from comfy.ldm.modules.attention import default, optimized_attention, optimized_attention_masked
from .style_functions import adain, concat_first
from comfy.ldm.modules.attention import default, optimized_attention, optimized_attention_masked, attention_basic
from .style_functions import adain, concat_first, swapping_attention
ATTENTION_LAYER_CANDIDATES = [
"input_blocks.1.1.transformer_blocks.0.attn1",
"input_blocks.1.1.transformer_blocks.0.attn2",
"input_blocks.2.1.transformer_blocks.0.attn1",
"input_blocks.2.1.transformer_blocks.0.attn2",
"input_blocks.4.1.transformer_blocks.0.attn1",
"input_blocks.4.1.transformer_blocks.0.attn2",
"input_blocks.5.1.transformer_blocks.0.attn1",
"input_blocks.5.1.transformer_blocks.0.attn2",
"input_blocks.7.1.transformer_blocks.0.attn1",
"input_blocks.7.1.transformer_blocks.0.attn2",
"input_blocks.8.1.transformer_blocks.0.attn1",
"input_blocks.8.1.transformer_blocks.0.attn2",
"middle_block.1.transformer_blocks.0.attn1",
"middle_block.1.transformer_blocks.0.attn2",
"output_blocks.3.1.transformer_blocks.0.attn1",
"output_blocks.3.1.transformer_blocks.0.attn2",
"output_blocks.4.1.transformer_blocks.0.attn1",
"output_blocks.4.1.transformer_blocks.0.attn2",
"output_blocks.5.1.transformer_blocks.0.attn1",
"output_blocks.5.1.transformer_blocks.0.attn2",
"output_blocks.6.1.transformer_blocks.0.attn1",
"output_blocks.6.1.transformer_blocks.0.attn2",
"output_blocks.7.1.transformer_blocks.0.attn1",
"output_blocks.7.1.transformer_blocks.0.attn2",
"output_blocks.8.1.transformer_blocks.0.attn1",
"output_blocks.8.1.transformer_blocks.0.attn2",
"output_blocks.9.1.transformer_blocks.0.attn1",
"output_blocks.9.1.transformer_blocks.0.attn2",
"output_blocks.10.1.transformer_blocks.0.attn1",
"output_blocks.10.1.transformer_blocks.0.attn2",
"output_blocks.11.1.transformer_blocks.0.attn1",
"output_blocks.11.1.transformer_blocks.0.attn2"
]
class VisualStyleProcessor(object):
def __init__(self,
module_self,
keys_scale: float = 1.0,
enabled: bool = True,
enabled_animatediff: bool = True,
adain_queries: bool = True,
adain_keys: bool = True,
adain_values: bool = False
@@ -13,6 +49,7 @@ class VisualStyleProcessor(object):
self.module_self = module_self
self.keys_scale = keys_scale
self.enabled = enabled
self.enabled_animatediff = enabled
self.adain_queries = adain_queries
self.adain_keys = adain_keys
self.adain_values = adain_values
@@ -42,4 +79,28 @@ class VisualStyleProcessor(object):
out = optimized_attention(q, k, v, self.module_self.heads)
else:
out = optimized_attention_masked(q, k, v, self.module_self.heads, mask)
return self.module_self.to_out(out)
return self.module_self.to_out(out)
def visual_style_forward_ad(self, x, context=None, value=None, mask=None):
q = module_self.to_q(x)
context = default(context, x)
k = module_self.to_k(context)
if value is not None:
v = module_self.to_v(value)
del value
else:
v = module_self.to_v(context)
# apply custom scale by multiplying k by scale factor
if module_self.scale is not None:
k *= module_self.scale
# apply scale mask, if present
if scale_mask is not None:
k *= scale_mask
if self.enable_animatediff:
k, v = swapping_attention(key, value)
out = attention_basic(q, k, v, module_self.heads, mask)
return self.to_out(out)