179 lines
7.8 KiB
Python
179 lines
7.8 KiB
Python
import torch
|
|
import comfy
|
|
import types
|
|
|
|
|
|
# Check and add 'brush_model_patch' to model.model_options['transformer_options']
|
|
def add_model_patch_option(model):
|
|
if 'transformer_options' not in model.model_options:
|
|
model.model_options['transformer_options'] = {}
|
|
to = model.model_options['transformer_options']
|
|
if "brush_model_patch" not in to:
|
|
to["brush_model_patch"] = {}
|
|
return to
|
|
|
|
|
|
# Patch model with model_function_wrapper
|
|
def patch_model_function_wrapper(model, forward_patch, remove=False):
|
|
|
|
patch_wrapper = {}
|
|
comfy.patcher_extension.add_wrapper(comfy.patcher_extension.WrappersMP.APPLY_MODEL,
|
|
apply_model_function_wrapper,
|
|
patch_wrapper,
|
|
is_model_options=False)
|
|
to = add_model_patch_option(model)
|
|
brush_model_patch = to['brush_model_patch']
|
|
|
|
if isinstance(model.model.model_config, comfy.supported_models.SD15):
|
|
brush_model_patch['SDXL'] = False
|
|
elif isinstance(model.model.model_config, comfy.supported_models.SDXL):
|
|
brush_model_patch['SDXL'] = True
|
|
else:
|
|
print('Base model type: ', type(model.model.model_config))
|
|
raise Exception("Unsupported model type: ", type(model.model.model_config))
|
|
|
|
if 'forward' not in brush_model_patch:
|
|
brush_model_patch['forward'] = []
|
|
|
|
if remove:
|
|
if forward_patch in brush_model_patch['forward']:
|
|
brush_model_patch['forward'].remove(forward_patch)
|
|
else:
|
|
brush_model_patch['forward'].append(forward_patch)
|
|
|
|
brush_model_patch['step'] = 0
|
|
brush_model_patch['total_steps'] = 1
|
|
|
|
# apply patches to code
|
|
comfy.patcher_extension.add_wrapper(comfy.patcher_extension.WrappersMP.OUTER_SAMPLE,
|
|
brushNet_out_sample_wrapper,
|
|
patch_wrapper,
|
|
is_model_options=False)
|
|
model.wrappers.update(patch_wrapper['wrappers'])
|
|
|
|
def apply_model_function_wrapper(apply_model_executor, x, t, c_concat=None, c_crossattn=None, control=None, transformer_options={}, **kwargs):
|
|
baseModel = apply_model_executor.class_obj
|
|
to = transformer_options
|
|
|
|
timestep = t
|
|
|
|
# check if there are patches to execute
|
|
if 'brush_model_patch' not in to or 'forward' not in to['brush_model_patch']:
|
|
return apply_model_executor(x, timestep, c_concat, c_crossattn, control, transformer_options, **kwargs)
|
|
|
|
brush_model_patch = to['brush_model_patch']
|
|
unet = baseModel.diffusion_model
|
|
|
|
all_sigmas = brush_model_patch['all_sigmas']
|
|
sigma = to['sigmas'][0].item()
|
|
total_steps = all_sigmas.shape[0] - 1
|
|
step = torch.argmin((all_sigmas - sigma).abs()).item()
|
|
|
|
brush_model_patch['step'] = step
|
|
brush_model_patch['total_steps'] = total_steps
|
|
|
|
# comfy.model_base.apply_model
|
|
xc = baseModel.model_sampling.calculate_input(timestep, x)
|
|
if c_concat is not None:
|
|
xc = torch.cat([xc] + [c_concat], dim=1)
|
|
brush_t = baseModel.model_sampling.timestep(timestep).float()
|
|
# execute all patches
|
|
for method in brush_model_patch['forward']:
|
|
method(unet, xc, brush_t, to, control)
|
|
|
|
return apply_model_executor(x, timestep, c_concat, c_crossattn, control, transformer_options, **kwargs)
|
|
|
|
def brushNet_out_sample_wrapper(wrapper_executor, noise, latent_image, sampler, sigmas, denoise_mask=None, callback=None, disable_pbar=False, seed=None):
|
|
cfg_guider = wrapper_executor.class_obj
|
|
# set hook
|
|
diffusion_model = cfg_guider.model_patcher.model.diffusion_model
|
|
set_brushNet_hook(diffusion_model)
|
|
# Use cfd_guider.model_options, which is copied from modelPatcher.model_options and will be restored after execution without any unexpected contamination
|
|
to = add_model_patch_option(cfg_guider)
|
|
to['brush_model_patch']['all_sigmas'] = sigmas
|
|
try:
|
|
out = wrapper_executor(noise, latent_image, sampler, sigmas, denoise_mask=denoise_mask, callback=callback, disable_pbar=disable_pbar, seed=seed)
|
|
finally:
|
|
# cleanup hook
|
|
clean_brushNet_hook(diffusion_model)
|
|
return out
|
|
|
|
|
|
def set_brushNet_hook(diffusion_model):
|
|
for i, block in enumerate(diffusion_model.input_blocks):
|
|
for j, layer in enumerate(block):
|
|
if not hasattr(layer, 'original_forward'):
|
|
layer.original_forward = layer.forward
|
|
layer.forward = types.MethodType(forward_patched_by_brushnet, layer)
|
|
layer.add_sample_after = 0
|
|
for j, layer in enumerate(diffusion_model.middle_block):
|
|
if not hasattr(layer, 'original_forward'):
|
|
layer.original_forward = layer.forward
|
|
layer.forward = types.MethodType(forward_patched_by_brushnet, layer)
|
|
layer.add_sample_after = 0
|
|
for i, block in enumerate(diffusion_model.output_blocks):
|
|
for j, layer in enumerate(block):
|
|
if not hasattr(layer, 'original_forward'):
|
|
layer.original_forward = layer.forward
|
|
layer.forward = types.MethodType(forward_patched_by_brushnet, layer)
|
|
layer.add_sample_after = 0
|
|
|
|
comfy.ldm.modules.diffusionmodules.openaimodel.original_apply_control = comfy.ldm.modules.diffusionmodules.openaimodel.apply_control
|
|
comfy.ldm.modules.diffusionmodules.openaimodel.apply_control = modified_apply_control
|
|
|
|
def clean_brushNet_hook(diffusion_model):
|
|
for i, block in enumerate(diffusion_model.input_blocks):
|
|
for j, layer in enumerate(block):
|
|
if hasattr(layer, 'original_forward'):
|
|
layer.forward = layer.original_forward
|
|
del layer.original_forward
|
|
del layer.add_sample_after
|
|
for j, layer in enumerate(diffusion_model.middle_block):
|
|
if hasattr(layer, 'original_forward'):
|
|
layer.forward = layer.original_forward
|
|
del layer.original_forward
|
|
del layer.add_sample_after
|
|
for i, block in enumerate(diffusion_model.output_blocks):
|
|
for j, layer in enumerate(block):
|
|
if hasattr(layer, 'original_forward'):
|
|
layer.forward = layer.original_forward
|
|
del layer.original_forward
|
|
del layer.add_sample_after
|
|
|
|
if hasattr(comfy.ldm.modules.diffusionmodules.openaimodel, 'original_apply_control'):
|
|
comfy.ldm.modules.diffusionmodules.openaimodel.apply_control = comfy.ldm.modules.diffusionmodules.openaimodel.original_apply_control
|
|
del comfy.ldm.modules.diffusionmodules.openaimodel.original_apply_control
|
|
|
|
# patch layers `forward` so we can apply brushnet
|
|
def forward_patched_by_brushnet(self, x, *args, **kwargs):
|
|
h = self.original_forward(x, *args, **kwargs)
|
|
if hasattr(self, 'add_sample_after') and type(self):
|
|
to_add = self.add_sample_after
|
|
if torch.is_tensor(to_add):
|
|
# interpolate due to RAUNet
|
|
if h.shape[2] != to_add.shape[2] or h.shape[3] != to_add.shape[3]:
|
|
to_add = torch.nn.functional.interpolate(to_add, size=(h.shape[2], h.shape[3]), mode='bicubic')
|
|
h += to_add.to(h.dtype).to(h.device)
|
|
else:
|
|
h += self.add_sample_after
|
|
self.add_sample_after = 0
|
|
return h
|
|
|
|
# To use Controlnet with RAUNet it is much easier to modify apply_control a little
|
|
def modified_apply_control(h, control, name):
|
|
'''
|
|
Modified by BrushNet nodes
|
|
'''
|
|
if control is not None and name in control and len(control[name]) > 0:
|
|
ctrl = control[name].pop()
|
|
if ctrl is not None:
|
|
if h.shape[2] != ctrl.shape[2] or h.shape[3] != ctrl.shape[3]:
|
|
ctrl = torch.nn.functional.interpolate(ctrl, size=(h.shape[2], h.shape[3]), mode='bicubic').to(h.dtype).to(h.device)
|
|
try:
|
|
h += ctrl
|
|
except:
|
|
print.warning("warning control could not be applied {} {}".format(h.shape, ctrl.shape))
|
|
return h
|
|
|
|
|