163 lines
7.0 KiB
Python
163 lines
7.0 KiB
Python
import enum
|
|
|
|
from .block_type import BlockType
|
|
|
|
|
|
def isinstance_str(x: object, cls_name: str):
|
|
for _cls in x.__class__.__mro__:
|
|
if _cls.__name__ == cls_name:
|
|
return True
|
|
|
|
return False
|
|
|
|
|
|
def is_temporal_block(module):
|
|
return isinstance_str(module, 'TemporalTransformerBlock')
|
|
|
|
|
|
def is_named_module_transformer_block(named_module):
|
|
if is_temporal_block(named_module[1]):
|
|
return True
|
|
return False
|
|
|
|
|
|
class RefMode(enum.Enum):
|
|
OFF = 'OFF'
|
|
WRITE = 'WRITE'
|
|
READ = 'READ'
|
|
|
|
|
|
class RefController:
|
|
modules = []
|
|
temp_modules = []
|
|
default_video_length = 16
|
|
default_full_length = 16
|
|
model = None
|
|
|
|
def add_module(self, module):
|
|
self.modules.append(module)
|
|
|
|
def set_mode(self, mode: RefMode):
|
|
for module in self.modules:
|
|
module.ref_mode = mode
|
|
if mode == RefMode.OFF:
|
|
self.set_kv_mode(mode)
|
|
self.set_normal_mode(mode)
|
|
|
|
def set_normal_mode(self, mode: RefMode):
|
|
for module in self.modules:
|
|
module.normal_mode = mode
|
|
|
|
def set_kv_mode(self, mode: RefMode, count = None):
|
|
for module in self.modules:
|
|
module.kv_mode = RefMode.OFF
|
|
if mode == RefMode.OFF or count is None:
|
|
count = len(self.modules)
|
|
output_modules = list(filter(lambda m: m.block_type == BlockType.OUTPUT, self.modules))
|
|
output_modules = sorted(output_modules, key=lambda m: -m.block_idx)[:count]
|
|
for module in output_modules:
|
|
module.kv_mode = mode
|
|
|
|
def set_motion_mode(self, config, mode: RefMode):
|
|
model = self.model
|
|
input_modules = list(filter(is_named_module_transformer_block, model.input_blocks.named_modules()))
|
|
i = 0
|
|
for _, module in input_modules: # 8
|
|
module.attention_blocks[0].mode = 'OFF'
|
|
module.attention_blocks[0].q_bank_enabled = False
|
|
module.attention_blocks[0].k_bank_enabled = False
|
|
module.attention_blocks[0].v_bank_enabled = False
|
|
module.attention_blocks[0].norm_bank_enabled = False
|
|
if len(module.attention_blocks) > 1:
|
|
module.attention_blocks[1].mode = 'OFF'
|
|
module.attention_blocks[1].q_bank_enabled = False
|
|
module.attention_blocks[1].k_bank_enabled = False
|
|
module.attention_blocks[1].v_bank_enabled = False
|
|
module.attention_blocks[1].norm_bank_enabled = False
|
|
if config is not None and config.inputs[i] and mode.name != 'OFF':
|
|
module.attention_blocks[0].mode = mode.name
|
|
module.attention_blocks[0].q_bank_enabled = config.q_bank_enabled
|
|
module.attention_blocks[0].k_bank_enabled = config.k_bank_enabled
|
|
module.attention_blocks[0].v_bank_enabled = config.v_bank_enabled
|
|
module.attention_blocks[0].norm_bank_enabled = config.norm_bank_enabled
|
|
if len(module.attention_blocks) > 1:
|
|
module.attention_blocks[1].mode = mode.name
|
|
module.attention_blocks[1].q_bank_enabled = config.q_bank_enabled
|
|
module.attention_blocks[1].k_bank_enabled = config.k_bank_enabled
|
|
module.attention_blocks[1].v_bank_enabled = config.v_bank_enabled
|
|
module.attention_blocks[1].norm_bank_enabled = config.norm_bank_enabled
|
|
if len(module.attention_blocks) > 2:
|
|
print('whoa in')
|
|
i += 1
|
|
|
|
middle_modules = [] # list(filter(is_named_module_transformer_block, model.middle_block.named_modules()))
|
|
output_modules = list(filter(is_named_module_transformer_block, model.output_blocks.named_modules()))
|
|
|
|
i = 0
|
|
for _, module in output_modules: # 12
|
|
module.attention_blocks[0].mode = 'OFF'
|
|
module.attention_blocks[0].q_bank_enabled = False
|
|
module.attention_blocks[0].k_bank_enabled = False
|
|
module.attention_blocks[0].v_bank_enabled = False
|
|
module.attention_blocks[0].norm_bank_enabled = False
|
|
if len(module.attention_blocks) > 1:
|
|
module.attention_blocks[1].mode = 'OFF'
|
|
module.attention_blocks[1].q_bank_enabled = False
|
|
module.attention_blocks[1].k_bank_enabled = False
|
|
module.attention_blocks[1].v_bank_enabled = False
|
|
module.attention_blocks[1].norm_bank_enabled = False
|
|
if config is not None and config.outputs[i] and mode.name != 'OFF':
|
|
module.attention_blocks[0].mode = mode.name
|
|
module.attention_blocks[0].q_bank_enabled = config.q_bank_enabled
|
|
module.attention_blocks[0].k_bank_enabled = config.k_bank_enabled
|
|
module.attention_blocks[0].v_bank_enabled = config.v_bank_enabled
|
|
module.attention_blocks[0].norm_bank_enabled = config.norm_bank_enabled
|
|
if len(module.attention_blocks) > 1:
|
|
module.attention_blocks[1].mode = mode.name
|
|
module.attention_blocks[1].q_bank_enabled = config.q_bank_enabled
|
|
module.attention_blocks[1].k_bank_enabled = config.k_bank_enabled
|
|
module.attention_blocks[1].v_bank_enabled = config.v_bank_enabled
|
|
module.attention_blocks[1].norm_bank_enabled = config.norm_bank_enabled
|
|
if len(module.attention_blocks) > 2:
|
|
print('whoa')
|
|
i += 1
|
|
|
|
def clear_modules(self):
|
|
for module in self.modules:
|
|
module.clean()
|
|
self.set_mode(RefMode.OFF)
|
|
self.set_kv_mode(RefMode.OFF, len(self.modules))
|
|
self.set_normal_mode(RefMode.OFF)
|
|
self.set_motion_mode(None, RefMode.OFF)
|
|
motion_modules = list(filter(is_named_module_transformer_block, self.model.named_modules()))
|
|
for _, module in motion_modules:
|
|
for i in range(2):
|
|
module.attention_blocks[i].q_bank = None
|
|
module.attention_blocks[i].k_bank = None
|
|
module.attention_blocks[i].v_bank = None
|
|
module.attention_blocks[i].norm_bank = None
|
|
module.attention_blocks[i].q_bank_enabled = False
|
|
module.attention_blocks[i].k_bank_enabled = False
|
|
module.attention_blocks[i].v_bank_enabled = False
|
|
module.attention_blocks[i].norm_bank_enabled = False
|
|
|
|
def set_cfg(self, cfg: bool):
|
|
for module in self.modules:
|
|
module.is_cfg = cfg
|
|
|
|
def set_attention_bound(self, attention_bound: float):
|
|
for module in self.modules:
|
|
module.attention_bound = attention_bound
|
|
|
|
def set_content_fidelity(self, content_fidelity: float):
|
|
for module in self.modules:
|
|
module.content_fidelity = content_fidelity
|
|
|
|
def set_normal_add(self, normal_add: bool):
|
|
for module in self.modules:
|
|
module.normal_add = normal_add
|
|
|
|
def set_spatial_query(self, spatial_query: bool):
|
|
for module in self.modules:
|
|
module.spatial_query = spatial_query
|