diff --git a/multitalk/multitalk.py b/multitalk/multitalk.py index 581b43f..5922ac5 100644 --- a/multitalk/multitalk.py +++ b/multitalk/multitalk.py @@ -256,12 +256,13 @@ class SingleStreamAttention(nn.Module): N_t, N_h, N_w = shape x_extra = None - if x.shape[0] != encoder_hidden_states.shape[0]: + try: + x = rearrange(x, "B (N_t S) C -> (B N_t) S C", N_t=N_t) + except: x_extra = x[:, -N_h * N_w:, :] x = x[:, :-N_h * N_w, :] N_t = N_t - 1 - - x = rearrange(x, "B (N_t S) C -> (B N_t) S C", N_t=N_t) + x = rearrange(x, "B (N_t S) C -> (B N_t) S C", N_t=N_t) # get q for hidden_state B, N, C = x.shape @@ -367,7 +368,7 @@ class SingleStreamMultiAttention(SingleStreamAttention): if human_num is None or human_num <= 1: return super().forward(x, encoder_hidden_states, shape) - N_t, _, _ = shape + N_t, N_h, N_w = shape x = rearrange(x, "B (N_t S) C -> (B N_t) S C", N_t=N_t) x_extra = None diff --git a/nodes.py b/nodes.py index d67f7a6..7b967b7 100644 --- a/nodes.py +++ b/nodes.py @@ -2157,15 +2157,17 @@ class WanVideoSampler: rope_function = "default" #echoshot does not support comfy rope function log.info(f"Number of shots in prompt: {shot_num}, Shot token lengths: {shot_len}") - #region transformer settings - - #blockswap init mm.unload_all_models() mm.soft_empty_cache() gc.collect() - if block_swap_args is not None and not weights_assigned: + #region transformer settings + if transformer_options is not None: + block_swap_args = transformer_options.get("block_swap_args", None) + + #blockswap init + if block_swap_args is not None: transformer.use_non_blocking = block_swap_args.get("use_non_blocking", False) for name, param in transformer.named_parameters(): if "block" not in name: @@ -2313,6 +2315,29 @@ class WanVideoSampler: for block in transformer.vace_blocks: block.rope_func = rope_function + #rope + freqs = None + transformer.rope_embedder.k = None + transformer.rope_embedder.num_frames = None + if "default" in rope_function or bidirectional_sampling: + d = transformer.dim // transformer.num_heads + freqs = torch.cat([ + rope_params(1024, d - 4 * (d // 6), L_test=latent_video_length, k=riflex_freq_index), + rope_params(1024, 2 * (d // 6)), + rope_params(1024, 2 * (d // 6)) + ], + dim=1) + elif "comfy" in rope_function: + transformer.rope_embedder.k = riflex_freq_index + transformer.rope_embedder.num_frames = latent_video_length + + transformer.rope_func = rope_function + for block in transformer.blocks: + block.rope_func = rope_function + if transformer.vace_layers is not None: + for block in transformer.vace_blocks: + block.rope_func = rope_function + #region model pred def predict_with_cfg(z, cfg_scale, positive_embeds, negative_embeds, timestep, idx, image_cond=None, clip_fea=None, control_latents=None, vace_data=None, unianim_data=None, audio_proj=None, control_camera_latents=None,