diff --git a/consistory/__init__.py b/consistory/__init__.py
index 8b13789..e69de29 100644
--- a/consistory/__init__.py
+++ b/consistory/__init__.py
@@ -1 +0,0 @@
-
diff --git a/consistory/attention_processor.py b/consistory/attention_processor.py
new file mode 100644
index 0000000..53368d1
--- /dev/null
+++ b/consistory/attention_processor.py
@@ -0,0 +1,292 @@
+# Copyright 2023 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Not a contribution
+# Changes made by NVIDIA CORPORATION & AFFILIATES enabling ConsiStory or otherwise documented as NVIDIA-proprietary
+# are not a contribution and subject to the license under the LICENSE file located at the root directory.
+
+
+from diffusers.utils import USE_PEFT_BACKEND
+from typing import Callable, Optional
+import torch
+from diffusers.models.attention_processor import Attention
+
+from .consistory_utils import AnchorCache, FeatureInjector, QueryStore, xformers
+
+
+class ConsistoryAttnStoreProcessor:
+    def __init__(self, attnstore, place_in_unet):
+        super().__init__()
+        self.attnstore = attnstore
+        self.place_in_unet = place_in_unet
+
+    def __call__(self, attn: Attention, hidden_states, encoder_hidden_states=None, attention_mask=None, record_attention=True, **kwargs):
+        batch_size, sequence_length, _ = hidden_states.shape
+        attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)
+
+        query = attn.to_q(hidden_states)
+
+        is_cross = encoder_hidden_states is not None
+        encoder_hidden_states = encoder_hidden_states if encoder_hidden_states is not None else hidden_states
+        key = attn.to_k(encoder_hidden_states)
+        value = attn.to_v(encoder_hidden_states)
+
+        query = attn.head_to_batch_dim(query)
+        key = attn.head_to_batch_dim(key)
+        value = attn.head_to_batch_dim(value)
+
+        attention_probs = attn.get_attention_scores(query, key, attention_mask)
+
+        # only need to store attention maps during the Attend and Excite process
+        # if attention_probs.requires_grad:
+        if record_attention:
+            self.attnstore(attention_probs, is_cross, self.place_in_unet, attn.heads)
+
+        hidden_states = torch.bmm(attention_probs, value)
+        hidden_states = attn.batch_to_head_dim(hidden_states)
+
+        # linear proj
+        hidden_states = attn.to_out[0](hidden_states)
+        # dropout
+        hidden_states = attn.to_out[1](hidden_states)
+
+        return hidden_states
+
+
+class ConsistoryExtendedAttnXFormersAttnProcessor:
+    r"""
+    Processor for implementing memory efficient attention using xFormers.
+
+    Args:
+        attention_op (`Callable`, *optional*, defaults to `None`):
+            The base
+            [operator](https://facebookresearch.github.io/xformers/components/ops.html#xformers.ops.AttentionOpBase) to
+            use as the attention operator. It is recommended to set to `None`, and allow xFormers to choose the best
+            operator.
+    """
+
+    def __init__(self, place_in_unet, attnstore, extended_attn_kwargs, attention_op: Optional[Callable] = None):
+        self.attention_op = attention_op
+        self.t_range = extended_attn_kwargs.get('t_range', [])
+        self.extend_kv_unet_parts = extended_attn_kwargs.get('extend_kv_unet_parts', ['down', 'mid', 'up'])
+
+        self.place_in_unet = place_in_unet
+        self.curr_unet_part = self.place_in_unet.split('_')[0]
+        self.attnstore = attnstore
+
+    def __call__(
+        self,
+        attn: Attention,
+        hidden_states: torch.FloatTensor,
+        encoder_hidden_states: Optional[torch.FloatTensor] = None,
+        attention_mask: Optional[torch.FloatTensor] = None,
+        temb: Optional[torch.FloatTensor] = None,
+        scale: float = 1.0,
+        perform_extend_attn: bool = False,
+        query_store: Optional[QueryStore] = None,
+        feature_injector: Optional[FeatureInjector] = None,
+        anchors_cache: Optional[AnchorCache] = None,
+        **kwargs
+    ) -> torch.FloatTensor:
+        residual = hidden_states
+
+        args = () if USE_PEFT_BACKEND else (scale,)
+
+        if attn.spatial_norm is not None:
+            hidden_states = attn.spatial_norm(hidden_states, temb)
+
+        input_ndim = hidden_states.ndim
+
+        if input_ndim == 4:
+            batch_size, channel, height, width = hidden_states.shape
+            hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)
+        else:
+            batch_size, wh, channel = hidden_states.shape
+            height = width = int(wh ** 0.5)
+
+        is_cross = encoder_hidden_states is not None
+        perform_extend_attn = perform_extend_attn and (not is_cross) and \
+                              any([self.attnstore.curr_iter >= x[0] and self.attnstore.curr_iter <= x[1] for x in self.t_range]) and \
+                              self.curr_unet_part in self.extend_kv_unet_parts
+
+        batch_size, key_tokens, _ = (
+            hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape
+        )
+
+        attention_mask = attn.prepare_attention_mask(attention_mask, key_tokens, batch_size)
+        if attention_mask is not None:
+            # expand our mask's singleton query_tokens dimension:
+            #   [batch*heads,            1, key_tokens] ->
+            #   [batch*heads, query_tokens, key_tokens]
+            # so that it can be added as a bias onto the attention scores that xformers computes:
+            #   [batch*heads, query_tokens, key_tokens]
+            # we do this explicitly because xformers doesn't broadcast the singleton dimension for us.
+            _, query_tokens, _ = hidden_states.shape
+            attention_mask = attention_mask.expand(-1, query_tokens, -1)
+
+        if attn.group_norm is not None:
+            hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2)
+
+        query = attn.to_q(hidden_states, *args)
+
+        if (self.curr_unet_part in self.extend_kv_unet_parts) and query_store and query_store.mode == 'cache':
+            query_store.cache_query(query, self.place_in_unet)
+        elif perform_extend_attn and query_store and query_store.mode == 'inject':
+            query = query_store.inject_query(query, self.place_in_unet, self.attnstore.curr_iter)
+
+        if encoder_hidden_states is None:
+            encoder_hidden_states = hidden_states
+        elif attn.norm_cross:
+            encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states)
+
+        key = attn.to_k(encoder_hidden_states, *args)
+        value = attn.to_v(encoder_hidden_states, *args)
+
+        query = attn.head_to_batch_dim(query).contiguous()
+
+        if perform_extend_attn:
+            # Anchor Caching
+            if anchors_cache and anchors_cache.is_cache_mode():
+                if self.place_in_unet not in anchors_cache.input_h_cache:
+                    anchors_cache.input_h_cache[self.place_in_unet] = {}
+
+                # Hidden states inside the mask, for uncond (index 0) and cond (index 1) prompts
+                subjects_hidden_states = torch.stack([x[self.attnstore.last_mask_dropout[width]] for x in hidden_states.chunk(2)])
+                anchors_cache.input_h_cache[self.place_in_unet][self.attnstore.curr_iter] = subjects_hidden_states
+
+            if anchors_cache and anchors_cache.is_inject_mode():
+                # We make extended key and value by concatenating the original key and value with the query.
+                anchors_hidden_states = anchors_cache.input_h_cache[self.place_in_unet][self.attnstore.curr_iter]
+
+                anchors_keys = attn.to_k(anchors_hidden_states, *args)
+                anchors_values = attn.to_v(anchors_hidden_states, *args)
+
+                extended_key = torch.cat([torch.cat([key.chunk(2, dim=0)[x], anchors_keys[x].unsqueeze(0)], dim=1) for x in range(2)])
+                extended_value = torch.cat([torch.cat([value.chunk(2, dim=0)[x], anchors_values[x].unsqueeze(0)], dim=1) for x in range(2)])
+
+                extended_key = attn.head_to_batch_dim(extended_key).contiguous()
+                extended_value = attn.head_to_batch_dim(extended_value).contiguous()
+
+                # attn_masks needs to be of shape [batch_size, query_tokens, key_tokens]
+                hidden_states = xformers.ops.memory_efficient_attention(
+                    query, extended_key, extended_value,  op=self.attention_op, scale=attn.scale
+                )
+            else:
+                # # We make extended key and value by concatenating the original key and value with the query.
+                # attention_mask_bias = self.attnstore.get_attn_mask_bias(tgt_size = width, bsz = batch_size)
+
+                # if attention_mask_bias is not None:
+                #     attention_mask_bias = torch.cat([x.unsqueeze(0).expand(attn.heads, -1, -1) for x in attention_mask_bias])
+
+                # Pre-allocate the output tensor
+                ex_out = torch.empty_like(query)
+
+                for i in range(batch_size):
+                    start_idx = i * attn.heads
+                    end_idx = start_idx + attn.heads
+
+                    attention_mask = self.attnstore.get_extended_attn_mask_instance(width, i%(batch_size//2))
+
+                    curr_q = query[start_idx:end_idx]
+
+                    if i < batch_size//2:
+                        curr_k = key[:batch_size//2]
+                        curr_v = value[:batch_size//2]
+                    else:
+                        curr_k = key[batch_size//2:]
+                        curr_v = value[batch_size//2:]
+
+                    curr_k = curr_k.flatten(0,1)[attention_mask].unsqueeze(0)
+                    curr_v = curr_v.flatten(0,1)[attention_mask].unsqueeze(0)
+
+                    curr_k = attn.head_to_batch_dim(curr_k).contiguous()
+                    curr_v = attn.head_to_batch_dim(curr_v).contiguous()
+
+                    hidden_states = xformers.ops.memory_efficient_attention(
+                        curr_q, curr_k, curr_v, 
+                        op=self.attention_op, scale=attn.scale
+                    )
+
+                    ex_out[start_idx:end_idx] = hidden_states
+
+                hidden_states = ex_out
+        else:
+            key = attn.head_to_batch_dim(key).contiguous()
+            value = attn.head_to_batch_dim(value).contiguous()
+
+            # attn_masks needs to be of shape [batch_size, query_tokens, key_tokens]
+            hidden_states = xformers.ops.memory_efficient_attention(
+                query, key, value, op=self.attention_op, scale=attn.scale
+            )
+
+        hidden_states = hidden_states.to(query.dtype)
+        hidden_states = attn.batch_to_head_dim(hidden_states)
+
+        # linear proj
+        hidden_states = attn.to_out[0](hidden_states, *args)
+        # dropout
+        hidden_states = attn.to_out[1](hidden_states)
+
+        if (feature_injector is not None):
+            output_res = int(hidden_states.shape[1] ** 0.5)
+
+            if anchors_cache and anchors_cache.is_inject_mode():
+                hidden_states[batch_size//2:] = feature_injector.inject_anchors(hidden_states[batch_size//2:], self.attnstore.curr_iter, output_res, self.attnstore.extended_mapping, self.place_in_unet, anchors_cache)
+            else:
+                hidden_states[batch_size//2:] = feature_injector.inject_outputs(hidden_states[batch_size//2:], self.attnstore.curr_iter, output_res, self.attnstore.extended_mapping, self.place_in_unet, anchors_cache)
+
+        if input_ndim == 4:
+            hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)
+
+        if attn.residual_connection:
+            hidden_states = hidden_states + residual
+
+        hidden_states = hidden_states / attn.rescale_output_factor
+
+        return hidden_states
+
+
+def register_extended_self_attn(unet, attnstore, extended_attn_kwargs):
+    DICT_PLACE_TO_RES = {'down_0': 64, 'down_1': 64, 'down_2': 64, 'down_3': 64, 'down_4': 64, 'down_5': 64, 'down_6': 64, 'down_7': 64,
+                         'down_8': 32, 'down_9': 32, 'down_10': 32, 'down_11': 32, 'down_12': 32, 'down_13': 32, 'down_14': 32, 'down_15': 32,
+                         'down_16': 32, 'down_17': 32, 'down_18': 32, 'down_19': 32, 'down_20': 32, 'down_21': 32, 'down_22': 32, 'down_23': 32,
+                         'down_24': 32, 'down_25': 32, 'down_26': 32, 'down_27': 32, 'down_28': 32, 'down_29': 32, 'down_30': 32, 'down_31': 32,
+                         'down_32': 32, 'down_33': 32, 'down_34': 32, 'down_35': 32, 'down_36': 32, 'down_37': 32, 'down_38': 32, 'down_39': 32,
+                         'down_40': 32, 'down_41': 32, 'down_42': 32, 'down_43': 32, 'down_44': 32, 'down_45': 32, 'down_46': 32, 'down_47': 32,
+                         'mid_120': 32, 'mid_121': 32, 'mid_122': 32, 'mid_123': 32, 'mid_124': 32, 'mid_125': 32, 'mid_126': 32, 'mid_127': 32,
+                         'mid_128': 32, 'mid_129': 32, 'mid_130': 32, 'mid_131': 32, 'mid_132': 32, 'mid_133': 32, 'mid_134': 32, 'mid_135': 32,
+                         'mid_136': 32, 'mid_137': 32, 'mid_138': 32, 'mid_139': 32, 'up_49': 32, 'up_51': 32, 'up_53': 32, 'up_55': 32, 'up_57': 32,
+                         'up_59': 32, 'up_61': 32, 'up_63': 32, 'up_65': 32, 'up_67': 32, 'up_69': 32, 'up_71': 32, 'up_73': 32, 'up_75': 32,
+                         'up_77': 32, 'up_79': 32, 'up_81': 32, 'up_83': 32, 'up_85': 32, 'up_87': 32, 'up_89': 32, 'up_91': 32, 'up_93': 32,
+                         'up_95': 32, 'up_97': 32, 'up_99': 32, 'up_101': 32, 'up_103': 32, 'up_105': 32, 'up_107': 32, 'up_109': 64, 'up_111': 64,
+                         'up_113': 64, 'up_115': 64, 'up_117': 64, 'up_119': 64}
+    attn_procs = {}
+    for i, name in enumerate(unet.attn_processors.keys()):
+        is_self_attn = (i % 2 == 0)
+
+        if name.startswith("mid_block"):
+            place_in_unet = f"mid_{i}"
+        elif name.startswith("up_blocks"):
+            place_in_unet = f"up_{i}"
+        elif name.startswith("down_blocks"):
+            place_in_unet = f"down_{i}"
+        else:
+            continue
+
+        if is_self_attn:
+            attn_procs[name] = ConsistoryExtendedAttnXFormersAttnProcessor(place_in_unet, attnstore, extended_attn_kwargs)
+        else:
+            attn_procs[name] = ConsistoryAttnStoreProcessor(attnstore, place_in_unet)
+
+    unet.set_attn_processor(attn_procs)
\ No newline at end of file
diff --git a/consistory/consistory_CLI.py b/consistory/consistory_CLI.py
new file mode 100644
index 0000000..54545cc
--- /dev/null
+++ b/consistory/consistory_CLI.py
@@ -0,0 +1,82 @@
+# Copyright (C) 2024 NVIDIA Corporation.  All rights reserved.
+#
+# This work is licensed under the LICENSE file
+# located at the root directory.
+
+import os
+import argparse
+from .consistory_run import load_pipeline, run_batch_generation, run_anchor_generation, run_extra_generation
+
+def run_batch(gpu, seed=40, mask_dropout=0.5, same_latent=False,
+              style="A photo of ", subject="a cute dog", concept_token=['dog'],
+              settings=["sitting in the beach", "standing in the snow"],
+              out_dir = None):
+    
+    story_pipeline = load_pipeline(gpu)
+    prompts = [f'{style}{subject} {setting}' for setting in settings]
+
+    images, image_all = run_batch_generation(story_pipeline, prompts, concept_token, seed, mask_dropout=mask_dropout, same_latent=same_latent)
+
+    if out_dir is not None:
+        for i, image in enumerate(images):
+            image.save(f'{out_dir}/image_{i}.png')
+
+    return images, image_all
+
+def run_cached_anchors(gpu, seed=40, mask_dropout=0.5, same_latent=False,
+                style="A photo of ", subject="a cute dog", concept_token=['dog'],
+                settings=["sitting in the beach", "standing in the snow"],
+                cache_cpu_offloading=False, out_dir = None):
+    
+    story_pipeline = load_pipeline(gpu)
+    prompts = [f'{style}{subject} {setting}' for setting in settings]
+    anchor_prompts = prompts[:2]
+    extra_prompts = prompts[2:]
+
+    anchor_out_images, anchor_image_all, anchor_cache_first_stage, anchor_cache_second_stage = run_anchor_generation(story_pipeline, anchor_prompts, concept_token, 
+                                                                                                        seed=seed, mask_dropout=mask_dropout, same_latent=same_latent,
+                                                                                                        cache_cpu_offloading=cache_cpu_offloading)
+    
+    if out_dir is not None:
+        for i, image in enumerate(anchor_out_images):
+            image.save(f'{out_dir}/anchor_image_{i}.png')
+
+    for i, extra_prompt in enumerate(extra_prompts):
+        extra_out_images, extra_image_all = run_extra_generation(story_pipeline, [extra_prompt], concept_token, anchor_cache_first_stage, anchor_cache_second_stage, 
+                                                    seed=seed, mask_dropout=mask_dropout, same_latent=same_latent, cache_cpu_offloading=cache_cpu_offloading)
+        
+        if out_dir is not None:
+            extra_out_images[0].save(f'{out_dir}/extra_image_{i}.png')
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--run_type', default="batch", type=str, required=False) # batch, cached
+
+    parser.add_argument('--gpu', default=0, type=int, required=False)
+    parser.add_argument('--seed', default=40, type=int, required=False)
+    parser.add_argument('--mask_dropout', default=0.5, type=float, required=False)
+    parser.add_argument('--same_latent', default=False, type=bool, required=False)
+
+    parser.add_argument('--style', default="A photo of ", type=str, required=False)
+    parser.add_argument('--subject', default="a cute dog", type=str, required=False)
+    parser.add_argument('--concept_token', default=["dog"], 
+                        type=str, nargs='*', required=False)
+    parser.add_argument('--settings', default=["sitting in the beach", "standing in the snow"], 
+                        type=str, nargs='*', required=False)
+    parser.add_argument('--cache_cpu_offloading', default=False, type=bool, required=False)
+    
+    parser.add_argument('--out_dir', default=None, type=str, required=False)
+
+    args = parser.parse_args()
+
+    if args.out_dir is not None:
+        os.makedirs(args.out_dir, exist_ok=True)
+
+    if args.run_type == "batch":
+        run_batch(args.gpu, args.seed, args.mask_dropout, args.same_latent, args.style, 
+                  args.subject, args.concept_token, args.settings, args.out_dir)
+    elif args.run_type == "cached":
+        run_cached_anchors(args.gpu, args.seed, args.mask_dropout, args.same_latent, args.style, 
+                           args.subject, args.concept_token, args.settings, args.cache_cpu_offloading, args.out_dir)
+    else:
+        print("Invalid run type")
\ No newline at end of file
diff --git a/consistory/consistory_cache.py b/consistory/consistory_cache.py
new file mode 100644
index 0000000..fb5718a
--- /dev/null
+++ b/consistory/consistory_cache.py
@@ -0,0 +1,245 @@
+# Copyright (C) 2024 NVIDIA Corporation.  All rights reserved.
+#
+# This work is licensed under the LICENSE file
+# located at the root directory.
+
+import torch
+from diffusers import DDIMScheduler
+from .consistory_unet_sdxl import ConsistorySDXLUNet2DConditionModel
+from .consistory_pipeline import ConsistoryExtendAttnSDXLPipeline
+from .consistory_utils import FeatureInjector, AnchorCache
+from .utils.general_utils import *
+import gc
+
+from .utils.ptp_utils import view_images
+
+
+def load_pipeline(gpu_id=0):
+    float_type = torch.float16
+    sd_id = "stabilityai/stable-diffusion-xl-base-1.0"
+    
+    device = torch.device(f'cuda:{gpu_id}') if torch.cuda.is_available() else torch.device('cpu')
+    unet = ConsistorySDXLUNet2DConditionModel.from_pretrained(sd_id, subfolder="unet", torch_dtype=float_type)
+    scheduler = DDIMScheduler.from_pretrained(sd_id, subfolder="scheduler")
+
+    story_pipeline = ConsistoryExtendAttnSDXLPipeline.from_pretrained(
+        sd_id, unet=unet, torch_dtype=float_type, variant="fp16", use_safetensors=True, scheduler=scheduler
+    ).to(device)
+    story_pipeline.enable_freeu(s1=0.6, s2=0.4, b1=1.1, b2=1.2)
+    
+    return story_pipeline
+
+def create_anchor_mapping(bsz, anchor_indices=[0]):
+    anchor_mapping = torch.eye(bsz, dtype=torch.bool)
+    for anchor_idx in anchor_indices:
+        anchor_mapping[:, anchor_idx] = True
+
+    return anchor_mapping
+
+def create_token_indices(prompts, batch_size, concept_token, tokenizer):
+    if isinstance(concept_token, str):
+        concept_token = [concept_token]
+
+    concept_token_id = [tokenizer.encode(x, add_special_tokens=False)[0] for x in concept_token]
+    tokens = tokenizer.batch_encode_plus(prompts, padding=True, return_tensors='pt')['input_ids']
+
+    token_indices = torch.full((len(concept_token), batch_size), -1, dtype=torch.int64)
+    for i, token_id in enumerate(concept_token_id):
+        batch_loc, token_loc = torch.where(tokens == token_id)
+        token_indices[i, batch_loc] = token_loc
+
+    return token_indices
+
+def create_latents(story_pipeline, seed, batch_size, same_latent, device, float_type):
+    # if seed is int
+    if isinstance(seed, int):
+        g = torch.Generator('cuda').manual_seed(seed)
+        shape = (batch_size, story_pipeline.unet.config.in_channels, 128, 128)
+        latents = randn_tensor(shape, generator=g, device=device, dtype=float_type)
+    elif isinstance(seed, list):
+        shape = (batch_size, story_pipeline.unet.config.in_channels, 128, 128)
+        latents = torch.empty(shape, device=device, dtype=float_type)
+        for i, seed_i in enumerate(seed):
+            g = torch.Generator('cuda').manual_seed(seed_i)
+            curr_latent = randn_tensor(shape, generator=g, device=device, dtype=float_type)
+            latents[i] = curr_latent[i]
+
+    if same_latent:
+        latents = latents[:1].repeat(batch_size, 1, 1, 1)
+
+    return latents, g
+
+def run_anchor_generation(story_pipeline, prompts, concept_token,
+                   seed=40, n_steps=50, mask_dropout=0.5,
+                   same_latent=False, share_queries=True,
+                   perform_sdsa=True, perform_injection=True,
+                   downscale_rate=4):
+    latent_resolutions = [32, 64]
+
+    device = story_pipeline.device
+    tokenizer = story_pipeline.tokenizer
+    float_type = story_pipeline.dtype
+    unet = story_pipeline.unet
+
+    batch_size = len(prompts)
+
+    token_indices = create_token_indices(prompts, batch_size, concept_token, tokenizer)
+
+    default_attention_store_kwargs = {
+        'token_indices': token_indices,
+        'mask_dropout': mask_dropout
+    }
+
+    default_extended_attn_kwargs = {'extend_kv_unet_parts': ['up']}
+    query_store_kwargs={'t_range': [0,n_steps//10], 'strength_start': 0.9, 'strength_end': 0.81836735}
+
+    latents, g = create_latents(story_pipeline, seed, batch_size, same_latent, device, float_type)
+
+    anchor_cache_first_stage = AnchorCache()
+    anchor_cache_second_stage = AnchorCache()
+
+    # ------------------ #
+    # Extended attention First Run #
+
+    if perform_sdsa:
+        extended_attn_kwargs = {**default_extended_attn_kwargs, 't_range': [(1, n_steps)]}
+    else:
+        extended_attn_kwargs = {**default_extended_attn_kwargs, 't_range': []}
+
+    print(extended_attn_kwargs['t_range'])
+    out = story_pipeline(prompt=prompts, generator=g, latents=latents, 
+                        attention_store_kwargs=default_attention_store_kwargs,
+                        extended_attn_kwargs=extended_attn_kwargs,
+                        share_queries=share_queries,
+                        query_store_kwargs=query_store_kwargs,
+                        anchors_cache=anchor_cache_first_stage,
+                        num_inference_steps=n_steps)
+    last_masks = story_pipeline.attention_store.last_mask
+
+    dift_features = unet.latent_store.dift_features['261_0'][batch_size:]
+    dift_features = torch.stack([gaussian_smooth(x, kernel_size=3, sigma=1) for x in dift_features], dim=0)
+
+    anchor_cache_first_stage.dift_cache = dift_features
+    anchor_cache_first_stage.anchors_last_mask = last_masks
+
+    nn_map, nn_distances = cyclic_nn_map(dift_features, last_masks, latent_resolutions, device)
+
+    torch.cuda.empty_cache()
+    gc.collect()
+
+    # ------------------ #
+    # Extended attention with nn_map #
+    
+    if perform_injection:
+        feature_injector = FeatureInjector(nn_map, nn_distances, last_masks, inject_range_alpha=[(n_steps//10, n_steps//3,0.8)], 
+                                        swap_strategy='min', inject_unet_parts=['up', 'down'], dist_thr='dynamic')
+
+        out = story_pipeline(prompt=prompts, generator=g, latents=latents, 
+                            attention_store_kwargs=default_attention_store_kwargs,
+                            extended_attn_kwargs=extended_attn_kwargs,
+                            share_queries=share_queries,
+                            query_store_kwargs=query_store_kwargs,
+                            feature_injector=feature_injector,
+                            anchors_cache=anchor_cache_second_stage,
+                            num_inference_steps=n_steps)
+        img_all = view_images([np.array(x) for x in out.images], display_image=False, downscale_rate=downscale_rate)
+        # display_attn_maps(story_pipeline.attention_store.last_mask, out.images)
+
+        anchor_cache_second_stage.dift_cache = dift_features
+        anchor_cache_second_stage.anchors_last_mask = last_masks
+
+        torch.cuda.empty_cache()
+        gc.collect()
+    else:
+        img_all = view_images([np.array(x) for x in out.images], display_image=False, downscale_rate=downscale_rate)
+    
+    return out.images, img_all, anchor_cache_first_stage, anchor_cache_second_stage
+
+def run_extra_generation(story_pipeline, prompts, concept_token, 
+                         anchor_cache_first_stage, anchor_cache_second_stage,
+                         seed=40, n_steps=50, mask_dropout=0.5,
+                         same_latent=False, share_queries=True,
+                         perform_sdsa=True, perform_injection=True,
+                         downscale_rate=4):
+    latent_resolutions = [32, 64]
+
+    device = story_pipeline.device
+    tokenizer = story_pipeline.tokenizer
+    float_type = story_pipeline.dtype
+    unet = story_pipeline.unet
+
+    batch_size = len(prompts)
+
+    token_indices = create_token_indices(prompts, batch_size, concept_token, tokenizer)
+
+    default_attention_store_kwargs = {
+        'token_indices': token_indices,
+        'mask_dropout': mask_dropout
+    }
+
+    default_extended_attn_kwargs = {'extend_kv_unet_parts': ['up']}
+    query_store_kwargs={'t_range': [0,n_steps//10], 'strength_start': 0.9, 'strength_end': 0.81836735}
+
+    extra_batch_size = batch_size + 2
+    if isinstance(seed, list):
+        seed = [seed[0], seed[0], *seed]
+
+    latents, g = create_latents(story_pipeline, seed, extra_batch_size, same_latent, device, float_type)
+    latents = latents[2:]
+
+    anchor_cache_first_stage.set_mode_inject()
+    anchor_cache_second_stage.set_mode_inject()
+
+    # ------------------ #
+    # Extended attention First Run #
+
+    if perform_sdsa:
+        extended_attn_kwargs = {**default_extended_attn_kwargs, 't_range': [(1, n_steps)]}
+    else:
+        extended_attn_kwargs = {**default_extended_attn_kwargs, 't_range': []}
+
+    print(extended_attn_kwargs['t_range'])
+    out = story_pipeline(prompt=prompts, generator=g, latents=latents, 
+                        attention_store_kwargs=default_attention_store_kwargs,
+                        extended_attn_kwargs=extended_attn_kwargs,
+                        share_queries=share_queries,
+                        query_store_kwargs=query_store_kwargs,
+                        anchors_cache=anchor_cache_first_stage,
+                        num_inference_steps=n_steps)
+    last_masks = story_pipeline.attention_store.last_mask
+
+    dift_features = unet.latent_store.dift_features['261_0'][batch_size:]
+    dift_features = torch.stack([gaussian_smooth(x, kernel_size=3, sigma=1) for x in dift_features], dim=0)
+
+    anchor_dift_features = anchor_cache_first_stage.dift_cache
+    anchor_last_masks = anchor_cache_first_stage.anchors_last_mask
+
+    nn_map, nn_distances = anchor_nn_map(dift_features, anchor_dift_features, last_masks, anchor_last_masks, latent_resolutions, device)
+
+    torch.cuda.empty_cache()
+    gc.collect()
+
+    # ------------------ #
+    # Extended attention with nn_map #
+    
+    if perform_injection:
+        feature_injector = FeatureInjector(nn_map, nn_distances, last_masks, inject_range_alpha=[(n_steps//10, n_steps//3,0.8)], 
+                                        swap_strategy='min', inject_unet_parts=['up', 'down'], dist_thr='dynamic')
+
+        out = story_pipeline(prompt=prompts, generator=g, latents=latents, 
+                            attention_store_kwargs=default_attention_store_kwargs,
+                            extended_attn_kwargs=extended_attn_kwargs,
+                            share_queries=share_queries,
+                            query_store_kwargs=query_store_kwargs,
+                            feature_injector=feature_injector,
+                            anchors_cache=anchor_cache_second_stage,
+                            num_inference_steps=n_steps)
+        img_all = view_images([np.array(x) for x in out.images], display_image=False, downscale_rate=downscale_rate)
+        # display_attn_maps(story_pipeline.attention_store.last_mask, out.images)
+
+        torch.cuda.empty_cache()
+        gc.collect()
+    else:
+        img_all = view_images([np.array(x) for x in out.images], display_image=False, downscale_rate=downscale_rate)
+    
+    return out.images, img_all
\ No newline at end of file
diff --git a/consistory/consistory_pipeline.py b/consistory/consistory_pipeline.py
new file mode 100644
index 0000000..97886fd
--- /dev/null
+++ b/consistory/consistory_pipeline.py
@@ -0,0 +1,580 @@
+# Copyright 2023 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Not a contribution
+# Changes made by NVIDIA CORPORATION & AFFILIATES enabling ConsiStory or otherwise documented as NVIDIA-proprietary
+# are not a contribution and subject to the license under the LICENSE file located at the root directory.
+
+import torch
+from diffusers.pipelines.stable_diffusion_xl.pipeline_output import StableDiffusionXLPipelineOutput
+from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl import StableDiffusionXLPipeline, \
+    rescale_noise_cfg, EXAMPLE_DOC_STRING
+from diffusers.models.attention_processor import (
+    AttnProcessor2_0,
+    FusedAttnProcessor2_0,
+    XFormersAttnProcessor,)
+from diffusers.loaders import FromSingleFileMixin
+from diffusers.utils import (
+    deprecate,
+    is_torch_xla_available,
+    logging,
+    replace_example_docstring,
+)
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+
+from .attention_processor import register_extended_self_attn
+from .consistory_utils import FeatureInjector, AnchorCache, QueryStore
+from .utils.ptp_utils import AttentionStore
+
+if is_torch_xla_available():
+    # import torch_xla.core.xla_model as xm
+
+    XLA_AVAILABLE = True
+else:
+    XLA_AVAILABLE = False
+
+logger = logging.get_logger(__name__)  # pylint: disable=invalid-name
+
+T = torch.Tensor
+
+class ConsistoryExtendAttnSDXLPipeline(
+    StableDiffusionXLPipeline
+    
+):
+    model_cpu_offload_seq = "text_encoder->text_encoder_2->image_encoder->unet->vae"
+    _optional_components = [
+        "tokenizer",
+        "tokenizer_2",
+        "text_encoder",
+        "text_encoder_2",
+        "image_encoder",
+        "feature_extractor",
+    ]
+    _callback_tensor_inputs = [
+        "latents",
+        "prompt_embeds",
+        "negative_prompt_embeds",
+        "add_text_embeds",
+        "add_time_ids",
+        "negative_pooled_prompt_embeds",
+        "negative_add_time_ids",
+    ]
+
+    
+    
+    def upcast_vae(self):
+        dtype = self.vae.dtype
+        self.vae.to(dtype=torch.float32)
+        use_torch_2_0_or_xformers = isinstance(
+            self.vae.decoder.mid_block.attentions[0].processor,
+            (
+                AttnProcessor2_0,
+                XFormersAttnProcessor,
+                FusedAttnProcessor2_0,
+            ),
+        )
+        # if xformers or torch_2_0 is used attention block does not need
+        # to be in float32 which can save lots of memory
+        if use_torch_2_0_or_xformers:
+            self.vae.post_quant_conv.to(dtype)
+            self.vae.decoder.conv_in.to(dtype)
+            self.vae.decoder.mid_block.to(dtype)
+    
+    @torch.no_grad()
+    @replace_example_docstring(EXAMPLE_DOC_STRING)
+    def __call__(
+        self,
+        prompt: Union[str, List[str]] = None,
+        prompt_2: Optional[Union[str, List[str]]] = None,
+        height: Optional[int] = None,
+        width: Optional[int] = None,
+        num_inference_steps: int = 50,
+        denoising_end: Optional[float] = None,
+        guidance_scale: float = 5.0,
+        negative_prompt: Optional[Union[str, List[str]]] = None,
+        negative_prompt_2: Optional[Union[str, List[str]]] = None,
+        num_images_per_prompt: Optional[int] = 1,
+        eta: float = 0.0,
+        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
+        latents: Optional[torch.FloatTensor] = None,
+        prompt_embeds: Optional[torch.FloatTensor] = None,
+        negative_prompt_embeds: Optional[torch.FloatTensor] = None,
+        pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
+        negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
+        output_type: Optional[str] = "pil",
+        return_dict: bool = True,
+        cross_attention_kwargs: Optional[Dict[str, Any]] = None,
+        guidance_rescale: float = 0.0,
+        original_size: Optional[Tuple[int, int]] = None,
+        crops_coords_top_left: Tuple[int, int] = (0, 0),
+        target_size: Optional[Tuple[int, int]] = None,
+        negative_original_size: Optional[Tuple[int, int]] = None,
+        negative_crops_coords_top_left: Tuple[int, int] = (0, 0),
+        negative_target_size: Optional[Tuple[int, int]] = None,
+        clip_skip: Optional[int] = None,
+        callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None,
+        callback_on_step_end_tensor_inputs: List[str] = ["latents"],
+        attention_store_kwargs: Optional[Dict] = None,
+        extended_attn_kwargs: Optional[Dict] = None,
+        share_queries: bool = False,
+        query_store_kwargs: Optional[Dict] = {},
+        feature_injector: Optional[FeatureInjector] = None,
+        anchors_cache: Optional[AnchorCache] = None,
+
+        instance_latents: Optional[torch.FloatTensor] = None,
+        **kwargs,
+    ):
+        r"""
+        Function invoked when calling the pipeline for generation.
+
+        Args:
+            prompt (`str` or `List[str]`, *optional*):
+                The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.
+                instead.
+            prompt_2 (`str` or `List[str]`, *optional*):
+                The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
+                used in both text-encoders
+            height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
+                The height in pixels of the generated image. This is set to 1024 by default for the best results.
+                Anything below 512 pixels won't work well for
+                [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0)
+                and checkpoints that are not specifically fine-tuned on low resolutions.
+            width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
+                The width in pixels of the generated image. This is set to 1024 by default for the best results.
+                Anything below 512 pixels won't work well for
+                [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0)
+                and checkpoints that are not specifically fine-tuned on low resolutions.
+            num_inference_steps (`int`, *optional*, defaults to 50):
+                The number of denoising steps. More denoising steps usually lead to a higher quality image at the
+                expense of slower inference.
+            denoising_end (`float`, *optional*):
+                When specified, determines the fraction (between 0.0 and 1.0) of the total denoising process to be
+                completed before it is intentionally prematurely terminated. As a result, the returned sample will
+                still retain a substantial amount of noise as determined by the discrete timesteps selected by the
+                scheduler. The denoising_end parameter should ideally be utilized when this pipeline forms a part of a
+                "Mixture of Denoisers" multi-pipeline setup, as elaborated in [**Refining the Image
+                Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output)
+            guidance_scale (`float`, *optional*, defaults to 5.0):
+                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
+                `guidance_scale` is defined as `w` of equation 2. of [Imagen
+                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
+                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
+                usually at the expense of lower image quality.
+            negative_prompt (`str` or `List[str]`, *optional*):
+                The prompt or prompts not to guide the image generation. If not defined, one has to pass
+                `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
+                less than `1`).
+            negative_prompt_2 (`str` or `List[str]`, *optional*):
+                The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and
+                `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders
+            num_images_per_prompt (`int`, *optional*, defaults to 1):
+                The number of images to generate per prompt.
+            eta (`float`, *optional*, defaults to 0.0):
+                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to
+                [`schedulers.DDIMScheduler`], will be ignored for others.
+            generator (`torch.Generator` or `List[torch.Generator]`, *optional*):
+                One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
+                to make generation deterministic.
+            latents (`torch.FloatTensor`, *optional*):
+                Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
+                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
+                tensor will ge generated by sampling using the supplied random `generator`.
+            prompt_embeds (`torch.FloatTensor`, *optional*):
+                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
+                provided, text embeddings will be generated from `prompt` input argument.
+            negative_prompt_embeds (`torch.FloatTensor`, *optional*):
+                Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
+                weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
+                argument.
+            pooled_prompt_embeds (`torch.FloatTensor`, *optional*):
+                Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
+                If not provided, pooled text embeddings will be generated from `prompt` input argument.
+            negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*):
+                Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
+                weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt`
+                input argument.
+            output_type (`str`, *optional*, defaults to `"pil"`):
+                The output format of the generate image. Choose between
+                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
+            return_dict (`bool`, *optional*, defaults to `True`):
+                Whether or not to return a [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] instead
+                of a plain tuple.
+            cross_attention_kwargs (`dict`, *optional*):
+                A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
+                `self.processor` in
+                [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
+            guidance_rescale (`float`, *optional*, defaults to 0.0):
+                Guidance rescale factor proposed by [Common Diffusion Noise Schedules and Sample Steps are
+                Flawed](https://arxiv.org/pdf/2305.08891.pdf) `guidance_scale` is defined as `φ` in equation 16. of
+                [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf).
+                Guidance rescale factor should fix overexposure when using zero terminal SNR.
+            original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)):
+                If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled.
+                `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as
+                explained in section 2.2 of
+                [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
+            crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)):
+                `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position
+                `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting
+                `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of
+                [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
+            target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)):
+                For most cases, `target_size` should be set to the desired height and width of the generated image. If
+                not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in
+                section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
+            negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)):
+                To negatively condition the generation process based on a specific image resolution. Part of SDXL's
+                micro-conditioning as explained in section 2.2 of
+                [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
+                information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
+            negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)):
+                To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's
+                micro-conditioning as explained in section 2.2 of
+                [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
+                information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
+            negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)):
+                To negatively condition the generation process based on a target image resolution. It should be as same
+                as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of
+                [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
+                information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
+            callback_on_step_end (`Callable`, *optional*):
+                A function that calls at the end of each denoising steps during the inference. The function is called
+                with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int,
+                callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by
+                `callback_on_step_end_tensor_inputs`.
+            callback_on_step_end_tensor_inputs (`List`, *optional*):
+                The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
+                will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
+                `._callback_tensor_inputs` attribute of your pipeine class.
+
+        Examples:
+
+        Returns:
+            [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] or `tuple`:
+            [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] if `return_dict` is True, otherwise a
+            `tuple`. When returning a tuple, the first element is a list with the generated images.
+        """
+        callback = kwargs.pop("callback", None)
+        callback_steps = kwargs.pop("callback_steps", None)
+
+        if callback is not None:
+            deprecate(
+                "callback",
+                "1.0.0",
+                "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`",
+            )
+        if callback_steps is not None:
+            deprecate(
+                "callback_steps",
+                "1.0.0",
+                "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`",
+            )
+
+        # 0. Default height and width to unet
+        height = height or self.default_sample_size * self.vae_scale_factor
+        width = width or self.default_sample_size * self.vae_scale_factor
+
+        original_size = original_size or (height, width)
+        target_size = target_size or (height, width)
+
+        # 1. Check inputs. Raise error if not correct
+        self.check_inputs(
+            prompt,
+            prompt_2,
+            height,
+            width,
+            callback_steps,
+            negative_prompt,
+            negative_prompt_2,
+            prompt_embeds,
+            negative_prompt_embeds,
+            pooled_prompt_embeds,
+            negative_pooled_prompt_embeds,
+            callback_on_step_end_tensor_inputs,
+        )
+
+        self._guidance_scale = guidance_scale
+        self._guidance_rescale = guidance_rescale
+        self._clip_skip = clip_skip
+        self._cross_attention_kwargs = cross_attention_kwargs
+        self._denoising_end = denoising_end
+
+        # 2. Define call parameters
+        if prompt is not None and isinstance(prompt, str):
+            batch_size = 1
+        elif prompt is not None and isinstance(prompt, list):
+            batch_size = len(prompt)
+        else:
+            batch_size = prompt_embeds.shape[0]
+
+        device = self._execution_device
+
+        # 3. Encode input prompt
+        lora_scale = (
+            self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None
+        )
+
+        (
+            prompt_embeds,
+            negative_prompt_embeds,
+            pooled_prompt_embeds,
+            negative_pooled_prompt_embeds,
+        ) = self.encode_prompt(
+            prompt=prompt,
+            prompt_2=prompt_2,
+            device=device,
+            num_images_per_prompt=num_images_per_prompt,
+            do_classifier_free_guidance=self.do_classifier_free_guidance,
+            negative_prompt=negative_prompt,
+            negative_prompt_2=negative_prompt_2,
+            prompt_embeds=prompt_embeds,
+            negative_prompt_embeds=negative_prompt_embeds,
+            pooled_prompt_embeds=pooled_prompt_embeds,
+            negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,
+            lora_scale=lora_scale,
+            clip_skip=self.clip_skip,
+        )
+
+        # 4. Prepare timesteps
+        self.scheduler.set_timesteps(num_inference_steps, device=device)
+
+        timesteps = self.scheduler.timesteps
+
+        # 5. Prepare latent variables
+        num_channels_latents = self.unet.config.in_channels
+        latents = self.prepare_latents(
+            batch_size * num_images_per_prompt,
+            num_channels_latents,
+            height,
+            width,
+            prompt_embeds.dtype,
+            device,
+            generator,
+            latents,
+        )
+
+        # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
+        extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
+
+        if share_queries:
+            query_store = QueryStore(**query_store_kwargs)
+        else:
+            query_store = None
+
+        self.attention_store = AttentionStore(attention_store_kwargs)
+        register_extended_self_attn(self.unet, self.attention_store, extended_attn_kwargs)
+
+        # 7. Prepare added time ids & embeddings
+        add_text_embeds = pooled_prompt_embeds
+        if self.text_encoder_2 is None:
+            text_encoder_projection_dim = int(pooled_prompt_embeds.shape[-1])
+        else:
+            text_encoder_projection_dim = self.text_encoder_2.config.projection_dim
+
+        add_time_ids = self._get_add_time_ids(
+            original_size,
+            crops_coords_top_left,
+            target_size,
+            dtype=prompt_embeds.dtype,
+            text_encoder_projection_dim=text_encoder_projection_dim,
+        )
+        if negative_original_size is not None and negative_target_size is not None:
+            negative_add_time_ids = self._get_add_time_ids(
+                negative_original_size,
+                negative_crops_coords_top_left,
+                negative_target_size,
+                dtype=prompt_embeds.dtype,
+                text_encoder_projection_dim=text_encoder_projection_dim,
+            )
+        else:
+            negative_add_time_ids = add_time_ids
+
+        if self.do_classifier_free_guidance:
+            prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0)
+            add_text_embeds = torch.cat([negative_pooled_prompt_embeds, add_text_embeds], dim=0)
+            add_time_ids = torch.cat([negative_add_time_ids, add_time_ids], dim=0)
+
+        prompt_embeds = prompt_embeds.to(device)
+        add_text_embeds = add_text_embeds.to(device)
+        add_time_ids = add_time_ids.to(device).repeat(batch_size * num_images_per_prompt, 1)
+
+        # 8. Denoising loop
+        num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)
+
+        # 8.1 Apply denoising_end
+        if (
+            self.denoising_end is not None
+            and isinstance(self.denoising_end, float)
+            and self.denoising_end > 0
+            and self.denoising_end < 1
+        ):
+            discrete_timestep_cutoff = int(
+                round(
+                    self.scheduler.config.num_train_timesteps
+                    - (self.denoising_end * self.scheduler.config.num_train_timesteps)
+                )
+            )
+            num_inference_steps = len(list(filter(lambda ts: ts >= discrete_timestep_cutoff, timesteps)))
+            timesteps = timesteps[:num_inference_steps]
+
+        # 9. Optionally get Guidance Scale Embedding
+        timestep_cond = None
+        if self.unet.config.time_cond_proj_dim is not None:
+            guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt)
+            timestep_cond = self.get_guidance_scale_embedding(
+                guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim
+            ).to(device=device, dtype=latents.dtype)
+
+        self._num_timesteps = len(timesteps)
+
+        if instance_latents is not None:
+            n_instances = instance_latents.shape[0]
+            instance_noise = latents[:n_instances].clone()
+
+        with self.progress_bar(total=num_inference_steps) as progress_bar:
+            for i, t in enumerate(timesteps):
+                self.attention_store.curr_iter = i
+
+                if instance_latents is not None:
+                    noised_instances = self.scheduler.add_noise(instance_latents, instance_noise, t.repeat(n_instances).long())
+                    latents[:n_instances] = noised_instances
+
+                # expand the latents if we are doing classifier free guidance
+                latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents
+                latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)
+
+                # predict the noise residual
+                added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids}
+
+                if share_queries and (i >= query_store.t_range[0] and i <= query_store.t_range[1]):
+                    query_store.set_mode('cache')
+                    noise_pred_vanilla = self.unet(
+                        latent_model_input,
+                        t,
+                        encoder_hidden_states=prompt_embeds,
+                        timestep_cond=timestep_cond,
+                        cross_attention_kwargs={'query_store': query_store, 
+                                                'perform_extend_attn': False, 
+                                                'record_attention': False},
+                        added_cond_kwargs=added_cond_kwargs,
+                        return_dict=False,
+                    )[0]
+
+                    query_store.set_mode('inject')
+
+                noise_pred = self.unet(
+                       latent_model_input,
+                        t,
+                        encoder_hidden_states=prompt_embeds,
+                        timestep_cond=timestep_cond,
+                        cross_attention_kwargs={'query_store': query_store, 
+                                                'perform_extend_attn': True, 
+                                                'record_attention': True, 
+                                                'feature_injector': feature_injector,
+                                                'anchors_cache': anchors_cache},
+                        added_cond_kwargs=added_cond_kwargs,
+                        return_dict=False,
+                    )[0]
+
+                # perform guidance
+                if self.do_classifier_free_guidance:
+                    noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
+                    noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond)
+
+                if self.do_classifier_free_guidance and self.guidance_rescale > 0.0:
+                    # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf
+                    noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=self.guidance_rescale)
+
+                # compute the previous noisy sample x_t -> x_t-1
+                latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]
+
+                if callback_on_step_end is not None:
+                    callback_kwargs = {}
+                    for k in callback_on_step_end_tensor_inputs:
+                        callback_kwargs[k] = locals()[k]
+                    callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
+
+                    latents = callback_outputs.pop("latents", latents)
+                    prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
+                    negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds)
+                    add_text_embeds = callback_outputs.pop("add_text_embeds", add_text_embeds)
+                    negative_pooled_prompt_embeds = callback_outputs.pop(
+                        "negative_pooled_prompt_embeds", negative_pooled_prompt_embeds
+                    )
+                    add_time_ids = callback_outputs.pop("add_time_ids", add_time_ids)
+                    negative_add_time_ids = callback_outputs.pop("negative_add_time_ids", negative_add_time_ids)
+
+                # call the callback, if provided
+                if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
+                    progress_bar.update()
+                    if callback is not None and i % callback_steps == 0:
+                        step_idx = i // getattr(self.scheduler, "order", 1)
+                        callback(step_idx, t, latents)
+
+                if XLA_AVAILABLE:
+                    # xm.mark_step()
+                    pass
+                
+                # Update attention store mask
+                self.attention_store.aggregate_last_steps_attention()
+        if not output_type == "latent":
+            # make sure the VAE is in float32 mode, as it overflows in float16
+            #print(self.vae.config.force_upcast)
+            needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast
+            #print(needs_upcasting,112233)
+            if needs_upcasting:
+                self.upcast_vae()
+                latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype)
+            elif latents.dtype != self.vae.dtype:
+                if torch.backends.mps.is_available():
+                    # some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272
+                    self.vae = self.vae.to(latents.dtype)
+                
+                # unscale/denormalize the latents
+                # denormalize with the mean and std if available and not None
+            has_latents_mean = hasattr(self.vae.config, "latents_mean") and self.vae.config.latents_mean is not None
+            has_latents_std = hasattr(self.vae.config, "latents_std") and self.vae.config.latents_std is not None
+            if has_latents_mean and has_latents_std:
+                latents_mean = (
+                    torch.tensor(self.vae.config.latents_mean).view(1, 4, 1, 1).to(latents.device, latents.dtype)
+                )
+                latents_std = (
+                    torch.tensor(self.vae.config.latents_std).view(1, 4, 1, 1).to(latents.device, latents.dtype)
+                )
+                latents = latents * latents_std / self.vae.config.scaling_factor + latents_mean
+            else:
+                latents = latents / self.vae.config.scaling_factor
+                
+            image = self.vae.decode(latents, return_dict=False)[0]
+            #image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0]
+
+            # cast back to fp16 if needed
+            if needs_upcasting:
+                self.vae.to(dtype=torch.float16)
+        else:
+            image = latents
+
+        if not output_type == "latent":
+            # apply watermark if available
+            # if self.watermark is not None:
+            #     image = self.watermark.apply_watermark(image)
+
+            image = self.image_processor.postprocess(image, output_type=output_type)
+        # Offload all models
+        self.maybe_free_model_hooks()
+       
+        if not return_dict:
+            return (image,)
+
+        return StableDiffusionXLPipelineOutput(images=image)
\ No newline at end of file
diff --git a/consistory/consistory_run.py b/consistory/consistory_run.py
new file mode 100644
index 0000000..f8a2e25
--- /dev/null
+++ b/consistory/consistory_run.py
@@ -0,0 +1,369 @@
+# Copyright (C) 2024 NVIDIA Corporation.  All rights reserved.
+#
+# This work is licensed under the LICENSE file
+# located at the root directory.
+import os.path
+
+import torch
+from diffusers import DDIMScheduler, UNet2DConditionModel
+from .consistory_unet_sdxl import ConsistorySDXLUNet2DConditionModel
+from .consistory_pipeline import ConsistoryExtendAttnSDXLPipeline
+from .consistory_utils import FeatureInjector, AnchorCache
+from .utils.general_utils import *
+import gc
+import folder_paths
+from .utils.ptp_utils import view_images
+
+LATENT_RESOLUTIONS = [32, 64]
+
+def clear_memory():
+    torch.cuda.empty_cache()
+    gc.collect()
+
+
+def load_pipeline(repo_id,unet_path,inject,gpu_id=0):
+    float_type = torch.float16
+    #sd_id = "stabilityai/stable-diffusion-xl-base-1.0"
+    device = torch.device(f'cuda:{gpu_id}') if torch.cuda.is_available() else torch.device('cpu')
+    if repo_id:
+        unet = ConsistorySDXLUNet2DConditionModel.from_pretrained(repo_id, subfolder="unet", torch_dtype=float_type)
+        scheduler = DDIMScheduler.from_pretrained(repo_id, subfolder="scheduler")
+        story_pipeline = ConsistoryExtendAttnSDXLPipeline.from_pretrained(
+            repo_id, unet=unet, torch_dtype=float_type, variant="fp16", use_safetensors=True, scheduler=scheduler
+        ).to(device)
+    elif not repo_id and unet_path:
+        config_file = os.path.join(folder_paths.base_path, "custom_nodes/ComfyUI_StoryDiffusion/local_repo/unet/config.json")
+        sdxl_repo = os.path.join(folder_paths.base_path, "custom_nodes/ComfyUI_StoryDiffusion/local_repo")
+        original_config_file = os.path.join(folder_paths.base_path, "custom_nodes/ComfyUI_StoryDiffusion/config/sd_xl_base.yaml")
+        from safetensors.torch import load_file
+        from diffusers.pipelines.stable_diffusion.convert_from_ckpt import convert_ldm_unet_checkpoint
+        state_dict = load_file(unet_path)
+        unet_config = ConsistorySDXLUNet2DConditionModel.load_config(config_file)
+        Unet = ConsistorySDXLUNet2DConditionModel.from_config(unet_config).to(device,torch.float16)
+        state_dict = convert_ldm_unet_checkpoint(state_dict, Unet.config)
+        Unet.load_state_dict(state_dict, strict=False)
+        del state_dict
+        clear_memory()
+        scheduler = DDIMScheduler.from_pretrained(sdxl_repo, subfolder="scheduler")
+        try:
+            story_pipeline = ConsistoryExtendAttnSDXLPipeline.from_single_file(
+                unet_path,unet=Unet, config=sdxl_repo, original_config=original_config_file, torch_dtype=float_type, variant="fp16", use_safetensors=True, scheduler=scheduler
+            ).to(device)
+        except:
+            story_pipeline = ConsistoryExtendAttnSDXLPipeline.from_single_file(
+                unet_path, unet=Unet,config=sdxl_repo, original_config_file=original_config_file, torch_dtype=float_type, variant="fp16", use_safetensors=True, scheduler=scheduler
+            ).to(device)
+       
+    else:
+        raise "need a repo or chocie a sdxl checkpoints"
+    
+    story_pipeline.enable_freeu(s1=0.6, s2=0.4, b1=1.1, b2=1.2)
+    return story_pipeline
+
+def create_anchor_mapping(bsz, anchor_indices=[0]):
+    anchor_mapping = torch.eye(bsz, dtype=torch.bool)
+    for anchor_idx in anchor_indices:
+        anchor_mapping[:, anchor_idx] = True
+
+    return anchor_mapping
+
+def create_token_indices(prompts, batch_size, concept_token, tokenizer):
+    if isinstance(concept_token, str):
+        concept_token = [concept_token]
+
+    concept_token_id = [tokenizer.encode(x, add_special_tokens=False)[0] for x in concept_token]
+    tokens = tokenizer.batch_encode_plus(prompts, padding=True, return_tensors='pt')['input_ids']
+
+    token_indices = torch.full((len(concept_token), batch_size), -1, dtype=torch.int64)
+    for i, token_id in enumerate(concept_token_id):
+        batch_loc, token_loc = torch.where(tokens == token_id)
+        token_indices[i, batch_loc] = token_loc
+
+    return token_indices
+
+def create_latents(story_pipeline, seed, batch_size, same_latent, device, float_type):
+    # if seed is int
+    if isinstance(seed, int):
+        #g = torch.Generator('cuda').manual_seed(seed)
+        g = torch.Generator(device).manual_seed(seed)
+        shape = (batch_size, story_pipeline.unet.config.in_channels, 128, 128)
+        latents = randn_tensor(shape, generator=g, device=device, dtype=float_type)
+    elif isinstance(seed, list):
+        shape = (batch_size, story_pipeline.unet.config.in_channels, 128, 128)
+        latents = torch.empty(shape, device=device, dtype=float_type)
+        for i, seed_i in enumerate(seed):
+            #g = torch.Generator('cuda').manual_seed(seed_i)
+            g = torch.Generator(device).manual_seed(seed_i)
+            curr_latent = randn_tensor(shape, generator=g, device=device, dtype=float_type)
+            latents[i] = curr_latent[i]
+
+    if same_latent:
+        latents = latents[:1].repeat(batch_size, 1, 1, 1)
+
+    return latents, g
+
+# Batch inference
+def run_batch_generation(story_pipeline, prompts, concept_token,negative_prompt,
+                        seed=40, n_steps=50, mask_dropout=0.5,
+                        same_latent=False, share_queries=True,
+                        perform_sdsa=True, perform_injection=True,
+                        downscale_rate=4, n_achors=2):
+    device = story_pipeline.device
+    tokenizer = story_pipeline.tokenizer
+    float_type = story_pipeline.dtype
+    unet = story_pipeline.unet
+
+    batch_size = len(prompts)
+
+    token_indices = create_token_indices(prompts, batch_size, concept_token, tokenizer)
+    anchor_mappings = create_anchor_mapping(batch_size, anchor_indices=list(range(n_achors)))
+
+    default_attention_store_kwargs = {
+        'token_indices': token_indices,
+        'mask_dropout': mask_dropout,
+        'extended_mapping': anchor_mappings
+    }
+
+    default_extended_attn_kwargs = {'extend_kv_unet_parts': ['up']}
+    query_store_kwargs= {'t_range': [0,n_steps//10], 'strength_start': 0.9, 'strength_end': 0.81836735}
+
+    latents, g = create_latents(story_pipeline, seed, batch_size, same_latent, device, float_type)
+
+    # ------------------ #
+    # Extended attention First Run #
+
+    if perform_sdsa:
+        extended_attn_kwargs = {**default_extended_attn_kwargs, 't_range': [(1, n_steps)]}
+    else:
+        extended_attn_kwargs = {**default_extended_attn_kwargs, 't_range': []}
+
+    print(extended_attn_kwargs['t_range'])
+    out = story_pipeline(prompt=prompts,negative_prompt=negative_prompt, generator=g, latents=latents,
+                        attention_store_kwargs=default_attention_store_kwargs,
+                        extended_attn_kwargs=extended_attn_kwargs,
+                        share_queries=share_queries,
+                        query_store_kwargs=query_store_kwargs,
+                        num_inference_steps=n_steps)
+    
+    # Extended attention with nn_map #
+    
+    if  perform_injection:
+        last_masks = story_pipeline.attention_store.last_mask
+        dift_features = unet.latent_store.dift_features['251_0'][batch_size:]
+        unet.latent_store.reset() # turn to {}
+        del out
+        clear_memory()
+        dift_features = torch.stack([gaussian_smooth(x, kernel_size=3, sigma=1) for x in dift_features], dim=0)
+        nn_map, nn_distances = cyclic_nn_map(dift_features, last_masks, LATENT_RESOLUTIONS, device)
+        
+        clear_memory()
+        feature_injector = FeatureInjector(nn_map, nn_distances, last_masks,
+                                           inject_range_alpha=[(n_steps // 10, n_steps // 3, 0.8)],
+                                           swap_strategy='min', inject_unet_parts=['up', 'down'],
+                                           dist_thr='dynamic')
+        
+        out = story_pipeline(prompt=prompts, generator=g, latents=latents,
+                             attention_store_kwargs=default_attention_store_kwargs,
+                             extended_attn_kwargs=extended_attn_kwargs,
+                             share_queries=share_queries,
+                             query_store_kwargs=query_store_kwargs,
+                             feature_injector=feature_injector,
+                             num_inference_steps=n_steps)
+        #img_all = view_images([np.array(x) for x in out.images], display_image=False, downscale_rate=downscale_rate)
+        # display_attn_maps(story_pipeline.attention_store.last_mask, out.images)
+        clear_memory()
+
+    return out.images
+
+# Anchors
+def run_anchor_generation(story_pipeline, prompts, concept_token,negative_prompt,
+                        seed=40, n_steps=50, mask_dropout=0.5,
+                        same_latent=False, share_queries=True,
+                        perform_sdsa=True, perform_injection=True,
+                        downscale_rate=4, cache_cpu_offloading=False):
+    device = story_pipeline.device
+    tokenizer = story_pipeline.tokenizer
+    float_type = story_pipeline.dtype
+    unet = story_pipeline.unet
+
+    batch_size = len(prompts)
+
+    token_indices = create_token_indices(prompts, batch_size, concept_token, tokenizer)
+
+    default_attention_store_kwargs = {
+        'token_indices': token_indices,
+        'mask_dropout': mask_dropout
+    }
+
+    default_extended_attn_kwargs = {'extend_kv_unet_parts': ['up']}
+    query_store_kwargs={'t_range': [0,n_steps//10], 'strength_start': 0.9, 'strength_end': 0.81836735}
+
+    latents, g = create_latents(story_pipeline, seed, batch_size, same_latent, device, float_type)
+
+    anchor_cache_first_stage = AnchorCache()
+    anchor_cache_second_stage = AnchorCache()
+
+    # ------------------ #
+    # Extended attention First Run #
+
+    if perform_sdsa:
+        extended_attn_kwargs = {**default_extended_attn_kwargs, 't_range': [(1, n_steps)]}
+    else:
+        extended_attn_kwargs = {**default_extended_attn_kwargs, 't_range': []}
+
+    print(extended_attn_kwargs['t_range'])
+    out = story_pipeline(prompt=prompts,negative_prompt=negative_prompt, generator=g, latents=latents,
+                        attention_store_kwargs=default_attention_store_kwargs,
+                        extended_attn_kwargs=extended_attn_kwargs,
+                        share_queries=share_queries,
+                        query_store_kwargs=query_store_kwargs,
+                        anchors_cache=anchor_cache_first_stage,
+                        num_inference_steps=n_steps)
+    
+    last_masks = story_pipeline.attention_store.last_mask
+    
+    dift_features = unet.latent_store.dift_features['251_0'][batch_size:]
+    unet.latent_store.reset()  # turn to {}
+    clear_memory()
+    dift_features = torch.stack([gaussian_smooth(x, kernel_size=3, sigma=1) for x in dift_features], dim=0)
+    
+    anchor_cache_first_stage.dift_cache = dift_features
+    anchor_cache_first_stage.anchors_last_mask = last_masks
+    
+    if cache_cpu_offloading:
+        anchor_cache_first_stage.to_device(torch.device('cpu'))
+    
+    nn_map, nn_distances = cyclic_nn_map(dift_features, last_masks, LATENT_RESOLUTIONS, device)
+    clear_memory()
+    # ------------------ #
+    # Extended attention with nn_map #
+    
+    if  perform_injection:
+        
+        feature_injector = FeatureInjector(nn_map, nn_distances, last_masks, inject_range_alpha=[(n_steps//10, n_steps//3,0.8)], 
+                                        swap_strategy='min', inject_unet_parts=['up', 'down'], dist_thr='dynamic')
+
+        out = story_pipeline(prompt=prompts,negative_prompt=negative_prompt, generator=g, latents=latents,
+                            attention_store_kwargs=default_attention_store_kwargs,
+                            extended_attn_kwargs=extended_attn_kwargs,
+                            share_queries=share_queries,
+                            query_store_kwargs=query_store_kwargs,
+                            feature_injector=feature_injector,
+                            anchors_cache=anchor_cache_second_stage,
+                            num_inference_steps=n_steps)
+        #img_all = view_images([np.array(x) for x in out.images], display_image=False, downscale_rate=downscale_rate)
+        # display_attn_maps(story_pipeline.attention_store.last_mask, out.images)
+
+        anchor_cache_second_stage.dift_cache = dift_features
+        anchor_cache_second_stage.anchors_last_mask = last_masks
+
+        if cache_cpu_offloading:
+            anchor_cache_second_stage.to_device(torch.device('cpu'))
+        
+        clear_memory()
+    # else:
+    #     img_all = view_images([np.array(x) for x in out.images], display_image=False, downscale_rate=downscale_rate)
+    
+    return out.images, anchor_cache_first_stage, anchor_cache_second_stage
+
+def run_extra_generation(story_pipeline, prompts, concept_token, negative_prompt,
+                         anchor_cache_first_stage, anchor_cache_second_stage,
+                         seed=40, n_steps=50, mask_dropout=0.5,
+                         same_latent=False, share_queries=True,
+                         perform_sdsa=True, perform_injection=True,
+                         downscale_rate=4, cache_cpu_offloading=False):
+    device = story_pipeline.device
+    tokenizer = story_pipeline.tokenizer
+    float_type = story_pipeline.dtype
+    unet = story_pipeline.unet
+
+    batch_size = len(prompts)
+
+    token_indices = create_token_indices(prompts, batch_size, concept_token, tokenizer)
+
+    default_attention_store_kwargs = {
+        'token_indices': token_indices,
+        'mask_dropout': mask_dropout
+    }
+
+    default_extended_attn_kwargs = {'extend_kv_unet_parts': ['up']}
+    query_store_kwargs={'t_range': [0,n_steps//10], 'strength_start': 0.9, 'strength_end': 0.81836735}
+
+    extra_batch_size = batch_size + 2
+    if isinstance(seed, list):
+        seed = [seed[0], seed[0], *seed]
+
+    latents, g = create_latents(story_pipeline, seed, extra_batch_size, same_latent, device, float_type)
+    latents = latents[2:]
+
+    anchor_cache_first_stage.set_mode_inject()
+    anchor_cache_second_stage.set_mode_inject()
+
+    # ------------------ #
+    # Extended attention First Run #
+
+    if cache_cpu_offloading:
+        anchor_cache_first_stage.to_device(device)
+
+    if perform_sdsa:
+        extended_attn_kwargs = {**default_extended_attn_kwargs, 't_range': [(1, n_steps)]}
+    else:
+        extended_attn_kwargs = {**default_extended_attn_kwargs, 't_range': []}
+
+    print(extended_attn_kwargs['t_range'])
+    out = story_pipeline(prompt=prompts,negative_prompt=negative_prompt, generator=g, latents=latents,
+                        attention_store_kwargs=default_attention_store_kwargs,
+                        extended_attn_kwargs=extended_attn_kwargs,
+                        share_queries=share_queries,
+                        query_store_kwargs=query_store_kwargs,
+                        anchors_cache=anchor_cache_first_stage,
+                        num_inference_steps=n_steps)
+   
+
+    # ------------------ #
+    # Extended attention with nn_map #
+    last_masks = story_pipeline.attention_store.last_mask
+    dift_features = unet.latent_store.dift_features['251_0'][batch_size:]
+    unet.latent_store.reset()  # turn to {}
+    clear_memory()
+    dift_features = torch.stack([gaussian_smooth(x, kernel_size=3, sigma=1) for x in dift_features], dim=0)
+    
+    anchor_dift_features = anchor_cache_first_stage.dift_cache
+    anchor_last_masks = anchor_cache_first_stage.anchors_last_mask
+    
+    nn_map, nn_distances = anchor_nn_map(dift_features, anchor_dift_features, last_masks, anchor_last_masks,
+                                         LATENT_RESOLUTIONS, device)
+    
+    if cache_cpu_offloading:
+        anchor_cache_first_stage.to_device(torch.device('cpu'))
+    
+    clear_memory()
+    
+    if  perform_injection:
+
+        if cache_cpu_offloading:
+            anchor_cache_second_stage.to_device(device)
+
+        feature_injector = FeatureInjector(nn_map, nn_distances, last_masks, inject_range_alpha=[(n_steps//10, n_steps//3,0.8)], 
+                                        swap_strategy='min', inject_unet_parts=['up', 'down'], dist_thr='dynamic')
+
+        out = story_pipeline(prompt=prompts,negative_prompt=negative_prompt, generator=g, latents=latents,
+                            attention_store_kwargs=default_attention_store_kwargs,
+                            extended_attn_kwargs=extended_attn_kwargs,
+                            share_queries=share_queries,
+                            query_store_kwargs=query_store_kwargs,
+                            feature_injector=feature_injector,
+                            anchors_cache=anchor_cache_second_stage,
+                            num_inference_steps=n_steps)
+        
+        
+        #img_all = view_images([np.array(x) for x in out.images], display_image=False, downscale_rate=downscale_rate)
+        # display_attn_maps(story_pipeline.attention_store.last_mask, out.images)
+
+        if cache_cpu_offloading:
+            anchor_cache_second_stage.to_device(torch.device('cpu'))
+        clear_memory()
+    # else:
+    #     img_all = view_images([np.array(x) for x in out.images], display_image=False, downscale_rate=downscale_rate)
+    
+    return out.images
\ No newline at end of file
diff --git a/consistory/consistory_unet_sdxl.py b/consistory/consistory_unet_sdxl.py
new file mode 100644
index 0000000..30e4553
--- /dev/null
+++ b/consistory/consistory_unet_sdxl.py
@@ -0,0 +1,1195 @@
+# Copyright 2023 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Not a contribution
+# Changes made by NVIDIA CORPORATION & AFFILIATES enabling ConsiStory or otherwise documented as NVIDIA-proprietary
+# are not a contribution and subject to the license under the LICENSE file located at the root directory.
+
+
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+import torch
+import torch.nn as nn
+import torch.utils.checkpoint
+
+from diffusers.configuration_utils import ConfigMixin, register_to_config
+from diffusers.loaders import UNet2DConditionLoadersMixin
+from diffusers.utils import USE_PEFT_BACKEND, BaseOutput, deprecate, logging, scale_lora_layers, unscale_lora_layers
+from diffusers.models.activations import get_activation
+from diffusers.models.attention_processor import (
+    ADDED_KV_ATTENTION_PROCESSORS,
+    CROSS_ATTENTION_PROCESSORS,
+    AttentionProcessor,
+    AttnAddedKVProcessor,
+    AttnProcessor,
+)
+from diffusers.models.embeddings import (
+    GaussianFourierProjection,
+    ImageHintTimeEmbedding,
+    ImageProjection,
+    ImageTimeEmbedding,
+    TextImageProjection,
+    TextImageTimeEmbedding,
+    TextTimeEmbedding,
+    TimestepEmbedding,
+    Timesteps,
+)
+try:
+    from diffusers.models.embeddings import PositionNet
+except:
+    from diffusers.models.embeddings import GLIGENTextBoundingboxProjection as PositionNet
+    
+from diffusers.models.modeling_utils import ModelMixin
+try:
+    from diffusers.models.unet_2d_blocks import (
+        UNetMidBlock2D,
+        UNetMidBlock2DCrossAttn,
+        UNetMidBlock2DSimpleCrossAttn,
+        get_down_block,
+        get_up_block,
+    )
+except:
+    from diffusers.models.unets.unet_2d_blocks import (
+        UNetMidBlock2D,
+        UNetMidBlock2DCrossAttn,
+        UNetMidBlock2DSimpleCrossAttn,
+        get_down_block,
+        get_up_block,
+    )
+    
+
+from .consistory_utils import DIFTLatentStore
+
+logger = logging.get_logger(__name__)  # pylint: disable=invalid-name
+
+
+@dataclass
+class UNet2DConditionOutput(BaseOutput):
+    """
+    The output of [`UNet2DConditionModel`].
+
+    Args:
+        sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
+            The hidden states output conditioned on `encoder_hidden_states` input. Output of last layer of model.
+    """
+
+    sample: torch.FloatTensor = None
+
+
+class ConsistorySDXLUNet2DConditionModel(ModelMixin, ConfigMixin, UNet2DConditionLoadersMixin):
+    r"""
+    A conditional 2D UNet model that takes a noisy sample, conditional state, and a timestep and returns a sample
+    shaped output.
+
+    This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented
+    for all models (such as downloading or saving).
+
+    Parameters:
+        sample_size (`int` or `Tuple[int, int]`, *optional*, defaults to `None`):
+            Height and width of input/output sample.
+        in_channels (`int`, *optional*, defaults to 4): Number of channels in the input sample.
+        out_channels (`int`, *optional*, defaults to 4): Number of channels in the output.
+        center_input_sample (`bool`, *optional*, defaults to `False`): Whether to center the input sample.
+        flip_sin_to_cos (`bool`, *optional*, defaults to `False`):
+            Whether to flip the sin to cos in the time embedding.
+        freq_shift (`int`, *optional*, defaults to 0): The frequency shift to apply to the time embedding.
+        down_block_types (`Tuple[str]`, *optional*, defaults to `("CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D")`):
+            The tuple of downsample blocks to use.
+        mid_block_type (`str`, *optional*, defaults to `"UNetMidBlock2DCrossAttn"`):
+            Block type for middle of UNet, it can be one of `UNetMidBlock2DCrossAttn`, `UNetMidBlock2D`, or
+            `UNetMidBlock2DSimpleCrossAttn`. If `None`, the mid block layer is skipped.
+        up_block_types (`Tuple[str]`, *optional*, defaults to `("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D")`):
+            The tuple of upsample blocks to use.
+        only_cross_attention(`bool` or `Tuple[bool]`, *optional*, default to `False`):
+            Whether to include self-attention in the basic transformer blocks, see
+            [`~models.attention.BasicTransformerBlock`].
+        block_out_channels (`Tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`):
+            The tuple of output channels for each block.
+        layers_per_block (`int`, *optional*, defaults to 2): The number of layers per block.
+        downsample_padding (`int`, *optional*, defaults to 1): The padding to use for the downsampling convolution.
+        mid_block_scale_factor (`float`, *optional*, defaults to 1.0): The scale factor to use for the mid block.
+        dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.
+        act_fn (`str`, *optional*, defaults to `"silu"`): The activation function to use.
+        norm_num_groups (`int`, *optional*, defaults to 32): The number of groups to use for the normalization.
+            If `None`, normalization and activation layers is skipped in post-processing.
+        norm_eps (`float`, *optional*, defaults to 1e-5): The epsilon to use for the normalization.
+        cross_attention_dim (`int` or `Tuple[int]`, *optional*, defaults to 1280):
+            The dimension of the cross attention features.
+        transformer_layers_per_block (`int`, `Tuple[int]`, or `Tuple[Tuple]` , *optional*, defaults to 1):
+            The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`]. Only relevant for
+            [`~models.unet_2d_blocks.CrossAttnDownBlock2D`], [`~models.unet_2d_blocks.CrossAttnUpBlock2D`],
+            [`~models.unet_2d_blocks.UNetMidBlock2DCrossAttn`].
+       reverse_transformer_layers_per_block : (`Tuple[Tuple]`, *optional*, defaults to None):
+            The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`], in the upsampling
+            blocks of the U-Net. Only relevant if `transformer_layers_per_block` is of type `Tuple[Tuple]` and for
+            [`~models.unet_2d_blocks.CrossAttnDownBlock2D`], [`~models.unet_2d_blocks.CrossAttnUpBlock2D`],
+            [`~models.unet_2d_blocks.UNetMidBlock2DCrossAttn`].
+        encoder_hid_dim (`int`, *optional*, defaults to None):
+            If `encoder_hid_dim_type` is defined, `encoder_hidden_states` will be projected from `encoder_hid_dim`
+            dimension to `cross_attention_dim`.
+        encoder_hid_dim_type (`str`, *optional*, defaults to `None`):
+            If given, the `encoder_hidden_states` and potentially other embeddings are down-projected to text
+            embeddings of dimension `cross_attention` according to `encoder_hid_dim_type`.
+        attention_head_dim (`int`, *optional*, defaults to 8): The dimension of the attention heads.
+        num_attention_heads (`int`, *optional*):
+            The number of attention heads. If not defined, defaults to `attention_head_dim`
+        resnet_time_scale_shift (`str`, *optional*, defaults to `"default"`): Time scale shift config
+            for ResNet blocks (see [`~models.resnet.ResnetBlock2D`]). Choose from `default` or `scale_shift`.
+        class_embed_type (`str`, *optional*, defaults to `None`):
+            The type of class embedding to use which is ultimately summed with the time embeddings. Choose from `None`,
+            `"timestep"`, `"identity"`, `"projection"`, or `"simple_projection"`.
+        addition_embed_type (`str`, *optional*, defaults to `None`):
+            Configures an optional embedding which will be summed with the time embeddings. Choose from `None` or
+            "text". "text" will use the `TextTimeEmbedding` layer.
+        addition_time_embed_dim: (`int`, *optional*, defaults to `None`):
+            Dimension for the timestep embeddings.
+        num_class_embeds (`int`, *optional*, defaults to `None`):
+            Input dimension of the learnable embedding matrix to be projected to `time_embed_dim`, when performing
+            class conditioning with `class_embed_type` equal to `None`.
+        time_embedding_type (`str`, *optional*, defaults to `positional`):
+            The type of position embedding to use for timesteps. Choose from `positional` or `fourier`.
+        time_embedding_dim (`int`, *optional*, defaults to `None`):
+            An optional override for the dimension of the projected time embedding.
+        time_embedding_act_fn (`str`, *optional*, defaults to `None`):
+            Optional activation function to use only once on the time embeddings before they are passed to the rest of
+            the UNet. Choose from `silu`, `mish`, `gelu`, and `swish`.
+        timestep_post_act (`str`, *optional*, defaults to `None`):
+            The second activation function to use in timestep embedding. Choose from `silu`, `mish` and `gelu`.
+        time_cond_proj_dim (`int`, *optional*, defaults to `None`):
+            The dimension of `cond_proj` layer in the timestep embedding.
+        conv_in_kernel (`int`, *optional*, default to `3`): The kernel size of `conv_in` layer. conv_out_kernel (`int`,
+        *optional*, default to `3`): The kernel size of `conv_out` layer. projection_class_embeddings_input_dim (`int`,
+        *optional*): The dimension of the `class_labels` input when
+            `class_embed_type="projection"`. Required when `class_embed_type="projection"`.
+        class_embeddings_concat (`bool`, *optional*, defaults to `False`): Whether to concatenate the time
+            embeddings with the class embeddings.
+        mid_block_only_cross_attention (`bool`, *optional*, defaults to `None`):
+            Whether to use cross attention with the mid block when using the `UNetMidBlock2DSimpleCrossAttn`. If
+            `only_cross_attention` is given as a single boolean and `mid_block_only_cross_attention` is `None`, the
+            `only_cross_attention` value is used as the value for `mid_block_only_cross_attention`. Default to `False`
+            otherwise.
+    """
+
+    _supports_gradient_checkpointing = True
+
+    @register_to_config
+    def __init__(
+        self,
+        sample_size: Optional[int] = None,
+        in_channels: int = 4,
+        out_channels: int = 4,
+        center_input_sample: bool = False,
+        flip_sin_to_cos: bool = True,
+        freq_shift: int = 0,
+        down_block_types: Tuple[str] = (
+            "CrossAttnDownBlock2D",
+            "CrossAttnDownBlock2D",
+            "CrossAttnDownBlock2D",
+            "DownBlock2D",
+        ),
+        mid_block_type: Optional[str] = "UNetMidBlock2DCrossAttn",
+        up_block_types: Tuple[str] = ("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D"),
+        only_cross_attention: Union[bool, Tuple[bool]] = False,
+        block_out_channels: Tuple[int] = (320, 640, 1280, 1280),
+        layers_per_block: Union[int, Tuple[int]] = 2,
+        downsample_padding: int = 1,
+        mid_block_scale_factor: float = 1,
+        dropout: float = 0.0,
+        act_fn: str = "silu",
+        norm_num_groups: Optional[int] = 32,
+        norm_eps: float = 1e-5,
+        cross_attention_dim: Union[int, Tuple[int]] = 1280,
+        transformer_layers_per_block: Union[int, Tuple[int], Tuple[Tuple]] = 1,
+        reverse_transformer_layers_per_block: Optional[Tuple[Tuple[int]]] = None,
+        encoder_hid_dim: Optional[int] = None,
+        encoder_hid_dim_type: Optional[str] = None,
+        attention_head_dim: Union[int, Tuple[int]] = 8,
+        num_attention_heads: Optional[Union[int, Tuple[int]]] = None,
+        dual_cross_attention: bool = False,
+        use_linear_projection: bool = False,
+        class_embed_type: Optional[str] = None,
+        addition_embed_type: Optional[str] = None,
+        addition_time_embed_dim: Optional[int] = None,
+        num_class_embeds: Optional[int] = None,
+        upcast_attention: bool = False,
+        resnet_time_scale_shift: str = "default",
+        resnet_skip_time_act: bool = False,
+        resnet_out_scale_factor: int = 1.0,
+        time_embedding_type: str = "positional",
+        time_embedding_dim: Optional[int] = None,
+        time_embedding_act_fn: Optional[str] = None,
+        timestep_post_act: Optional[str] = None,
+        time_cond_proj_dim: Optional[int] = None,
+        conv_in_kernel: int = 3,
+        conv_out_kernel: int = 3,
+        projection_class_embeddings_input_dim: Optional[int] = None,
+        attention_type: str = "default",
+        class_embeddings_concat: bool = False,
+        mid_block_only_cross_attention: Optional[bool] = None,
+        cross_attention_norm: Optional[str] = None,
+        addition_embed_type_num_heads=64,
+    ):
+        super().__init__()
+
+        self.latent_store = DIFTLatentStore(steps=list(range(261)), up_ft_indices=[0])
+        self.sample_size = sample_size
+
+        if num_attention_heads is not None:
+            raise ValueError(
+                "At the moment it is not possible to define the number of attention heads via `num_attention_heads` because of a naming issue as described in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131. Passing `num_attention_heads` will only be supported in diffusers v0.19."
+            )
+
+        # If `num_attention_heads` is not defined (which is the case for most models)
+        # it will default to `attention_head_dim`. This looks weird upon first reading it and it is.
+        # The reason for this behavior is to correct for incorrectly named variables that were introduced
+        # when this library was created. The incorrect naming was only discovered much later in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131
+        # Changing `attention_head_dim` to `num_attention_heads` for 40,000+ configurations is too backwards breaking
+        # which is why we correct for the naming here.
+        num_attention_heads = num_attention_heads or attention_head_dim
+
+        # Check inputs
+        if len(down_block_types) != len(up_block_types):
+            raise ValueError(
+                f"Must provide the same number of `down_block_types` as `up_block_types`. `down_block_types`: {down_block_types}. `up_block_types`: {up_block_types}."
+            )
+
+        if len(block_out_channels) != len(down_block_types):
+            raise ValueError(
+                f"Must provide the same number of `block_out_channels` as `down_block_types`. `block_out_channels`: {block_out_channels}. `down_block_types`: {down_block_types}."
+            )
+
+        if not isinstance(only_cross_attention, bool) and len(only_cross_attention) != len(down_block_types):
+            raise ValueError(
+                f"Must provide the same number of `only_cross_attention` as `down_block_types`. `only_cross_attention`: {only_cross_attention}. `down_block_types`: {down_block_types}."
+            )
+
+        if not isinstance(num_attention_heads, int) and len(num_attention_heads) != len(down_block_types):
+            raise ValueError(
+                f"Must provide the same number of `num_attention_heads` as `down_block_types`. `num_attention_heads`: {num_attention_heads}. `down_block_types`: {down_block_types}."
+            )
+
+        if not isinstance(attention_head_dim, int) and len(attention_head_dim) != len(down_block_types):
+            raise ValueError(
+                f"Must provide the same number of `attention_head_dim` as `down_block_types`. `attention_head_dim`: {attention_head_dim}. `down_block_types`: {down_block_types}."
+            )
+
+        if isinstance(cross_attention_dim, list) and len(cross_attention_dim) != len(down_block_types):
+            raise ValueError(
+                f"Must provide the same number of `cross_attention_dim` as `down_block_types`. `cross_attention_dim`: {cross_attention_dim}. `down_block_types`: {down_block_types}."
+            )
+
+        if not isinstance(layers_per_block, int) and len(layers_per_block) != len(down_block_types):
+            raise ValueError(
+                f"Must provide the same number of `layers_per_block` as `down_block_types`. `layers_per_block`: {layers_per_block}. `down_block_types`: {down_block_types}."
+            )
+        if isinstance(transformer_layers_per_block, list) and reverse_transformer_layers_per_block is None:
+            for layer_number_per_block in transformer_layers_per_block:
+                if isinstance(layer_number_per_block, list):
+                    raise ValueError("Must provide 'reverse_transformer_layers_per_block` if using asymmetrical UNet.")
+
+        # input
+        conv_in_padding = (conv_in_kernel - 1) // 2
+        self.conv_in = nn.Conv2d(
+            in_channels, block_out_channels[0], kernel_size=conv_in_kernel, padding=conv_in_padding
+        )
+
+        # time
+        if time_embedding_type == "fourier":
+            time_embed_dim = time_embedding_dim or block_out_channels[0] * 2
+            if time_embed_dim % 2 != 0:
+                raise ValueError(f"`time_embed_dim` should be divisible by 2, but is {time_embed_dim}.")
+            self.time_proj = GaussianFourierProjection(
+                time_embed_dim // 2, set_W_to_weight=False, log=False, flip_sin_to_cos=flip_sin_to_cos
+            )
+            timestep_input_dim = time_embed_dim
+        elif time_embedding_type == "positional":
+            time_embed_dim = time_embedding_dim or block_out_channels[0] * 4
+
+            self.time_proj = Timesteps(block_out_channels[0], flip_sin_to_cos, freq_shift)
+            timestep_input_dim = block_out_channels[0]
+        else:
+            raise ValueError(
+                f"{time_embedding_type} does not exist. Please make sure to use one of `fourier` or `positional`."
+            )
+
+        self.time_embedding = TimestepEmbedding(
+            timestep_input_dim,
+            time_embed_dim,
+            act_fn=act_fn,
+            post_act_fn=timestep_post_act,
+            cond_proj_dim=time_cond_proj_dim,
+        )
+
+        if encoder_hid_dim_type is None and encoder_hid_dim is not None:
+            encoder_hid_dim_type = "text_proj"
+            self.register_to_config(encoder_hid_dim_type=encoder_hid_dim_type)
+            logger.info("encoder_hid_dim_type defaults to 'text_proj' as `encoder_hid_dim` is defined.")
+
+        if encoder_hid_dim is None and encoder_hid_dim_type is not None:
+            raise ValueError(
+                f"`encoder_hid_dim` has to be defined when `encoder_hid_dim_type` is set to {encoder_hid_dim_type}."
+            )
+
+        if encoder_hid_dim_type == "text_proj":
+            self.encoder_hid_proj = nn.Linear(encoder_hid_dim, cross_attention_dim)
+        elif encoder_hid_dim_type == "text_image_proj":
+            # image_embed_dim DOESN'T have to be `cross_attention_dim`. To not clutter the __init__ too much
+            # they are set to `cross_attention_dim` here as this is exactly the required dimension for the currently only use
+            # case when `addition_embed_type == "text_image_proj"` (Kadinsky 2.1)`
+            self.encoder_hid_proj = TextImageProjection(
+                text_embed_dim=encoder_hid_dim,
+                image_embed_dim=cross_attention_dim,
+                cross_attention_dim=cross_attention_dim,
+            )
+        elif encoder_hid_dim_type == "image_proj":
+            # Kandinsky 2.2
+            self.encoder_hid_proj = ImageProjection(
+                image_embed_dim=encoder_hid_dim,
+                cross_attention_dim=cross_attention_dim,
+            )
+        elif encoder_hid_dim_type is not None:
+            raise ValueError(
+                f"encoder_hid_dim_type: {encoder_hid_dim_type} must be None, 'text_proj' or 'text_image_proj'."
+            )
+        else:
+            self.encoder_hid_proj = None
+
+        # class embedding
+        if class_embed_type is None and num_class_embeds is not None:
+            self.class_embedding = nn.Embedding(num_class_embeds, time_embed_dim)
+        elif class_embed_type == "timestep":
+            self.class_embedding = TimestepEmbedding(timestep_input_dim, time_embed_dim, act_fn=act_fn)
+        elif class_embed_type == "identity":
+            self.class_embedding = nn.Identity(time_embed_dim, time_embed_dim)
+        elif class_embed_type == "projection":
+            if projection_class_embeddings_input_dim is None:
+                raise ValueError(
+                    "`class_embed_type`: 'projection' requires `projection_class_embeddings_input_dim` be set"
+                )
+            # The projection `class_embed_type` is the same as the timestep `class_embed_type` except
+            # 1. the `class_labels` inputs are not first converted to sinusoidal embeddings
+            # 2. it projects from an arbitrary input dimension.
+            #
+            # Note that `TimestepEmbedding` is quite general, being mainly linear layers and activations.
+            # When used for embedding actual timesteps, the timesteps are first converted to sinusoidal embeddings.
+            # As a result, `TimestepEmbedding` can be passed arbitrary vectors.
+            self.class_embedding = TimestepEmbedding(projection_class_embeddings_input_dim, time_embed_dim)
+        elif class_embed_type == "simple_projection":
+            if projection_class_embeddings_input_dim is None:
+                raise ValueError(
+                    "`class_embed_type`: 'simple_projection' requires `projection_class_embeddings_input_dim` be set"
+                )
+            self.class_embedding = nn.Linear(projection_class_embeddings_input_dim, time_embed_dim)
+        else:
+            self.class_embedding = None
+
+        if addition_embed_type == "text":
+            if encoder_hid_dim is not None:
+                text_time_embedding_from_dim = encoder_hid_dim
+            else:
+                text_time_embedding_from_dim = cross_attention_dim
+
+            self.add_embedding = TextTimeEmbedding(
+                text_time_embedding_from_dim, time_embed_dim, num_heads=addition_embed_type_num_heads
+            )
+        elif addition_embed_type == "text_image":
+            # text_embed_dim and image_embed_dim DON'T have to be `cross_attention_dim`. To not clutter the __init__ too much
+            # they are set to `cross_attention_dim` here as this is exactly the required dimension for the currently only use
+            # case when `addition_embed_type == "text_image"` (Kadinsky 2.1)`
+            self.add_embedding = TextImageTimeEmbedding(
+                text_embed_dim=cross_attention_dim, image_embed_dim=cross_attention_dim, time_embed_dim=time_embed_dim
+            )
+        elif addition_embed_type == "text_time":
+            self.add_time_proj = Timesteps(addition_time_embed_dim, flip_sin_to_cos, freq_shift)
+            self.add_embedding = TimestepEmbedding(projection_class_embeddings_input_dim, time_embed_dim)
+        elif addition_embed_type == "image":
+            # Kandinsky 2.2
+            self.add_embedding = ImageTimeEmbedding(image_embed_dim=encoder_hid_dim, time_embed_dim=time_embed_dim)
+        elif addition_embed_type == "image_hint":
+            # Kandinsky 2.2 ControlNet
+            self.add_embedding = ImageHintTimeEmbedding(image_embed_dim=encoder_hid_dim, time_embed_dim=time_embed_dim)
+        elif addition_embed_type is not None:
+            raise ValueError(f"addition_embed_type: {addition_embed_type} must be None, 'text' or 'text_image'.")
+
+        if time_embedding_act_fn is None:
+            self.time_embed_act = None
+        else:
+            self.time_embed_act = get_activation(time_embedding_act_fn)
+
+        self.down_blocks = nn.ModuleList([])
+        self.up_blocks = nn.ModuleList([])
+
+        if isinstance(only_cross_attention, bool):
+            if mid_block_only_cross_attention is None:
+                mid_block_only_cross_attention = only_cross_attention
+
+            only_cross_attention = [only_cross_attention] * len(down_block_types)
+
+        if mid_block_only_cross_attention is None:
+            mid_block_only_cross_attention = False
+
+        if isinstance(num_attention_heads, int):
+            num_attention_heads = (num_attention_heads,) * len(down_block_types)
+
+        if isinstance(attention_head_dim, int):
+            attention_head_dim = (attention_head_dim,) * len(down_block_types)
+
+        if isinstance(cross_attention_dim, int):
+            cross_attention_dim = (cross_attention_dim,) * len(down_block_types)
+
+        if isinstance(layers_per_block, int):
+            layers_per_block = [layers_per_block] * len(down_block_types)
+
+        if isinstance(transformer_layers_per_block, int):
+            transformer_layers_per_block = [transformer_layers_per_block] * len(down_block_types)
+
+        if class_embeddings_concat:
+            # The time embeddings are concatenated with the class embeddings. The dimension of the
+            # time embeddings passed to the down, middle, and up blocks is twice the dimension of the
+            # regular time embeddings
+            blocks_time_embed_dim = time_embed_dim * 2
+        else:
+            blocks_time_embed_dim = time_embed_dim
+
+        # down
+        output_channel = block_out_channels[0]
+        for i, down_block_type in enumerate(down_block_types):
+            input_channel = output_channel
+            output_channel = block_out_channels[i]
+            is_final_block = i == len(block_out_channels) - 1
+
+            down_block = get_down_block(
+                down_block_type,
+                num_layers=layers_per_block[i],
+                transformer_layers_per_block=transformer_layers_per_block[i],
+                in_channels=input_channel,
+                out_channels=output_channel,
+                temb_channels=blocks_time_embed_dim,
+                add_downsample=not is_final_block,
+                resnet_eps=norm_eps,
+                resnet_act_fn=act_fn,
+                resnet_groups=norm_num_groups,
+                cross_attention_dim=cross_attention_dim[i],
+                num_attention_heads=num_attention_heads[i],
+                downsample_padding=downsample_padding,
+                dual_cross_attention=dual_cross_attention,
+                use_linear_projection=use_linear_projection,
+                only_cross_attention=only_cross_attention[i],
+                upcast_attention=upcast_attention,
+                resnet_time_scale_shift=resnet_time_scale_shift,
+                attention_type=attention_type,
+                resnet_skip_time_act=resnet_skip_time_act,
+                resnet_out_scale_factor=resnet_out_scale_factor,
+                cross_attention_norm=cross_attention_norm,
+                attention_head_dim=attention_head_dim[i] if attention_head_dim[i] is not None else output_channel,
+                dropout=dropout,
+            )
+            self.down_blocks.append(down_block)
+
+        # mid
+        if mid_block_type == "UNetMidBlock2DCrossAttn":
+            self.mid_block = UNetMidBlock2DCrossAttn(
+                transformer_layers_per_block=transformer_layers_per_block[-1],
+                in_channels=block_out_channels[-1],
+                temb_channels=blocks_time_embed_dim,
+                dropout=dropout,
+                resnet_eps=norm_eps,
+                resnet_act_fn=act_fn,
+                output_scale_factor=mid_block_scale_factor,
+                resnet_time_scale_shift=resnet_time_scale_shift,
+                cross_attention_dim=cross_attention_dim[-1],
+                num_attention_heads=num_attention_heads[-1],
+                resnet_groups=norm_num_groups,
+                dual_cross_attention=dual_cross_attention,
+                use_linear_projection=use_linear_projection,
+                upcast_attention=upcast_attention,
+                attention_type=attention_type,
+            )
+        elif mid_block_type == "UNetMidBlock2DSimpleCrossAttn":
+            self.mid_block = UNetMidBlock2DSimpleCrossAttn(
+                in_channels=block_out_channels[-1],
+                temb_channels=blocks_time_embed_dim,
+                dropout=dropout,
+                resnet_eps=norm_eps,
+                resnet_act_fn=act_fn,
+                output_scale_factor=mid_block_scale_factor,
+                cross_attention_dim=cross_attention_dim[-1],
+                attention_head_dim=attention_head_dim[-1],
+                resnet_groups=norm_num_groups,
+                resnet_time_scale_shift=resnet_time_scale_shift,
+                skip_time_act=resnet_skip_time_act,
+                only_cross_attention=mid_block_only_cross_attention,
+                cross_attention_norm=cross_attention_norm,
+            )
+        elif mid_block_type == "UNetMidBlock2D":
+            self.mid_block = UNetMidBlock2D(
+                in_channels=block_out_channels[-1],
+                temb_channels=blocks_time_embed_dim,
+                dropout=dropout,
+                num_layers=0,
+                resnet_eps=norm_eps,
+                resnet_act_fn=act_fn,
+                output_scale_factor=mid_block_scale_factor,
+                resnet_groups=norm_num_groups,
+                resnet_time_scale_shift=resnet_time_scale_shift,
+                add_attention=False,
+            )
+        elif mid_block_type is None:
+            self.mid_block = None
+        else:
+            raise ValueError(f"unknown mid_block_type : {mid_block_type}")
+
+        # count how many layers upsample the images
+        self.num_upsamplers = 0
+
+        # up
+        reversed_block_out_channels = list(reversed(block_out_channels))
+        reversed_num_attention_heads = list(reversed(num_attention_heads))
+        reversed_layers_per_block = list(reversed(layers_per_block))
+        reversed_cross_attention_dim = list(reversed(cross_attention_dim))
+        reversed_transformer_layers_per_block = (
+            list(reversed(transformer_layers_per_block))
+            if reverse_transformer_layers_per_block is None
+            else reverse_transformer_layers_per_block
+        )
+        only_cross_attention = list(reversed(only_cross_attention))
+
+        output_channel = reversed_block_out_channels[0]
+        for i, up_block_type in enumerate(up_block_types):
+            is_final_block = i == len(block_out_channels) - 1
+
+            prev_output_channel = output_channel
+            output_channel = reversed_block_out_channels[i]
+            input_channel = reversed_block_out_channels[min(i + 1, len(block_out_channels) - 1)]
+
+            # add upsample block for all BUT final layer
+            if not is_final_block:
+                add_upsample = True
+                self.num_upsamplers += 1
+            else:
+                add_upsample = False
+
+            up_block = get_up_block(
+                up_block_type,
+                num_layers=reversed_layers_per_block[i] + 1,
+                transformer_layers_per_block=reversed_transformer_layers_per_block[i],
+                in_channels=input_channel,
+                out_channels=output_channel,
+                prev_output_channel=prev_output_channel,
+                temb_channels=blocks_time_embed_dim,
+                add_upsample=add_upsample,
+                resnet_eps=norm_eps,
+                resnet_act_fn=act_fn,
+                resolution_idx=i,
+                resnet_groups=norm_num_groups,
+                cross_attention_dim=reversed_cross_attention_dim[i],
+                num_attention_heads=reversed_num_attention_heads[i],
+                dual_cross_attention=dual_cross_attention,
+                use_linear_projection=use_linear_projection,
+                only_cross_attention=only_cross_attention[i],
+                upcast_attention=upcast_attention,
+                resnet_time_scale_shift=resnet_time_scale_shift,
+                attention_type=attention_type,
+                resnet_skip_time_act=resnet_skip_time_act,
+                resnet_out_scale_factor=resnet_out_scale_factor,
+                cross_attention_norm=cross_attention_norm,
+                attention_head_dim=attention_head_dim[i] if attention_head_dim[i] is not None else output_channel,
+                dropout=dropout,
+            )
+            self.up_blocks.append(up_block)
+            prev_output_channel = output_channel
+
+        # out
+        if norm_num_groups is not None:
+            self.conv_norm_out = nn.GroupNorm(
+                num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=norm_eps
+            )
+
+            self.conv_act = get_activation(act_fn)
+
+        else:
+            self.conv_norm_out = None
+            self.conv_act = None
+
+        conv_out_padding = (conv_out_kernel - 1) // 2
+        self.conv_out = nn.Conv2d(
+            block_out_channels[0], out_channels, kernel_size=conv_out_kernel, padding=conv_out_padding
+        )
+
+        if attention_type in ["gated", "gated-text-image"]:
+            positive_len = 768
+            if isinstance(cross_attention_dim, int):
+                positive_len = cross_attention_dim
+            elif isinstance(cross_attention_dim, tuple) or isinstance(cross_attention_dim, list):
+                positive_len = cross_attention_dim[0]
+
+            feature_type = "text-only" if attention_type == "gated" else "text-image"
+            self.position_net = PositionNet(
+                positive_len=positive_len, out_dim=cross_attention_dim, feature_type=feature_type
+            )
+
+    @property
+    def attn_processors(self) -> Dict[str, AttentionProcessor]:
+        r"""
+        Returns:
+            `dict` of attention processors: A dictionary containing all attention processors used in the model with
+            indexed by its weight name.
+        """
+        # set recursively
+        processors = {}
+
+        def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]):
+            if hasattr(module, "get_processor"):
+                processors[f"{name}.processor"] = module.get_processor(return_deprecated_lora=True)
+
+            for sub_name, child in module.named_children():
+                fn_recursive_add_processors(f"{name}.{sub_name}", child, processors)
+
+            return processors
+
+        for name, module in self.named_children():
+            fn_recursive_add_processors(name, module, processors)
+
+        return processors
+
+    # def set_attn_processor(
+    #     self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]], _remove_lora=False
+    # ):
+    def set_attn_processor(
+            self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]
+    ):
+        r"""
+        Sets the attention processor to use to compute attention.
+
+        Parameters:
+            processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`):
+                The instantiated processor class or a dictionary of processor classes that will be set as the processor
+                for **all** `Attention` layers.
+
+                If `processor` is a dict, the key needs to define the path to the corresponding cross attention
+                processor. This is strongly recommended when setting trainable attention processors.
+
+        """
+        count = len(self.attn_processors.keys())
+
+        if isinstance(processor, dict) and len(processor) != count:
+            raise ValueError(
+                f"A dict of processors was passed, but the number of processors {len(processor)} does not match the"
+                f" number of attention layers: {count}. Please make sure to pass {count} processor classes."
+            )
+
+        def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor):
+            if hasattr(module, "set_processor"):
+                # if not isinstance(processor, dict):
+                #     module.set_processor(processor, _remove_lora=_remove_lora)
+                # else:
+                #     module.set_processor(processor.pop(f"{name}.processor"), _remove_lora=_remove_lora)
+                if not isinstance(processor, dict):
+                    module.set_processor(processor)
+                else:
+                    module.set_processor(processor.pop(f"{name}.processor"))
+
+            for sub_name, child in module.named_children():
+                fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor)
+
+        for name, module in self.named_children():
+            fn_recursive_attn_processor(name, module, processor)
+
+    def set_default_attn_processor(self):
+        """
+        Disables custom attention processors and sets the default attention implementation.
+        """
+        if all(proc.__class__ in ADDED_KV_ATTENTION_PROCESSORS for proc in self.attn_processors.values()):
+            processor = AttnAddedKVProcessor()
+        elif all(proc.__class__ in CROSS_ATTENTION_PROCESSORS for proc in self.attn_processors.values()):
+            processor = AttnProcessor()
+        else:
+            raise ValueError(
+                f"Cannot call `set_default_attn_processor` when attention processors are of type {next(iter(self.attn_processors.values()))}"
+            )
+
+        #self.set_attn_processor(processor, _remove_lora=True)
+        self.set_attn_processor(processor)
+
+    def set_attention_slice(self, slice_size):
+        r"""
+        Enable sliced attention computation.
+
+        When this option is enabled, the attention module splits the input tensor in slices to compute attention in
+        several steps. This is useful for saving some memory in exchange for a small decrease in speed.
+
+        Args:
+            slice_size (`str` or `int` or `list(int)`, *optional*, defaults to `"auto"`):
+                When `"auto"`, input to the attention heads is halved, so attention is computed in two steps. If
+                `"max"`, maximum amount of memory is saved by running only one slice at a time. If a number is
+                provided, uses as many slices as `attention_head_dim // slice_size`. In this case, `attention_head_dim`
+                must be a multiple of `slice_size`.
+        """
+        sliceable_head_dims = []
+
+        def fn_recursive_retrieve_sliceable_dims(module: torch.nn.Module):
+            if hasattr(module, "set_attention_slice"):
+                sliceable_head_dims.append(module.sliceable_head_dim)
+
+            for child in module.children():
+                fn_recursive_retrieve_sliceable_dims(child)
+
+        # retrieve number of attention layers
+        for module in self.children():
+            fn_recursive_retrieve_sliceable_dims(module)
+
+        num_sliceable_layers = len(sliceable_head_dims)
+
+        if slice_size == "auto":
+            # half the attention head size is usually a good trade-off between
+            # speed and memory
+            slice_size = [dim // 2 for dim in sliceable_head_dims]
+        elif slice_size == "max":
+            # make smallest slice possible
+            slice_size = num_sliceable_layers * [1]
+
+        slice_size = num_sliceable_layers * [slice_size] if not isinstance(slice_size, list) else slice_size
+
+        if len(slice_size) != len(sliceable_head_dims):
+            raise ValueError(
+                f"You have provided {len(slice_size)}, but {self.config} has {len(sliceable_head_dims)} different"
+                f" attention layers. Make sure to match `len(slice_size)` to be {len(sliceable_head_dims)}."
+            )
+
+        for i in range(len(slice_size)):
+            size = slice_size[i]
+            dim = sliceable_head_dims[i]
+            if size is not None and size > dim:
+                raise ValueError(f"size {size} has to be smaller or equal to {dim}.")
+
+        # Recursively walk through all the children.
+        # Any children which exposes the set_attention_slice method
+        # gets the message
+        def fn_recursive_set_attention_slice(module: torch.nn.Module, slice_size: List[int]):
+            if hasattr(module, "set_attention_slice"):
+                module.set_attention_slice(slice_size.pop())
+
+            for child in module.children():
+                fn_recursive_set_attention_slice(child, slice_size)
+
+        reversed_slice_size = list(reversed(slice_size))
+        for module in self.children():
+            fn_recursive_set_attention_slice(module, reversed_slice_size)
+
+    def _set_gradient_checkpointing(self, module, value=False):
+        if hasattr(module, "gradient_checkpointing"):
+            module.gradient_checkpointing = value
+
+    def enable_freeu(self, s1, s2, b1, b2):
+        r"""Enables the FreeU mechanism from https://arxiv.org/abs/2309.11497.
+
+        The suffixes after the scaling factors represent the stage blocks where they are being applied.
+
+        Please refer to the [official repository](https://github.com/ChenyangSi/FreeU) for combinations of values that
+        are known to work well for different pipelines such as Stable Diffusion v1, v2, and Stable Diffusion XL.
+
+        Args:
+            s1 (`float`):
+                Scaling factor for stage 1 to attenuate the contributions of the skip features. This is done to
+                mitigate the "oversmoothing effect" in the enhanced denoising process.
+            s2 (`float`):
+                Scaling factor for stage 2 to attenuate the contributions of the skip features. This is done to
+                mitigate the "oversmoothing effect" in the enhanced denoising process.
+            b1 (`float`): Scaling factor for stage 1 to amplify the contributions of backbone features.
+            b2 (`float`): Scaling factor for stage 2 to amplify the contributions of backbone features.
+        """
+        for i, upsample_block in enumerate(self.up_blocks):
+            setattr(upsample_block, "s1", s1)
+            setattr(upsample_block, "s2", s2)
+            setattr(upsample_block, "b1", b1)
+            setattr(upsample_block, "b2", b2)
+
+    def disable_freeu(self):
+        """Disables the FreeU mechanism."""
+        freeu_keys = {"s1", "s2", "b1", "b2"}
+        for i, upsample_block in enumerate(self.up_blocks):
+            for k in freeu_keys:
+                if hasattr(upsample_block, k) or getattr(upsample_block, k, None) is not None:
+                    setattr(upsample_block, k, None)
+
+    def forward(
+        self,
+        sample: torch.FloatTensor,
+        timestep: Union[torch.Tensor, float, int],
+        encoder_hidden_states: torch.Tensor,
+        class_labels: Optional[torch.Tensor] = None,
+        timestep_cond: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        cross_attention_kwargs: Optional[Dict[str, Any]] = None,
+        added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None,
+        down_block_additional_residuals: Optional[Tuple[torch.Tensor]] = None,
+        mid_block_additional_residual: Optional[torch.Tensor] = None,
+        down_intrablock_additional_residuals: Optional[Tuple[torch.Tensor]] = None,
+        encoder_attention_mask: Optional[torch.Tensor] = None,
+        return_dict: bool = True,
+    ) -> Union[UNet2DConditionOutput, Tuple]:
+        r"""
+        The [`UNet2DConditionModel`] forward method.
+
+        Args:
+            sample (`torch.FloatTensor`):
+                The noisy input tensor with the following shape `(batch, channel, height, width)`.
+            timestep (`torch.FloatTensor` or `float` or `int`): The number of timesteps to denoise an input.
+            encoder_hidden_states (`torch.FloatTensor`):
+                The encoder hidden states with shape `(batch, sequence_length, feature_dim)`.
+            class_labels (`torch.Tensor`, *optional*, defaults to `None`):
+                Optional class labels for conditioning. Their embeddings will be summed with the timestep embeddings.
+            timestep_cond: (`torch.Tensor`, *optional*, defaults to `None`):
+                Conditional embeddings for timestep. If provided, the embeddings will be summed with the samples passed
+                through the `self.time_embedding` layer to obtain the timestep embeddings.
+            attention_mask (`torch.Tensor`, *optional*, defaults to `None`):
+                An attention mask of shape `(batch, key_tokens)` is applied to `encoder_hidden_states`. If `1` the mask
+                is kept, otherwise if `0` it is discarded. Mask will be converted into a bias, which adds large
+                negative values to the attention scores corresponding to "discard" tokens.
+            cross_attention_kwargs (`dict`, *optional*):
+                A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
+                `self.processor` in
+                [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
+            added_cond_kwargs: (`dict`, *optional*):
+                A kwargs dictionary containing additional embeddings that if specified are added to the embeddings that
+                are passed along to the UNet blocks.
+            down_block_additional_residuals: (`tuple` of `torch.Tensor`, *optional*):
+                A tuple of tensors that if specified are added to the residuals of down unet blocks.
+            mid_block_additional_residual: (`torch.Tensor`, *optional*):
+                A tensor that if specified is added to the residual of the middle unet block.
+            encoder_attention_mask (`torch.Tensor`):
+                A cross-attention mask of shape `(batch, sequence_length)` is applied to `encoder_hidden_states`. If
+                `True` the mask is kept, otherwise if `False` it is discarded. Mask will be converted into a bias,
+                which adds large negative values to the attention scores corresponding to "discard" tokens.
+            return_dict (`bool`, *optional*, defaults to `True`):
+                Whether or not to return a [`~models.unet_2d_condition.UNet2DConditionOutput`] instead of a plain
+                tuple.
+            cross_attention_kwargs (`dict`, *optional*):
+                A kwargs dictionary that if specified is passed along to the [`AttnProcessor`].
+            added_cond_kwargs: (`dict`, *optional*):
+                A kwargs dictionary containin additional embeddings that if specified are added to the embeddings that
+                are passed along to the UNet blocks.
+            down_block_additional_residuals (`tuple` of `torch.Tensor`, *optional*):
+                additional residuals to be added to UNet long skip connections from down blocks to up blocks for
+                example from ControlNet side model(s)
+            mid_block_additional_residual (`torch.Tensor`, *optional*):
+                additional residual to be added to UNet mid block output, for example from ControlNet side model
+            down_intrablock_additional_residuals (`tuple` of `torch.Tensor`, *optional*):
+                additional residuals to be added within UNet down blocks, for example from T2I-Adapter side model(s)
+
+        Returns:
+            [`~models.unet_2d_condition.UNet2DConditionOutput`] or `tuple`:
+                If `return_dict` is True, an [`~models.unet_2d_condition.UNet2DConditionOutput`] is returned, otherwise
+                a `tuple` is returned where the first element is the sample tensor.
+        """
+        # By default samples have to be AT least a multiple of the overall upsampling factor.
+        # The overall upsampling factor is equal to 2 ** (# num of upsampling layers).
+        # However, the upsampling interpolation output size can be forced to fit any upsampling size
+        # on the fly if necessary.
+        default_overall_up_factor = 2**self.num_upsamplers
+
+        # upsample size should be forwarded when sample is not a multiple of `default_overall_up_factor`
+        forward_upsample_size = False
+        upsample_size = None
+
+        for dim in sample.shape[-2:]:
+            if dim % default_overall_up_factor != 0:
+                # Forward upsample size to force interpolation output size.
+                forward_upsample_size = True
+                break
+
+        # ensure attention_mask is a bias, and give it a singleton query_tokens dimension
+        # expects mask of shape:
+        #   [batch, key_tokens]
+        # adds singleton query_tokens dimension:
+        #   [batch,                    1, key_tokens]
+        # this helps to broadcast it as a bias over attention scores, which will be in one of the following shapes:
+        #   [batch,  heads, query_tokens, key_tokens] (e.g. torch sdp attn)
+        #   [batch * heads, query_tokens, key_tokens] (e.g. xformers or classic attn)
+        if attention_mask is not None:
+            # assume that mask is expressed as:
+            #   (1 = keep,      0 = discard)
+            # convert mask into a bias that can be added to attention scores:
+            #       (keep = +0,     discard = -10000.0)
+            attention_mask = (1 - attention_mask.to(sample.dtype)) * -10000.0
+            attention_mask = attention_mask.unsqueeze(1)
+
+        # convert encoder_attention_mask to a bias the same way we do for attention_mask
+        if encoder_attention_mask is not None:
+            encoder_attention_mask = (1 - encoder_attention_mask.to(sample.dtype)) * -10000.0
+            encoder_attention_mask = encoder_attention_mask.unsqueeze(1)
+
+        # 0. center input if necessary
+        if self.config.center_input_sample:
+            sample = 2 * sample - 1.0
+
+        # 1. time
+        timesteps = timestep
+        if not torch.is_tensor(timesteps):
+            # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can
+            # This would be a good case for the `match` statement (Python 3.10+)
+            is_mps = sample.device.type == "mps"
+            if isinstance(timestep, float):
+                dtype = torch.float32 if is_mps else torch.float64
+            else:
+                dtype = torch.int32 if is_mps else torch.int64
+            timesteps = torch.tensor([timesteps], dtype=dtype, device=sample.device)
+        elif len(timesteps.shape) == 0:
+            timesteps = timesteps[None].to(sample.device)
+
+        # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
+        timesteps = timesteps.expand(sample.shape[0])
+
+        t_emb = self.time_proj(timesteps)
+
+        # `Timesteps` does not contain any weights and will always return f32 tensors
+        # but time_embedding might actually be running in fp16. so we need to cast here.
+        # there might be better ways to encapsulate this.
+        t_emb = t_emb.to(dtype=sample.dtype)
+
+        emb = self.time_embedding(t_emb, timestep_cond)
+        aug_emb = None
+
+        if self.class_embedding is not None:
+            if class_labels is None:
+                raise ValueError("class_labels should be provided when num_class_embeds > 0")
+
+            if self.config.class_embed_type == "timestep":
+                class_labels = self.time_proj(class_labels)
+
+                # `Timesteps` does not contain any weights and will always return f32 tensors
+                # there might be better ways to encapsulate this.
+                class_labels = class_labels.to(dtype=sample.dtype)
+
+            class_emb = self.class_embedding(class_labels).to(dtype=sample.dtype)
+
+            if self.config.class_embeddings_concat:
+                emb = torch.cat([emb, class_emb], dim=-1)
+            else:
+                emb = emb + class_emb
+
+        if self.config.addition_embed_type == "text":
+            aug_emb = self.add_embedding(encoder_hidden_states)
+        elif self.config.addition_embed_type == "text_image":
+            # Kandinsky 2.1 - style
+            if "image_embeds" not in added_cond_kwargs:
+                raise ValueError(
+                    f"{self.__class__} has the config param `addition_embed_type` set to 'text_image' which requires the keyword argument `image_embeds` to be passed in `added_cond_kwargs`"
+                )
+
+            image_embs = added_cond_kwargs.get("image_embeds")
+            text_embs = added_cond_kwargs.get("text_embeds", encoder_hidden_states)
+            aug_emb = self.add_embedding(text_embs, image_embs)
+        elif self.config.addition_embed_type == "text_time":
+            # SDXL - style
+            if "text_embeds" not in added_cond_kwargs:
+                raise ValueError(
+                    f"{self.__class__} has the config param `addition_embed_type` set to 'text_time' which requires the keyword argument `text_embeds` to be passed in `added_cond_kwargs`"
+                )
+            text_embeds = added_cond_kwargs.get("text_embeds")
+            if "time_ids" not in added_cond_kwargs:
+                raise ValueError(
+                    f"{self.__class__} has the config param `addition_embed_type` set to 'text_time' which requires the keyword argument `time_ids` to be passed in `added_cond_kwargs`"
+                )
+            time_ids = added_cond_kwargs.get("time_ids")
+            time_embeds = self.add_time_proj(time_ids.flatten())
+            time_embeds = time_embeds.reshape((text_embeds.shape[0], -1))
+            add_embeds = torch.concat([text_embeds, time_embeds], dim=-1)
+            add_embeds = add_embeds.to(emb.dtype)
+            aug_emb = self.add_embedding(add_embeds)
+        elif self.config.addition_embed_type == "image":
+            # Kandinsky 2.2 - style
+            if "image_embeds" not in added_cond_kwargs:
+                raise ValueError(
+                    f"{self.__class__} has the config param `addition_embed_type` set to 'image' which requires the keyword argument `image_embeds` to be passed in `added_cond_kwargs`"
+                )
+            image_embs = added_cond_kwargs.get("image_embeds")
+            aug_emb = self.add_embedding(image_embs)
+        elif self.config.addition_embed_type == "image_hint":
+            # Kandinsky 2.2 - style
+            if "image_embeds" not in added_cond_kwargs or "hint" not in added_cond_kwargs:
+                raise ValueError(
+                    f"{self.__class__} has the config param `addition_embed_type` set to 'image_hint' which requires the keyword arguments `image_embeds` and `hint` to be passed in `added_cond_kwargs`"
+                )
+            image_embs = added_cond_kwargs.get("image_embeds")
+            hint = added_cond_kwargs.get("hint")
+            aug_emb, hint = self.add_embedding(image_embs, hint)
+            sample = torch.cat([sample, hint], dim=1)
+
+        emb = emb + aug_emb if aug_emb is not None else emb
+
+        if self.time_embed_act is not None:
+            emb = self.time_embed_act(emb)
+
+        if self.encoder_hid_proj is not None and self.config.encoder_hid_dim_type == "text_proj":
+            encoder_hidden_states = self.encoder_hid_proj(encoder_hidden_states)
+        elif self.encoder_hid_proj is not None and self.config.encoder_hid_dim_type == "text_image_proj":
+            # Kadinsky 2.1 - style
+            if "image_embeds" not in added_cond_kwargs:
+                raise ValueError(
+                    f"{self.__class__} has the config param `encoder_hid_dim_type` set to 'text_image_proj' which requires the keyword argument `image_embeds` to be passed in  `added_conditions`"
+                )
+
+            image_embeds = added_cond_kwargs.get("image_embeds")
+            encoder_hidden_states = self.encoder_hid_proj(encoder_hidden_states, image_embeds)
+        elif self.encoder_hid_proj is not None and self.config.encoder_hid_dim_type == "image_proj":
+            # Kandinsky 2.2 - style
+            if "image_embeds" not in added_cond_kwargs:
+                raise ValueError(
+                    f"{self.__class__} has the config param `encoder_hid_dim_type` set to 'image_proj' which requires the keyword argument `image_embeds` to be passed in  `added_conditions`"
+                )
+            image_embeds = added_cond_kwargs.get("image_embeds")
+            encoder_hidden_states = self.encoder_hid_proj(image_embeds)
+        # 2. pre-process
+        sample = self.conv_in(sample)
+
+        # 2.5 GLIGEN position net
+        if cross_attention_kwargs is not None and cross_attention_kwargs.get("gligen", None) is not None:
+            cross_attention_kwargs = cross_attention_kwargs.copy()
+            gligen_args = cross_attention_kwargs.pop("gligen")
+            cross_attention_kwargs["gligen"] = {"objs": self.position_net(**gligen_args)}
+
+        # 3. down
+        lora_scale = cross_attention_kwargs.get("scale", 1.0) if cross_attention_kwargs is not None else 1.0
+        if USE_PEFT_BACKEND:
+            # weight the lora layers by setting `lora_scale` for each PEFT layer
+            scale_lora_layers(self, lora_scale)
+
+        is_controlnet = mid_block_additional_residual is not None and down_block_additional_residuals is not None
+        # using new arg down_intrablock_additional_residuals for T2I-Adapters, to distinguish from controlnets
+        is_adapter = down_intrablock_additional_residuals is not None
+        # maintain backward compatibility for legacy usage, where
+        #       T2I-Adapter and ControlNet both use down_block_additional_residuals arg
+        #       but can only use one or the other
+        if not is_adapter and mid_block_additional_residual is None and down_block_additional_residuals is not None:
+            deprecate(
+                "T2I should not use down_block_additional_residuals",
+                "1.3.0",
+                "Passing intrablock residual connections with `down_block_additional_residuals` is deprecated \
+                       and will be removed in diffusers 1.3.0.  `down_block_additional_residuals` should only be used \
+                       for ControlNet. Please make sure use `down_intrablock_additional_residuals` instead. ",
+                standard_warn=False,
+            )
+            down_intrablock_additional_residuals = down_block_additional_residuals
+            is_adapter = True
+
+        down_block_res_samples = (sample,)
+        for downsample_block in self.down_blocks:
+            if hasattr(downsample_block, "has_cross_attention") and downsample_block.has_cross_attention:
+                # For t2i-adapter CrossAttnDownBlock2D
+                additional_residuals = {}
+                if is_adapter and len(down_intrablock_additional_residuals) > 0:
+                    additional_residuals["additional_residuals"] = down_intrablock_additional_residuals.pop(0)
+
+                sample, res_samples = downsample_block(
+                    hidden_states=sample,
+                    temb=emb,
+                    encoder_hidden_states=encoder_hidden_states,
+                    attention_mask=attention_mask,
+                    cross_attention_kwargs=cross_attention_kwargs,
+                    encoder_attention_mask=encoder_attention_mask,
+                    **additional_residuals,
+                )
+            else:
+                sample, res_samples = downsample_block(hidden_states=sample, temb=emb, scale=lora_scale)
+                if is_adapter and len(down_intrablock_additional_residuals) > 0:
+                    sample += down_intrablock_additional_residuals.pop(0)
+
+            down_block_res_samples += res_samples
+
+        if is_controlnet:
+            new_down_block_res_samples = ()
+
+            for down_block_res_sample, down_block_additional_residual in zip(
+                down_block_res_samples, down_block_additional_residuals
+            ):
+                down_block_res_sample = down_block_res_sample + down_block_additional_residual
+                new_down_block_res_samples = new_down_block_res_samples + (down_block_res_sample,)
+
+            down_block_res_samples = new_down_block_res_samples
+
+        # 4. mid
+        if self.mid_block is not None:
+            if hasattr(self.mid_block, "has_cross_attention") and self.mid_block.has_cross_attention:
+                sample = self.mid_block(
+                    sample,
+                    emb,
+                    encoder_hidden_states=encoder_hidden_states,
+                    attention_mask=attention_mask,
+                    cross_attention_kwargs=cross_attention_kwargs,
+                    encoder_attention_mask=encoder_attention_mask,
+                )
+            else:
+                sample = self.mid_block(sample, emb)
+
+            # To support T2I-Adapter-XL
+            if (
+                is_adapter
+                and len(down_intrablock_additional_residuals) > 0
+                and sample.shape == down_intrablock_additional_residuals[0].shape
+            ):
+                sample += down_intrablock_additional_residuals.pop(0)
+
+        if is_controlnet:
+            sample = sample + mid_block_additional_residual
+
+        # 5. up
+        for i, upsample_block in enumerate(self.up_blocks):
+            is_final_block = i == len(self.up_blocks) - 1
+
+            res_samples = down_block_res_samples[-len(upsample_block.resnets) :]
+            down_block_res_samples = down_block_res_samples[: -len(upsample_block.resnets)]
+
+            # if we have not reached the final block and need to forward the
+            # upsample size, we do it here
+            if not is_final_block and forward_upsample_size:
+                upsample_size = down_block_res_samples[-1].shape[2:]
+
+            if hasattr(upsample_block, "has_cross_attention") and upsample_block.has_cross_attention:
+                sample = upsample_block(
+                    hidden_states=sample,
+                    temb=emb,
+                    res_hidden_states_tuple=res_samples,
+                    encoder_hidden_states=encoder_hidden_states,
+                    cross_attention_kwargs=cross_attention_kwargs,
+                    upsample_size=upsample_size,
+                    attention_mask=attention_mask,
+                    encoder_attention_mask=encoder_attention_mask,
+                )
+            else:
+                sample = upsample_block(
+                    hidden_states=sample,
+                    temb=emb,
+                    res_hidden_states_tuple=res_samples,
+                    upsample_size=upsample_size,
+                    scale=lora_scale,
+                )
+           
+            self.latent_store(sample.detach(), t=timestep, layer_index=i)
+
+        # 6. post-process
+        if self.conv_norm_out:
+            sample = self.conv_norm_out(sample)
+            sample = self.conv_act(sample)
+        sample = self.conv_out(sample)
+
+        if USE_PEFT_BACKEND:
+            # remove `lora_scale` from each PEFT layer
+            unscale_lora_layers(self, lora_scale)
+
+        if not return_dict:
+            return (sample,)
+
+        return UNet2DConditionOutput(sample=sample)
diff --git a/consistory/consistory_utils.py b/consistory/consistory_utils.py
new file mode 100644
index 0000000..c9526ec
--- /dev/null
+++ b/consistory/consistory_utils.py
@@ -0,0 +1,200 @@
+# Copyright (C) 2024 NVIDIA Corporation.  All rights reserved.
+#
+# This work is licensed under the LICENSE file
+# located at the root directory.
+
+import numpy as np
+import torch
+from collections import defaultdict
+from diffusers.utils.import_utils import is_xformers_available
+from typing import Optional, List
+
+from .utils.general_utils import get_dynamic_threshold
+
+if is_xformers_available():
+    import xformers
+    import xformers.ops
+else:
+    xformers = None
+
+class FeatureInjector:
+    def __init__(self, nn_map, nn_distances, attn_masks, inject_range_alpha=[(10,20,0.8)], swap_strategy='min', dist_thr='dynamic', inject_unet_parts=['up']):
+        self.nn_map = nn_map
+        self.nn_distances = nn_distances
+        self.attn_masks = attn_masks
+
+        self.inject_range_alpha = inject_range_alpha if isinstance(inject_range_alpha, list) else [inject_range_alpha]
+        self.swap_strategy = swap_strategy # 'min / 'mean' / 'first'
+        self.dist_thr = dist_thr
+        self.inject_unet_parts = inject_unet_parts
+        self.inject_res = [64]
+
+    def inject_outputs(self, output, curr_iter, output_res, extended_mapping, place_in_unet, anchors_cache=None):
+        curr_unet_part = place_in_unet.split('_')[0]
+
+        # Inject only in the specified unet parts (up, mid, down)
+        if (curr_unet_part not in self.inject_unet_parts) or output_res not in self.inject_res:
+            return output
+
+        bsz = output.shape[0]
+        nn_map = self.nn_map[output_res]
+        nn_distances = self.nn_distances[output_res]
+        attn_masks = self.attn_masks[output_res]
+        vector_dim = output_res**2
+
+        alpha = next((alpha for min_range, max_range, alpha in self.inject_range_alpha if min_range <= curr_iter <= max_range), None)
+        if alpha:
+            old_output = output#.clone()
+            for i in range(bsz):
+                other_outputs = []
+
+                if self.swap_strategy == 'min':
+                    curr_mapping = extended_mapping[i]
+
+                    # If the current image is not mapped to any other image, skip
+                    if not torch.any(torch.cat([curr_mapping[:i], curr_mapping[i+1:]])):
+                        continue
+
+                    min_dists = nn_distances[i][curr_mapping].argmin(dim=0)
+                    curr_nn_map = nn_map[i][curr_mapping][min_dists, torch.arange(vector_dim)]
+
+                    curr_nn_distances = nn_distances[i][curr_mapping][min_dists, torch.arange(vector_dim)]
+                    dist_thr = get_dynamic_threshold(curr_nn_distances) if self.dist_thr == 'dynamic' else self.dist_thr
+                    dist_mask = curr_nn_distances < dist_thr
+                    final_mask_tgt = attn_masks[i] & dist_mask
+
+                    other_outputs = old_output[curr_mapping][min_dists, curr_nn_map][final_mask_tgt]
+
+                    output[i][final_mask_tgt] = alpha * other_outputs + (1 - alpha)*old_output[i][final_mask_tgt]
+
+            if anchors_cache and anchors_cache.is_cache_mode():
+                if place_in_unet not in anchors_cache.h_out_cache:
+                    anchors_cache.h_out_cache[place_in_unet] = {}
+
+                anchors_cache.h_out_cache[place_in_unet][curr_iter] = output
+
+        return output
+
+    def inject_anchors(self, output, curr_iter, output_res, extended_mapping, place_in_unet, anchors_cache):
+        curr_unet_part = place_in_unet.split('_')[0]
+
+        # Inject only in the specified unet parts (up, mid, down)
+        if (curr_unet_part not in self.inject_unet_parts) or output_res not in self.inject_res:
+            return output
+
+        bsz = output.shape[0]
+        nn_map = self.nn_map[output_res]
+        nn_distances = self.nn_distances[output_res]
+        attn_masks = self.attn_masks[output_res]
+        vector_dim = output_res**2
+
+        alpha = next((alpha for min_range, max_range, alpha in self.inject_range_alpha if min_range <= curr_iter <= max_range), None)
+        if alpha:
+
+            anchor_outputs = anchors_cache.h_out_cache[place_in_unet][curr_iter]
+
+            old_output = output#.clone()
+            for i in range(bsz):
+                other_outputs = []
+
+                if self.swap_strategy == 'min':
+                    min_dists = nn_distances[i].argmin(dim=0)
+                    curr_nn_map = nn_map[i][min_dists, torch.arange(vector_dim)]
+
+                    curr_nn_distances = nn_distances[i][min_dists, torch.arange(vector_dim)]
+                    dist_thr = get_dynamic_threshold(curr_nn_distances) if self.dist_thr == 'dynamic' else self.dist_thr
+                    dist_mask = curr_nn_distances < dist_thr
+                    final_mask_tgt = attn_masks[i] & dist_mask
+
+                    other_outputs = anchor_outputs[min_dists, curr_nn_map][final_mask_tgt]
+
+                    output[i][final_mask_tgt] = alpha * other_outputs + (1 - alpha)*old_output[i][final_mask_tgt]
+
+        return output
+
+
+class AnchorCache:
+    def __init__(self):
+        self.input_h_cache = {} # place_in_unet, iter, h_in
+        self.h_out_cache = {} # place_in_unet, iter, h_out
+        self.anchors_last_mask = None
+        self.dift_cache = None
+
+        self.mode = 'cache' # mode can be 'cache' or 'inject'
+
+    def set_mode(self, mode):
+        self.mode = mode
+
+    def set_mode_inject(self):
+        self.mode = 'inject'
+
+    def set_mode_cache(self):
+        self.mode = 'cache'
+
+    def is_inject_mode(self):
+        return self.mode == 'inject'
+
+    def is_cache_mode(self):
+        return self.mode == 'cache'
+
+
+    def to_device(self, device):
+        for key, value in self.input_h_cache.items():
+            self.input_h_cache[key] = {k: v.to(device) for k, v in value.items()}
+
+        for key, value in self.h_out_cache.items():
+            self.h_out_cache[key] = {k: v.to(device) for k, v in value.items()}
+
+        if self.anchors_last_mask:
+            self.anchors_last_mask = {k: v.to(device) for k, v in self.anchors_last_mask.items()}
+
+        if self.dift_cache is not None:
+            self.dift_cache = self.dift_cache.to(device)
+
+
+class QueryStore:
+    def __init__(self, mode='store', t_range=[0, 1000], strength_start=1, strength_end=1):
+        """
+        Initialize an empty ActivationsStore
+        """
+        self.query_store = defaultdict(list)
+        self.mode = mode
+        self.t_range = t_range
+        self.strengthes = np.linspace(strength_start, strength_end, (t_range[1] - t_range[0])+1)
+
+    def set_mode(self, mode): # mode can be 'cache' or 'inject'
+        self.mode = mode
+
+    def cache_query(self, query, place_in_unet: str):
+        self.query_store[place_in_unet] = query
+
+    def inject_query(self, query, place_in_unet, t):
+        if t >= self.t_range[0] and t <= self.t_range[1]:
+            relative_t = t - self.t_range[0]
+            strength = self.strengthes[relative_t]
+            new_query = strength * self.query_store[place_in_unet] + (1 - strength) * query
+        else:
+            new_query = query
+
+        return new_query
+
+class DIFTLatentStore:
+    def __init__(self, steps: List[int], up_ft_indices: List[int]):
+        self.steps = steps
+        self.up_ft_indices = up_ft_indices
+        self.dift_features = {}
+
+    def __call__(self, features: torch.Tensor, t: int, layer_index: int):
+        if t in self.steps and layer_index in self.up_ft_indices:
+            self.dift_features[f'{int(t)}_{layer_index}'] = features
+
+    def copy(self):
+        copy_dift = DIFTLatentStore(self.steps, self.up_ft_indices)
+
+        for key, value in self.dift_features.items():
+            copy_dift.dift_features[key] = value.clone()
+
+        return copy_dift
+
+    def reset(self):
+        self.dift_features = {}
diff --git a/consistory/environment.yml b/consistory/environment.yml
new file mode 100644
index 0000000..1a45fb2
--- /dev/null
+++ b/consistory/environment.yml
@@ -0,0 +1,251 @@
+name: consistory
+channels:
+  - pytorch
+  - nvidia
+  - conda-forge
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _openmp_mutex=5.1=1_gnu
+  - asttokens=2.4.1=pyhd8ed1ab_0
+  - backports=1.0=pyhd8ed1ab_3
+  - backports.functools_lru_cache=1.6.5=pyhd8ed1ab_0
+  - blas=1.0=mkl
+  - brotli-python=1.0.9=py310h6a678d5_7
+  - bzip2=1.0.8=h7b6447c_0
+  - ca-certificates=2023.7.22=hbcca054_0
+  - certifi=2023.7.22=pyhd8ed1ab_0
+  - cffi=1.15.1=py310h5eee18b_3
+  - charset-normalizer=2.0.4=pyhd3eb1b0_0
+  - comm=0.1.4=pyhd8ed1ab_0
+  - cryptography=41.0.3=py310hdda0065_0
+  - cuda-cudart=12.1.105=0
+  - cuda-cupti=12.1.105=0
+  - cuda-libraries=12.1.0=0
+  - cuda-nvrtc=12.1.105=0
+  - cuda-nvtx=12.1.105=0
+  - cuda-opencl=12.3.52=0
+  - cuda-runtime=12.1.0=0
+  - debugpy=1.6.7=py310h6a678d5_0
+  - decorator=5.1.1=pyhd8ed1ab_0
+  - entrypoints=0.4=pyhd8ed1ab_0
+  - exceptiongroup=1.1.3=pyhd8ed1ab_0
+  - executing=2.0.1=pyhd8ed1ab_0
+  - ffmpeg=4.3=hf484d3e_0
+  - filelock=3.9.0=py310h06a4308_0
+  - freetype=2.12.1=h4a9f257_0
+  - giflib=5.2.1=h5eee18b_3
+  - gmp=6.2.1=h295c915_3
+  - gmpy2=2.1.2=py310heeb90bb_0
+  - gnutls=3.6.15=he1e5248_0
+  - idna=3.4=py310h06a4308_0
+  - intel-openmp=2023.1.0=hdb19cb5_46306
+  - ipykernel=6.26.0=pyhf8b6a83_0
+  - ipython=8.17.2=pyh41d4057_0
+  - jedi=0.19.1=pyhd8ed1ab_0
+  - jinja2=3.1.2=py310h06a4308_0
+  - jpeg=9e=h5eee18b_1
+  - jupyter_core=5.5.0=py310hff52083_0
+  - lame=3.100=h7b6447c_0
+  - lcms2=2.12=h3be6417_0
+  - ld_impl_linux-64=2.38=h1181459_1
+  - lerc=3.0=h295c915_0
+  - libcublas=12.1.0.26=0
+  - libcufft=11.0.2.4=0
+  - libcufile=1.8.0.34=0
+  - libcurand=10.3.4.52=0
+  - libcusolver=11.4.4.55=0
+  - libcusparse=12.0.2.55=0
+  - libdeflate=1.17=h5eee18b_1
+  - libffi=3.4.4=h6a678d5_0
+  - libgcc-ng=11.2.0=h1234567_1
+  - libgomp=11.2.0=h1234567_1
+  - libiconv=1.16=h7f8727e_2
+  - libidn2=2.3.4=h5eee18b_0
+  - libjpeg-turbo=2.0.0=h9bf148f_0
+  - libnpp=12.0.2.50=0
+  - libnvjitlink=12.1.105=0
+  - libnvjpeg=12.1.1.14=0
+  - libpng=1.6.39=h5eee18b_0
+  - libsodium=1.0.18=h36c2ea0_1
+  - libstdcxx-ng=11.2.0=h1234567_1
+  - libtasn1=4.19.0=h5eee18b_0
+  - libtiff=4.5.1=h6a678d5_0
+  - libunistring=0.9.10=h27cfd23_0
+  - libuuid=1.41.5=h5eee18b_0
+  - libwebp=1.3.2=h11a3e52_0
+  - libwebp-base=1.3.2=h5eee18b_0
+  - llvm-openmp=14.0.6=h9e868ea_0
+  - lz4-c=1.9.4=h6a678d5_0
+  - markupsafe=2.1.1=py310h7f8727e_0
+  - matplotlib-inline=0.1.6=pyhd8ed1ab_0
+  - mkl=2023.1.0=h213fc3f_46344
+  - mkl-service=2.4.0=py310h5eee18b_1
+  - mkl_fft=1.3.8=py310h5eee18b_0
+  - mkl_random=1.2.4=py310hdb19cb5_0
+  - mpc=1.1.0=h10f8cd9_1
+  - mpfr=4.0.2=hb69a4c5_1
+  - mpmath=1.3.0=py310h06a4308_0
+  - ncurses=6.4=h6a678d5_0
+  - nest-asyncio=1.5.8=pyhd8ed1ab_0
+  - nettle=3.7.3=hbbd107a_1
+  - networkx=3.1=py310h06a4308_0
+  - numpy=1.26.0=py310h5f9d8c6_0
+  - numpy-base=1.26.0=py310hb5e798b_0
+  - openh264=2.1.1=h4ff587b_0
+  - openjpeg=2.4.0=h3ad879b_0
+  - openssl=3.0.12=h7f8727e_0
+  - packaging=23.2=pyhd8ed1ab_0
+  - parso=0.8.3=pyhd8ed1ab_0
+  - pexpect=4.8.0=pyh1a96a4e_2
+  - pickleshare=0.7.5=py_1003
+  - pillow=10.0.1=py310ha6cbd5a_0
+  - pip=23.3=py310h06a4308_0
+  - platformdirs=4.0.0=pyhd8ed1ab_0
+  - prompt-toolkit=3.0.40=pyha770c72_0
+  - prompt_toolkit=3.0.40=hd8ed1ab_0
+  - ptyprocess=0.7.0=pyhd3deb0d_0
+  - pure_eval=0.2.2=pyhd8ed1ab_0
+  - pycparser=2.21=pyhd3eb1b0_0
+  - pygments=2.16.1=pyhd8ed1ab_0
+  - pyopenssl=23.2.0=py310h06a4308_0
+  - pysocks=1.7.1=py310h06a4308_0
+  - python=3.10.13=h955ad1f_0
+  - python-dateutil=2.8.2=pyhd8ed1ab_0
+  - python_abi=3.10=2_cp310
+  - pytorch=2.1.0=py3.10_cuda12.1_cudnn8.9.2_0
+  - pytorch-cuda=12.1=ha16c6d3_5
+  - pytorch-mutex=1.0=cuda
+  - pyyaml=6.0.1=py310h5eee18b_0
+  - pyzmq=25.1.0=py310h6a678d5_0
+  - readline=8.2=h5eee18b_0
+  - requests=2.31.0=py310h06a4308_0
+  - setuptools=68.0.0=py310h06a4308_0
+  - six=1.16.0=pyh6c4a22f_0
+  - sqlite=3.41.2=h5eee18b_0
+  - stack_data=0.6.2=pyhd8ed1ab_0
+  - sympy=1.11.1=py310h06a4308_0
+  - tbb=2021.8.0=hdb19cb5_0
+  - tk=8.6.12=h1ccaba5_0
+  - torchaudio=2.1.0=py310_cu121
+  - torchtriton=2.1.0=py310
+  - torchvision=0.16.0=py310_cu121
+  - traitlets=5.13.0=pyhd8ed1ab_0
+  - typing_extensions=4.7.1=py310h06a4308_0
+  - urllib3=1.26.18=py310h06a4308_0
+  - wcwidth=0.2.9=pyhd8ed1ab_0
+  - wheel=0.41.2=py310h06a4308_0
+  - xz=5.4.2=h5eee18b_0
+  - yaml=0.2.5=h7b6447c_0
+  - zeromq=4.3.4=h2531618_0
+  - zlib=1.2.13=h5eee18b_0
+  - zstd=1.5.5=hc292b87_0
+  - pip:
+      - accelerate==0.30.0
+      - aiohttp==3.9.0
+      - aiosignal==1.3.1
+      - anyio==4.3.0
+      - appdirs==1.4.4
+      - argon2-cffi==23.1.0
+      - argon2-cffi-bindings==21.2.0
+      - arrow==1.3.0
+      - async-lru==2.0.4
+      - async-timeout==4.0.3
+      - attrs==23.1.0
+      - babel==2.15.0
+      - beautifulsoup4==4.12.3
+      - bitsandbytes==0.41.2.post2
+      - bleach==6.1.0
+      - click==8.1.7
+      - contourpy==1.2.0
+      - cycler==0.12.1
+      - datasets==2.14.7
+      - defusedxml==0.7.1
+      - diffusers==0.23.1
+      - dill==0.3.7
+      - docker-pycreds==0.4.0
+      - fastjsonschema==2.19.1
+      - fonttools==4.44.0
+      - fqdn==1.5.1
+      - frozenlist==1.4.0
+      - fsspec==2023.10.0
+      - gitdb==4.0.11
+      - gitpython==3.1.40
+      - h11==0.14.0
+      - httpcore==1.0.5
+      - httpx==0.27.0
+      - huggingface-hub==0.17.3
+      - imageio==2.32.0
+      - importlib-metadata==6.8.0
+      - isoduration==20.11.0
+      - joblib==1.3.2
+      - json5==0.9.25
+      - jsonpointer==2.4
+      - jsonschema==4.22.0
+      - jsonschema-specifications==2023.12.1
+      - jupyter-client==8.6.1
+      - jupyter-events==0.10.0
+      - jupyter-lsp==2.2.5
+      - jupyter-server==2.14.0
+      - jupyter-server-terminals==0.5.3
+      - jupyterlab==4.1.8
+      - jupyterlab-pygments==0.3.0
+      - jupyterlab-server==2.27.1
+      - kiwisolver==1.4.5
+      - lazy-loader==0.3
+      - matplotlib==3.8.1
+      - mistune==3.0.2
+      - multidict==6.0.4
+      - multiprocess==0.70.15
+      - nbclient==0.10.0
+      - nbconvert==7.16.4
+      - nbformat==5.10.4
+      - notebook==7.1.3
+      - notebook-shim==0.2.4
+      - opencv-python==4.8.1.78
+      - overrides==7.7.0
+      - pandas==2.1.3
+      - pandocfilters==1.5.1
+      - prometheus-client==0.20.0
+      - protobuf==4.25.1
+      - psutil==5.9.6
+      - pyarrow==14.0.1
+      - pyarrow-hotfix==0.5
+      - pyparsing==3.1.1
+      - python-json-logger==2.0.7
+      - pytz==2023.3.post1
+      - referencing==0.35.1
+      - regex==2023.10.3
+      - rfc3339-validator==0.1.4
+      - rfc3986-validator==0.1.1
+      - rpds-py==0.18.1
+      - safetensors==0.4.0
+      - scikit-image==0.22.0
+      - scikit-learn==1.3.2
+      - scipy==1.11.3
+      - send2trash==1.8.3
+      - sentry-sdk==1.35.0
+      - setproctitle==1.3.3
+      - smmap==5.0.1
+      - sniffio==1.3.1
+      - soupsieve==2.5
+      - terminado==0.18.1
+      - threadpoolctl==3.2.0
+      - tifffile==2023.9.26
+      - tinycss2==1.3.0
+      - tokenizers==0.14.1
+      - tomli==2.0.1
+      - tornado==6.4
+      - tqdm==4.66.1
+      - transformers==4.35.0
+      - types-python-dateutil==2.9.0.20240316
+      - tzdata==2023.3
+      - uri-template==1.3.0
+      - wandb==0.16.0
+      - webcolors==1.13
+      - webencodings==0.5.1
+      - websocket-client==1.8.0
+      - xformers==0.0.22.post7
+      - xxhash==3.4.1
+      - yarl==1.9.2
+      - zipp==3.17.0