From 9e65b9dd432a26aa57fc2c1409ed6b96bf649041 Mon Sep 17 00:00:00 2001
From: smthem <2102561739@qq.com>
Date: Fri, 11 Apr 2025 11:32:17 +0800
Subject: [PATCH] init
---
PuLID/flux/util.py | 6 +-
README.md | 35 +-
Storydiffusion_node.py | 197 +-
UNO/LICENSE | 201 +
UNO/README.md | 131 +
UNO/app.py | 148 +
UNO/assets/cat_cafe.png | Bin 0 -> 1029762 bytes
UNO/assets/clock.png | Bin 0 -> 1897795 bytes
UNO/assets/crystal_ball.png | Bin 0 -> 1035058 bytes
UNO/assets/cup.png | Bin 0 -> 377963 bytes
UNO/assets/examples/0_0.json | 22 +
UNO/assets/examples/0_0.png | Bin 0 -> 501883 bytes
UNO/assets/examples/1_0.json | 23 +
UNO/assets/examples/1_0.png | Bin 0 -> 583130 bytes
UNO/assets/examples/1one2one/config.json | 8 +
UNO/assets/examples/1one2one/ref1.jpg | Bin 0 -> 139451 bytes
UNO/assets/examples/1one2one/result.png | Bin 0 -> 287640 bytes
UNO/assets/examples/2_0.json | 23 +
UNO/assets/examples/2_0.png | Bin 0 -> 304299 bytes
UNO/assets/examples/2one2one/config.json | 8 +
UNO/assets/examples/2one2one/ref1.png | Bin 0 -> 1006887 bytes
UNO/assets/examples/2one2one/result.png | Bin 0 -> 345627 bytes
UNO/assets/examples/3two2one/config.json | 9 +
UNO/assets/examples/3two2one/ref1.png | Bin 0 -> 760957 bytes
UNO/assets/examples/3two2one/ref2.png | Bin 0 -> 1035058 bytes
UNO/assets/examples/3two2one/result.png | Bin 0 -> 292270 bytes
UNO/assets/examples/4two2one/config.json | 9 +
UNO/assets/examples/4two2one/ref1.png | Bin 0 -> 1029762 bytes
UNO/assets/examples/4two2one/ref2.png | Bin 0 -> 377963 bytes
UNO/assets/examples/4two2one/result.png | Bin 0 -> 165940 bytes
UNO/assets/examples/5many2one/config.json | 10 +
UNO/assets/examples/5many2one/ref1.png | Bin 0 -> 1102955 bytes
UNO/assets/examples/5many2one/ref2.png | Bin 0 -> 725600 bytes
UNO/assets/examples/5many2one/ref3.png | Bin 0 -> 714736 bytes
UNO/assets/examples/5many2one/result.png | Bin 0 -> 358515 bytes
UNO/assets/examples/6t2i/config.json | 7 +
UNO/assets/examples/6t2i/result.png | Bin 0 -> 326423 bytes
UNO/assets/figurine.png | Bin 0 -> 760957 bytes
UNO/assets/logo.png | Bin 0 -> 57492 bytes
UNO/assets/simplecase.jpeg | Bin 0 -> 646283 bytes
UNO/assets/simplecase.jpg | Bin 0 -> 646283 bytes
UNO/assets/teaser.jpg | Bin 0 -> 708037 bytes
UNO/config/deepspeed/zero2_config.json | 15 +
UNO/config/deepspeed/zero3_config.json | 33 +
UNO/datasets/dreambench_multiip.json | 5252 +++++++++++++++++
UNO/datasets/dreambench_singleip.json | 4502 ++++++++++++++
UNO/inference.py | 125 +
UNO/requirements.txt | 12 +
UNO/train.py | 482 ++
UNO/uno/dataset/uno.py | 132 +
UNO/uno/flux/math.py | 45 +
UNO/uno/flux/model.py | 223 +
UNO/uno/flux/modules/autoencoder.py | 327 +
UNO/uno/flux/modules/conditioner.py | 53 +
UNO/uno/flux/modules/layers.py | 436 ++
UNO/uno/flux/pipeline.py | 293 +
UNO/uno/flux/pipeline_origin.py | 304 +
UNO/uno/flux/sampling.py | 388 ++
UNO/uno/flux/util.py | 495 ++
UNO/uno/utils/convert_yaml_to_args_file.py | 34 +
.../flux_dev_quantization_map.json | 1 +
images/UNO.jpg | Bin 0 -> 478036 bytes
.../__pycache__/__init__.cpython-311.pyc | Bin 195 -> 0 bytes
...atglm_256_ipadapter_FaceID.cpython-311.pyc | Bin 48997 -> 0 bytes
model_loader_utils.py | 58 +
example.json => storydiffusion_workflow.json | 1449 +++--
utils/pipeline_img.py | 4 +-
67 files changed, 15046 insertions(+), 454 deletions(-)
create mode 100644 UNO/LICENSE
create mode 100644 UNO/README.md
create mode 100644 UNO/app.py
create mode 100644 UNO/assets/cat_cafe.png
create mode 100644 UNO/assets/clock.png
create mode 100644 UNO/assets/crystal_ball.png
create mode 100644 UNO/assets/cup.png
create mode 100644 UNO/assets/examples/0_0.json
create mode 100644 UNO/assets/examples/0_0.png
create mode 100644 UNO/assets/examples/1_0.json
create mode 100644 UNO/assets/examples/1_0.png
create mode 100644 UNO/assets/examples/1one2one/config.json
create mode 100644 UNO/assets/examples/1one2one/ref1.jpg
create mode 100644 UNO/assets/examples/1one2one/result.png
create mode 100644 UNO/assets/examples/2_0.json
create mode 100644 UNO/assets/examples/2_0.png
create mode 100644 UNO/assets/examples/2one2one/config.json
create mode 100644 UNO/assets/examples/2one2one/ref1.png
create mode 100644 UNO/assets/examples/2one2one/result.png
create mode 100644 UNO/assets/examples/3two2one/config.json
create mode 100644 UNO/assets/examples/3two2one/ref1.png
create mode 100644 UNO/assets/examples/3two2one/ref2.png
create mode 100644 UNO/assets/examples/3two2one/result.png
create mode 100644 UNO/assets/examples/4two2one/config.json
create mode 100644 UNO/assets/examples/4two2one/ref1.png
create mode 100644 UNO/assets/examples/4two2one/ref2.png
create mode 100644 UNO/assets/examples/4two2one/result.png
create mode 100644 UNO/assets/examples/5many2one/config.json
create mode 100644 UNO/assets/examples/5many2one/ref1.png
create mode 100644 UNO/assets/examples/5many2one/ref2.png
create mode 100644 UNO/assets/examples/5many2one/ref3.png
create mode 100644 UNO/assets/examples/5many2one/result.png
create mode 100644 UNO/assets/examples/6t2i/config.json
create mode 100644 UNO/assets/examples/6t2i/result.png
create mode 100644 UNO/assets/figurine.png
create mode 100644 UNO/assets/logo.png
create mode 100644 UNO/assets/simplecase.jpeg
create mode 100644 UNO/assets/simplecase.jpg
create mode 100644 UNO/assets/teaser.jpg
create mode 100644 UNO/config/deepspeed/zero2_config.json
create mode 100644 UNO/config/deepspeed/zero3_config.json
create mode 100644 UNO/datasets/dreambench_multiip.json
create mode 100644 UNO/datasets/dreambench_singleip.json
create mode 100644 UNO/inference.py
create mode 100644 UNO/requirements.txt
create mode 100644 UNO/train.py
create mode 100644 UNO/uno/dataset/uno.py
create mode 100644 UNO/uno/flux/math.py
create mode 100644 UNO/uno/flux/model.py
create mode 100644 UNO/uno/flux/modules/autoencoder.py
create mode 100644 UNO/uno/flux/modules/conditioner.py
create mode 100644 UNO/uno/flux/modules/layers.py
create mode 100644 UNO/uno/flux/pipeline.py
create mode 100644 UNO/uno/flux/pipeline_origin.py
create mode 100644 UNO/uno/flux/sampling.py
create mode 100644 UNO/uno/flux/util.py
create mode 100644 UNO/uno/utils/convert_yaml_to_args_file.py
create mode 100644 config/flux-dev-fp8/flux_dev_quantization_map.json
create mode 100644 images/UNO.jpg
delete mode 100644 kolors/pipelines/__pycache__/__init__.cpython-311.pyc
delete mode 100644 kolors/pipelines/__pycache__/pipeline_stable_diffusion_xl_chatglm_256_ipadapter_FaceID.cpython-311.pyc
rename example.json => storydiffusion_workflow.json (92%)
diff --git a/PuLID/flux/util.py b/PuLID/flux/util.py
index 1e08edb..1968c38 100644
--- a/PuLID/flux/util.py
+++ b/PuLID/flux/util.py
@@ -201,9 +201,9 @@ def load_flow_model_quintized(name: str, ckpt_path,quantized_mode,aggressive_off
import logging
sd= load_torch_file(ckpt_path)
diffusion_model_prefix = model_detection.unet_prefix_from_state_dict(sd)
- weight_dtype = weight_dtype( sd, diffusion_model_prefix)
- logging.info("loading weight_dtype is {}".format(weight_dtype))
- if weight_dtype is None:
+ weight_dtype_ = weight_dtype( sd, diffusion_model_prefix)
+ logging.info("loading weight_dtype is {}".format(weight_dtype_))
+ if weight_dtype_ is None:
if str(torch.__version__).split(".")[-1]=="4.1" and "xlab" in ckpt_path.lower():#"XLabs-AI/flux-dev-fp8" need high env torch 24.1
json_path = os.path.join(folder_paths.base_path, "custom_nodes", "ComfyUI_StoryDiffusion", "utils","flux_dev_quantization_map.json")
else: # Kijai/flux-fp8,Shakker-Labs/AWPortrait-FL
diff --git a/README.md b/README.md
index 50873d9..85455e4 100644
--- a/README.md
+++ b/README.md
@@ -3,14 +3,18 @@
Using StoryDiffusion and other methods to make storys in ComfyUI
----
-* Origin from [StoryDiffusion](https://github.com/HVision-NKU/StoryDiffusion) [MS-Diffusion](https://github.com/MS-Diffusion/MS-Diffusion),[StoryMaker](https://github.com/RedAIGC/StoryMaker),[Consistory](https://github.com/NVlabs/consistory),[Kolor](https://github.com/Kwai-Kolors/Kolors),[Pulid](https://github.com/ToTheBeginning/PuLID),[Flux](https://github.com/black-forest-labs/flux),[photomaker](https://github.com/TencentARC/PhotoMaker),[IP-Adapter](https://github.com/tencent-ailab/IP-Adapter) [InfiniteYou](https://github.com/bytedance/InfiniteYou)
+* Origin from [StoryDiffusion](https://github.com/HVision-NKU/StoryDiffusion) [MS-Diffusion](https://github.com/MS-Diffusion/MS-Diffusion),[StoryMaker](https://github.com/RedAIGC/StoryMaker),[Consistory](https://github.com/NVlabs/consistory),[Kolor](https://github.com/Kwai-Kolors/Kolors),[Pulid](https://github.com/ToTheBeginning/PuLID),[Flux](https://github.com/black-forest-labs/flux),[photomaker](https://github.com/TencentARC/PhotoMaker),[IP-Adapter](https://github.com/tencent-ailab/IP-Adapter), [InfiniteYou](https://github.com/bytedance/InfiniteYou), [UNO](https://github.com/bytedance/UNO)
+
## Updates:
-* 2025/04/06
+* 2025/04/11
+* Add UNO support,Only the single FLUX model (27G) and UNO's Lora are needed. Please enable FP8 quantization and use storydiffusionw_flowjson workflow testing ,fix a bug,
+* 新增UNO支持,只需要单体FLUX模型(27G)和UNO的lora,请开启fp8量化和使用storydiffusion_workflow.json工作流测试,修复tokens过长的bug;
+
+* previous
* Add infinite svdq v0.2 support,it'work well when your svdq update v0.2,[download wheel](https://huggingface.co/mit-han-lab/nunchaku/tree/main) 更新 svdq v0.2的支持,infinite工作正常,[轮子](https://huggingface.co/mit-han-lab/nunchaku/tree/main)下载地址。
-* 2025/04/05
-* 1.修改了模型加载的流程,更新到V2版本,如果你喜欢旧的,可以下载V1.0版本的,2.请使用example.json,它集成了主要的工作流;3.剔除掉一些过时的功能;
-* 1.Modified the model loading process.Update to V2 version, If you like the old one, you can download version 1.0,2.Please use 'example.json', which integrates the main workflow,3.Remove some outdated features;
+* 1.修改了模型加载的流程,更新到V2版本,如果你喜欢旧的,可以下载V1.0版本的,2.请使用storydiffusion_workflow.json,它集成了主要的工作流;3.剔除掉一些过时的功能;
+* 1.Modified the model loading process.Update to V2 version, If you like the old one, you can download version 1.0,2.Please use 'storydiffusion_workflow.json', which integrates the main workflow,3.Remove some outdated features;
1.Installation
@@ -165,6 +169,13 @@ or
download svdquant repo from [here](https://huggingface.co/mit-han-lab/svdq-fp4-flux.1-dev/tree/main) and fill local path in 'easyfunction_lite' node's 'select_method'
+**3.7 UNO mode**
+download lora [dit_lora.safetensor](https://huggingface.co/bytedance-research/UNO/tree/main),use fp8,if Vram <24.
+```
+├── ComfyUI/models/
+| ├── diffusion_models/flux1-dev.safetensors #
+| ├── loras/dit_lora.safetensors #
+```
4 Example
----
@@ -206,7 +217,10 @@ or
* svdq,升级到v.2工作正常
-**4.8 comfyUI classic(comfyUI经典模式,可以接任意适配CF的流程,主要是方便使用多角色的clip)**
+**4.8 UNO**
+
+
+**4.9 comfyUI classic(comfyUI经典模式,可以接任意适配CF的流程,主要是方便使用多角色的clip)**
* any mode SD1.5 SDXL SD3.5 FLUX...
@@ -306,3 +320,12 @@ svdquant
```
[GGUF](https://github.com/city96/ComfyUI-GGUF)
[FLUX LICENSE](https://huggingface.co/black-forest-labs/FLUX.1-dev/blob/main/LICENSE.md)
+
+```
+@article{wu2025less,
+ title={Less-to-More Generalization: Unlocking More Controllability by In-Context Generation},
+ author={Wu, Shaojin and Huang, Mengqi and Wu, Wenxu and Cheng, Yufeng and Ding, Fei and He, Qian},
+ journal={arXiv preprint arXiv:2504.02160},
+ year={2025}
+}
+```
\ No newline at end of file
diff --git a/Storydiffusion_node.py b/Storydiffusion_node.py
index dbbac96..0401d7a 100644
--- a/Storydiffusion_node.py
+++ b/Storydiffusion_node.py
@@ -11,9 +11,10 @@ import folder_paths
from comfy.model_management import total_vram
import comfy
import latent_preview
+import torchvision.transforms.functional as TVF
from .utils.utils import get_comic
from .model_loader_utils import (phi2narry,Loader_storydiffusion,Loader_Flux_Pulid,load_pipeline_consistory,Loader_KOLOR,replicate_data_by_indices,glm_single_encode,
- get_float,gc_cleanup,tensor_to_image,photomaker_clip,tensortopil_list,tensortopil_list_upscale,
+ get_float,gc_cleanup,tensor_to_image,photomaker_clip,Loader_UNO,tensortopil_list_upscale,tensortopil_list,
narry_list_pil,pre_text2infer,cf_clip,get_phrases_idx_cf,get_eot_idx_cf,get_ms_phrase_emb,get_extra_function,photomaker_clip_v2,adjust_indices,
get_scheduler,apply_style_positive,fitter_cf_model_type,Infer_MSdiffusion,Loader_story_maker,Loader_InfiniteYou,
nomarl_upscale,SAMPLER_NAMES,SCHEDULER_NAMES,lora_lightning_list)
@@ -412,8 +413,7 @@ class Pre_Translate_prompt:
def translate_prompt(self, scene_prompts, keep_character_name):
captions = scene_prompts.splitlines()
if not keep_character_name:
- captions = [caption.split(")", 1)[-1] if ")" in caption else caption
- for caption in captions] # del character
+ captions = [caption.split(")", 1)[-1] if ")" in caption else caption for caption in captions] # del character
else:
captions = [caption.replace("(", "").replace(")", "") if "(" or ")" in caption else caption
for caption in captions] # del ()
@@ -434,7 +434,7 @@ class EasyFunction_Lite:
def INPUT_TYPES(s):
return {"required": {"extra_repo": ("STRING", { "default": ""}),
"checkpoints": (["none"] + folder_paths.get_filename_list("diffusion_models")+folder_paths.get_filename_list("gguf")+folder_paths.get_filename_list("clip"),),
- "clip_vision": (["none"] + folder_paths.get_filename_list("clip_vision"),),
+ "clip_vision": (["none"] + folder_paths.get_filename_list("clip_vision")+folder_paths.get_filename_list("loras"),),
"function_mode": (["none","tag", "clip","mask","infinite",],),
"select_method":("STRING",{ "default":""}),
"temperature": (
@@ -448,7 +448,7 @@ class EasyFunction_Lite:
def easy_function_main(self, extra_repo,checkpoints,clip_vision,function_mode,select_method,temperature):
use_gguf,use_unet=False,False
- model_path,clip=None,None
+ model_path,clip,lora_path=None,None,None
if checkpoints != "none":
if checkpoints.endswith(".gguf"):
@@ -461,7 +461,12 @@ class EasyFunction_Lite:
use_unet=True
if clip_vision != "none":
- clip_vision_path = folder_paths.get_full_path("clip_vision", clip_vision)
+ if "clip_vision" in clip_vision:
+ clip_vision_path = folder_paths.get_full_path("clip_vision", clip_vision)
+ else:
+ lora_path = folder_paths.get_full_path("loras", clip_vision)
+ clip_vision_path = None
+
else:
clip_vision_path=None
if function_mode=="tag":
@@ -478,24 +483,27 @@ class EasyFunction_Lite:
use_svdq=True if "svdq" in select_method else False
- pipe={"model":model,"extra_repo":extra_repo,"model_path":model_path,"use_svdq":use_svdq,"use_gguf":use_gguf,"use_unet":use_unet,"extra_easy":function_mode,"select_method":select_method,"clip_vision_path":clip_vision_path}
+ pipe={"model":model,"extra_repo":extra_repo,"model_path":model_path,"use_svdq":use_svdq,"use_gguf":use_gguf,"lora_ckpt_path":lora_path,
+ "use_unet":use_unet,"extra_easy":function_mode,"select_method":select_method,"clip_vision_path":clip_vision_path}
return (pipe,clip)
class StoryDiffusion_Apply:
@classmethod
def INPUT_TYPES(s):
- return {"required": {
- "model": ("MODEL",),
- "vae": ("VAE",),
- "infer_mode": (["story", "classic","flux_pulid","infiniteyou","story_maker","story_and_maker","consistory","kolor_face","msdiffusion" ],),
- "photomake_ckpt": (["none"] + [i for i in folder_paths.get_filename_list("photomaker") if "v1" in i or "v2" in i],),
- "ipadapter_ckpt": (["none"] + folder_paths.get_filename_list("photomaker"),),
- "quantize_mode": ([ "fp8", "nf4","fp16", ],),
- "lora_scale": ("FLOAT", {"default": 0.8, "min": 0.1, "max": 1.0, "step": 0.1}),
- "extra_funtion":("STRING", {"default": ""}),
- },
- "optional":{"CLIP_VISION": ("CLIP_VISION",),
+ return {"required":
+ {
+ "model": ("MODEL",),
+ "vae": ("VAE",),
+ "infer_mode": (["story", "classic","flux_pulid","infiniteyou","uno","story_maker","story_and_maker","consistory","kolor_face","msdiffusion" ],),
+ "photomake_ckpt": (["none"] + [i for i in folder_paths.get_filename_list("photomaker") if "v1" in i or "v2" in i],),
+ "ipadapter_ckpt": (["none"] + folder_paths.get_filename_list("photomaker"),),
+ "quantize_mode": ([ "fp8", "nf4","fp16", ],),
+ "lora_scale": ("FLOAT", {"default": 0.8, "min": 0.1, "max": 1.0, "step": 0.1}),
+ "extra_funtion":("STRING", {"default": ""}),
+ },
+ "optional":{
+ "CLIP_VISION": ("CLIP_VISION",),
}
}
@@ -512,7 +520,7 @@ class StoryDiffusion_Apply:
CLIP_VISION=kwargs.get("CLIP_VISION")
-
+
unet_type=torch.float16 #use for sdxl
image_proj_model=None
@@ -526,6 +534,8 @@ class StoryDiffusion_Apply:
clip_vision_path=None
repo=None
+
+ save_quantezed=True if "save" in extra_funtion and quantize_mode=="fp8" else False
print(f"infer model type is {cf_model_type}")
@@ -543,27 +553,24 @@ class StoryDiffusion_Apply:
if infer_mode in ["story_maker" ,"story_and_maker"] and not CLIP_VISION and ipadapter_ckpt_path is None:
raise "story_maker need a clipvison H model,mask.bin"
-
+ # check vram only using in flux pulid or UNO
+ if total_vram > 45000.0:
+ aggressive_offload = False
+ offload = False
+ elif 17000.0 < total_vram < 45000.0:
+ aggressive_offload = False
+ offload = True
+ else:
+ aggressive_offload = True
+ offload = True
+
+ logging.info(f"total_vram is {total_vram},aggressive_offload is {aggressive_offload},offload is {offload}")
if infer_mode in["story", "story_and_maker","msdiffusion"]:# mix mode,use maker or ms to make 2 roles in on image
model = Loader_storydiffusion(model,photomake_ckpt_path,vae)
elif infer_mode =="story_maker":
model = Loader_story_maker(model,ipadapter_ckpt_path,vae,False,lora_scale)
elif infer_mode == "flux_pulid":
-
- # check vram only using in flux pulid
- if total_vram > 45000.0:
- aggressive_offload = False
- offload = False
- elif 17000.0 < total_vram < 45000.0:
- aggressive_offload = False
- offload = True
- else:
- aggressive_offload = True
- offload = True
-
- logging.info(f"total_vram is {total_vram},aggressive_offload is {aggressive_offload},offload is {offload}")
-
from .PuLID.app_flux import get_models
if isinstance(model,dict):
ckpt_path=model.get("model_path")
@@ -585,9 +592,10 @@ class StoryDiffusion_Apply:
if repo is None:
raise "EasyFunction_Lite node extra_repo must fill kolor repo"
model = Loader_KOLOR(repo,clip_vision_path,ipadapter_ckpt_path)
+ elif infer_mode == "uno":
+ model = Loader_UNO(model,offload,quantize_mode,save_quantezed,lora_rank=512)
else: # can not choice a mode
print("infer use comfyui classic mode")
-
story_img=True if photomake_ckpt_path and infer_mode in["story","story_maker","story_and_maker","msdiffusion"] else False
model_=model if infer_mode=="flux_pulid" or story_img else None
@@ -692,6 +700,22 @@ class StoryDiffusion_CLIPTextEncode:
role_key_list=replicate_data_by_indices(role_list, index_char_1_list, index_char_2_list)
role_key_list=[i for i in role_key_list if i is not None]
+ if len(role_list)>1: #重新整理prompt排序,便于后续emb和ID的对应
+ nc_dual_list=[]
+ for i in nc_indexs:
+ nc_dual_list.append(i)
+ for i in positions_index_dual:
+ nc_dual_list.append(i)
+ adjusted_a = adjust_indices(index_char_1_list, nc_dual_list)
+ adjusted_b = adjust_indices(index_char_2_list, nc_dual_list)
+ a_list = [only_role_list[i] for i in adjusted_a if 0 <= i < len(only_role_list)]
+ b_list = [only_role_list[i] for i in adjusted_b if 0 <= i < len(only_role_list)]
+
+ inf_list_split=[a_list,b_list]
+ else:
+ inf_list_split=only_role_list
+
+
# if prompts_dual and infer_mode not in ["story_and_maker","story_maker","msdiffusion"]:
# raise "only support prompts_dual role in story maker and msdiffusion"
@@ -710,10 +734,10 @@ class StoryDiffusion_CLIPTextEncode:
if infer_mode=="msdiffusion" and photomake_ckpt_path and not img2img_mode:
raise "if use msdiffusion txt2img mode,can not use photomake_ckpt_path"
+ uno_pe="d" #Literal['d', 'h', 'w', 'o']
+
+ only_role_emb_ne,x_1_refs_dict,image_emb=None,None,None
-
- only_role_emb_ne=None
- image_emb = None
if img2img_mode:
if infer_mode == "story_and_maker" or infer_mode == "story_maker":
k1, _, _, _ = image.size()
@@ -737,8 +761,53 @@ class StoryDiffusion_CLIPTextEncode:
pass
elif infer_mode == "infiniteyou":
pass
+ elif infer_mode == "uno":
+ from .UNO.uno.flux.sampling import get_noise
+ from .UNO.uno.flux.pipeline import preprocess_ref
+
+ ref_pil_list=tensortopil_list(image) #原方法需要图片缩放,先转回pil
+
+ if control_image is not None:
+ control_pil_list=tensortopil_list(control_image)
+ else:
+ control_pil_list=None
+
+ if control_pil_list is None:# 无控制图时,默认时单图模式,x_1_refs用key取值
+ x_1_refs_dict={}
+ for key ,role_pil,prompts in zip(role_list,ref_pil_list,inf_list_split):
+ role_tensor=phi2narry(preprocess_ref(role_pil, 512))#单图默认512
+ x_1_refs_dict[key]=[[vae.encode(role_tensor[:,:,:,:3]).to(torch.bfloat16)]*len(prompts)] # {key:[[1],[2],key2:[[3],[4]]}
+ else:
+ if len(control_pil_list) != len(only_role_list):
+ raise "control image must same size as prompts,多少句话就多少张图"
+ else:
+ if len(role_list)>1:
+ nc_dual_list=[]
+ for i in nc_indexs:
+ nc_dual_list.append(i)
+ for i in positions_index_dual:
+ nc_dual_list.append(i)
+ adjusted_a = adjust_indices(index_char_1_list, nc_dual_list)
+ adjusted_b = adjust_indices(index_char_2_list, nc_dual_list)
+ a_list = [control_pil_list[i] for i in adjusted_a if 0 <= i < len(control_pil_list)]
+ b_list = [control_pil_list[i] for i in adjusted_b if 0 <= i < len(control_pil_list)]
+
+ control_pil_list=[a_list,b_list]
+ else:
+ control_pil_list=[control_pil_list]
+ x_1_refs_dict={}
+ for key ,role_pil,control_pil in zip(role_list,ref_pil_list,control_pil_list):
+ mix_list=[]
+ role_tensor=phi2narry(preprocess_ref(role_pil, 320) )#多图默认320
+ for c in (control_pil):
+
+ c_tensor=phi2narry(preprocess_ref(c, 320))#多图默认320
+ mix_list.append([vae.encode(role_tensor[:,:,:,:3]).to(torch.bfloat16),vae.encode(c_tensor[:,:,:,:3]).to(torch.bfloat16)])
+ x_1_refs_dict[key]=mix_list #{key:[[1,2],[3,4]]}
else:
pass
+ else:
+ pass
# pre insight face model and emb
if img2img_mode:
@@ -775,23 +844,6 @@ class StoryDiffusion_CLIPTextEncode:
image_list = [nomarl_upscale(img, width, height) for img in img_list]
for index, key in enumerate(role_list):
input_id_images_dict[key]=image_list[index]
-
-
-
- if len(role_list)>1: #重新整理prompt排序,便于后续emb和ID的对应
- nc_dual_list=[]
- for i in nc_indexs:
- nc_dual_list.append(i)
- for i in positions_index_dual:
- nc_dual_list.append(i)
- adjusted_a = adjust_indices(index_char_1_list, nc_dual_list)
- adjusted_b = adjust_indices(index_char_2_list, nc_dual_list)
- a_list = [only_role_list[i] for i in adjusted_a if 0 <= i < len(only_role_list)]
- b_list = [only_role_list[i] for i in adjusted_b if 0 <= i < len(only_role_list)]
-
- inf_list_split=[a_list,b_list]
- else:
- inf_list_split=only_role_list
#print("inf_list_split",inf_list_split)
# pre clip txt emb
@@ -817,8 +869,9 @@ class StoryDiffusion_CLIPTextEncode:
for key ,prompts in zip(role_list,inf_list_split):
ip_emb,noise_,inp_n=[],[],[]
- seed_random = random.randint(0, MAX_SEED) #pulid的emb需要随机数
+
for p,n in zip(prompts,[neg_text]*len(prompts)):
+ seed_random = random.randint(0, MAX_SEED) #pulid 和uno的emb需要随机数
inp,inp_neg,x=get_emb_flux_pulid(t5_,clip_,if_repo,seed_random,p,n,width,height,num_steps=20,guidance=3.5,device=device)
ip_emb.append(inp)
inp_n.append(inp_neg)
@@ -826,6 +879,20 @@ class StoryDiffusion_CLIPTextEncode:
only_role_emb[key]=ip_emb
noise_x[key]=noise_
inp_neg_list[key]=inp_n
+ elif infer_mode == "uno":
+ only_role_emb={}
+ from .UNO.uno.flux.sampling import prepare_multi_ip_wrapper
+ for key ,prompts in zip(role_list,inf_list_split):
+ ip_emb=[]
+
+ for p,x_1 in zip(prompts,x_1_refs_dict[key]):
+ seed_random = random.randint(0, MAX_SEED) #pulid 和uno的emb需要随机数
+ uno_x = get_noise(1, height, width, device=device,dtype=torch.bfloat16, seed=seed_random)
+ inp = prepare_multi_ip_wrapper(clip,img=uno_x,prompt=p, ref_imgs=x_1, pe=uno_pe)
+ ip_emb.append(inp)
+ only_role_emb[key]=ip_emb
+
+
elif infer_mode=="kolor_face":
from .kolors.models.tokenization_chatglm import ChatGLMTokenizer
tokenizer = ChatGLMTokenizer.from_pretrained(os.path.join(switch.get("repo"),'text_encoder'))
@@ -927,6 +994,9 @@ class StoryDiffusion_CLIPTextEncode:
elif infer_mode=="flux_pulid":
postive_dict= {"role": only_role_emb, "nc": None, "daul": daul_emb} #不支持NC
negative = [inp_neg_list,noise_x]
+ elif infer_mode=="uno":
+ postive_dict= {"role": only_role_emb, "nc": None, "daul": daul_emb} #TODO
+ negative=None
elif infer_mode=="kolor_face":
postive_dict = {"role": only_role_emb, "nc": nc_emb, "daul": daul_emb}
negative = only_role_emb_ne[0]
@@ -1615,6 +1685,21 @@ class StoryDiffusion_KSampler:
out = {}
out["samples"] = torch.cat(samples_list, dim=0)
return (out,)
+ elif infer_mode=="uno":
+ samples_list = []
+ for key in role_list:
+ for index,emb in enumerate(only_role_emb[key]):
+ #print(emb)
+ samples = model(width=width,
+ height=height,guidance=cfg,
+ num_steps=steps,
+ inp_cond=emb,
+ ) # torch.Size([1, 4, 64, 64])
+
+ samples_list.append(samples)
+ out = {}
+ out["samples"] = torch.cat(samples_list, dim=0)
+ return (out,)
else: #none:
return
diff --git a/UNO/LICENSE b/UNO/LICENSE
new file mode 100644
index 0000000..f49a4e1
--- /dev/null
+++ b/UNO/LICENSE
@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
\ No newline at end of file
diff --git a/UNO/README.md b/UNO/README.md
new file mode 100644
index 0000000..05376cd
--- /dev/null
+++ b/UNO/README.md
@@ -0,0 +1,131 @@
+
+ Less-to-More Generalization: Unlocking More Controllability by In-Context Generation
+ Shaojin Wu, Mengqi Huang*, Wenxu Wu, Yufeng Cheng, Fei Ding+, Qian He
+>Intelligent Creation Team, ByteDance
+
+
+
+
+We open-source this project for academic research. The vast majority of images
+used in this project are either generated or licensed. If you have any concerns,
+please contact us, and we will promptly remove any inappropriate content.
+Our code is released under the Apache 2.0 License,, while our models are under
+the CC BY-NC 4.0 License. Any models related to FLUX.1-dev
+base model must adhere to the original licensing terms.
+
This research aims to advance the field of generative AI. Users are free to
+create images using this tool, provided they comply with local laws and exercise
+responsible usage. The developers are not liable for any misuse of the tool by users.