diff --git a/PuLID/app_flux.py b/PuLID/app_flux.py index 8a9f6f2..294ca0c 100644 --- a/PuLID/app_flux.py +++ b/PuLID/app_flux.py @@ -8,29 +8,31 @@ from .flux.util import load_ae, load_clip, load_flow_model, load_t5, load_flow_m from .pulid.pipeline_flux import PuLIDPipeline -def get_models(name: str,ckpt_path, device: torch.device, offload: bool,quantized_mode): - t5 = load_t5(name,quantized_mode,device, max_length=128) - clip = load_clip(name,quantized_mode,device) +def get_models(name: str,ckpt_path,vae_cf, clip_cf,device: torch.device, offload: bool,quantized_mode): + t5 = load_t5(name,clip_cf,quantized_mode,device, max_length=128) + clip = load_clip(name,clip_cf,quantized_mode,device) if quantized_mode=="fp8" or quantized_mode=="nf4": model = load_flow_model_quintized(name, ckpt_path,quantized_mode,device="cpu" if offload else device) else: model = load_flow_model(name, ckpt_path,device="cpu" if offload else device) model.eval() - ae = load_ae(name, device="cpu" if offload else device) + ae = load_ae(name,vae_cf, device="cpu" if offload else device) return model, ae, t5, clip class FluxGenerator: - def __init__(self, model_name: str, ckpt_path,device: str, offload: bool,aggressive_offload: bool, pretrained_model,quantized_mode,clip_vision_path): + def __init__(self, model_name: str, ckpt_path,device: str, offload: bool,aggressive_offload: bool, pretrained_model,quantized_mode,clip_vision_path,clip_cf,vae_cf): self.device = torch.device(device) self.offload = offload self.model_name = model_name self.aggressive_offload= aggressive_offload self.ckpt_path=ckpt_path self.clip_vision_path=clip_vision_path + self.clip_cf=clip_cf + self.vae_cf=vae_cf self.quantized_mode=quantized_mode self.model, self.ae, self.t5, self.clip = get_models( - model_name,self.ckpt_path, + model_name,self.ckpt_path,self.vae_cf,self.clip_cf, device=self.device, offload=self.offload,quantized_mode=self.quantized_mode, ) @@ -55,7 +57,7 @@ class FluxGenerator: timestep_to_start_cfg=1, max_sequence_length=128, ): - self.t5.max_length = max_sequence_length + #self.t5.max_length = max_sequence_length seed = int(seed) if seed == -1: @@ -69,9 +71,9 @@ class FluxGenerator: raise "prompt must be list or str" img_output = [] - for i,prompt in enumerate(prompt_list): + for i in prompt_list: opts = SamplingOptions( - prompt=prompt, + text=i, width=width, height=height, num_steps=num_steps, @@ -81,7 +83,8 @@ class FluxGenerator: if opts.seed is None: opts.seed = torch.Generator(device="cpu").seed() - print(f"Generating '{opts.prompt}' with seed {opts.seed}") + opts.text = opts.text if isinstance(opts.text,str) else str(opts.text) + print(f"Generating '{opts.text}' with seed {opts.seed}") t0 = time.perf_counter() use_true_cfg = abs(true_cfg - 1.0) > 1e-2 @@ -103,7 +106,7 @@ class FluxGenerator: if self.offload: self.t5, self.clip = self.t5.to(self.device), self.clip.to(self.device) - inp = prepare(t5=self.t5, clip=self.clip, img=x, prompt=opts.prompt) + inp = prepare(t5=self.t5, clip=self.clip, img=x, prompt=opts.text) inp_neg = prepare(t5=self.t5, clip=self.clip, img=x, prompt=neg_prompt) if use_true_cfg else None # offload TEs to CPU, load model to gpu @@ -156,4 +159,4 @@ class FluxGenerator: img = Image.fromarray((127.5 * (x + 1.0)).cpu().byte().numpy()) # return img, str(opts.seed), self.pulid_model.debug_img_list img_output.append(img) - return img_output + return img_output \ No newline at end of file diff --git a/PuLID/flux/modules/conditioner.py b/PuLID/flux/modules/conditioner.py index 234ecec..d56c56a 100644 --- a/PuLID/flux/modules/conditioner.py +++ b/PuLID/flux/modules/conditioner.py @@ -3,41 +3,47 @@ from transformers import CLIPTextModel, CLIPTokenizer, T5EncoderModel, T5Tokeniz class HFEmbedder(nn.Module): - def __init__(self, version: str, tokenizer_path,is_clip,quantized_mode, max_length: int, **hf_kwargs): + def __init__(self, version: str, tokenizer_path,clip_cf,is_clip,quantized_mode, max_length: int, **hf_kwargs): super().__init__() #self.is_clip = version.startswith("openai") self.is_clip=is_clip self.quantized_mode=quantized_mode + self.clip_cf=clip_cf self.max_length = max_length - self.output_key = "pooler_output" if self.is_clip else "last_hidden_state" + #self.output_key = "pooler_output" if self.is_clip else "last_hidden_state" + self.output_key = "pooled_output" if self.is_clip else "cond" - if self.is_clip: - self.tokenizer: CLIPTokenizer = CLIPTokenizer.from_pretrained(tokenizer_path, max_length=max_length) - self.hf_module: CLIPTextModel = CLIPTextModel.from_pretrained(version, **hf_kwargs) - else: - self.tokenizer: T5Tokenizer = T5Tokenizer.from_pretrained(tokenizer_path, max_length=max_length) - self.hf_module: T5EncoderModel = T5EncoderModel.from_pretrained(version, **hf_kwargs) - if self.quantized_mode=="fp8": - from optimum.quanto import freeze, qfloat8, quantize - quantize(self.hf_module, weights=qfloat8) - freeze(self.hf_module) - - self.hf_module = self.hf_module.eval().requires_grad_(False) + # if self.is_clip: + # self.tokenizer: CLIPTokenizer = CLIPTokenizer.from_pretrained(tokenizer_path, max_length=max_length) + # self.hf_module: CLIPTextModel = CLIPTextModel.from_pretrained(version, **hf_kwargs) + # else: + # self.tokenizer: T5Tokenizer = T5Tokenizer.from_pretrained(tokenizer_path, max_length=max_length) + # self.hf_module: T5EncoderModel = T5EncoderModel.from_pretrained(version, **hf_kwargs) + # if self.quantized_mode=="fp8": + # from optimum.quanto import freeze, qfloat8, quantize + # quantize(self.hf_module, weights=qfloat8) + # freeze(self.hf_module) + # + # self.hf_module = self.hf_module.eval().requires_grad_(False) def forward(self, text: list[str]) -> Tensor: - batch_encoding = self.tokenizer( - text, - truncation=True, - max_length=self.max_length, - return_length=False, - return_overflowing_tokens=False, - padding="max_length", - return_tensors="pt", - ) + # batch_encoding = self.tokenizer( + # text, + # truncation=True, + # max_length=self.max_length, + # return_length=False, + # return_overflowing_tokens=False, + # padding="max_length", + # return_tensors="pt", + # ) - outputs = self.hf_module( - input_ids=batch_encoding["input_ids"].to(self.hf_module.device), - attention_mask=None, - output_hidden_states=False, - ) - return outputs[self.output_key] + # outputs = self.hf_module( + # input_ids=batch_encoding["input_ids"].to(self.hf_module.device), + # attention_mask=None, + # output_hidden_states=False, + # ) + tokens = self.clip_cf.tokenize(text) + outputs = self.clip_cf.encode_from_tokens(tokens, return_pooled=True, return_dict=True) + # cond = outputs.pop("cond") + # pooled_output=outputs.pop("pooled_output") + return outputs.pop(self.output_key) diff --git a/PuLID/flux/sampling.py b/PuLID/flux/sampling.py index a4a6480..e22b81b 100644 --- a/PuLID/flux/sampling.py +++ b/PuLID/flux/sampling.py @@ -43,8 +43,8 @@ def prepare(t5: HFEmbedder, clip: HFEmbedder, img: Tensor, prompt: str) -> dict[ img_ids[..., 2] = img_ids[..., 2] + torch.arange(w // 2)[None, :] img_ids = repeat(img_ids, "h w c -> b (h w) c", b=bs) - if isinstance(prompt, str): - prompt = [prompt] + # if isinstance(prompt, str): + # prompt = [prompt] txt = t5(prompt) if txt.shape[0] == 1 and bs > 1: txt = repeat(txt, "1 ... -> bs ...", bs=bs) @@ -56,10 +56,10 @@ def prepare(t5: HFEmbedder, clip: HFEmbedder, img: Tensor, prompt: str) -> dict[ return { "img": img, - "img_ids": img_ids.to(img.device), - "txt": txt.to(img.device), - "txt_ids": txt_ids.to(img.device), - "vec": vec.to(img.device), + "img_ids": img_ids.to(img.device,torch.bfloat16), + "txt": txt.to(img.device,torch.bfloat16), + "txt_ids": txt_ids.to(img.device,torch.bfloat16), + "vec": vec.to(img.device,torch.bfloat16), } diff --git a/PuLID/flux/util.py b/PuLID/flux/util.py index 6942cbe..c5e1e73 100644 --- a/PuLID/flux/util.py +++ b/PuLID/flux/util.py @@ -2,7 +2,6 @@ import os from dataclasses import dataclass import torch -#from .transformer_flux import FluxTransformer2DModel from huggingface_hub import hf_hub_download from safetensors.torch import load_file as load_sft import json @@ -13,7 +12,7 @@ import folder_paths @dataclass class SamplingOptions: - prompt: str + text: str width: int height: int num_steps: int @@ -144,23 +143,60 @@ def print_load_warning(missing: list[str], unexpected: list[str]) -> None: # from XLabs-AI https://github.com/XLabs-AI/x-flux/blob/1f8ef54972105ad9062be69fe6b7f841bce02a08/src/flux/util.py#L330 def load_flow_model_quintized(name: str, ckpt_path,quantized_mode,device: str = "cuda", hf_download: bool = True): + from optimum.quanto import requantize,qint4,quantize,freeze # Loading Flux - print("Init model") - if "dev" in name: - name = "flux-dev" - else: - name = "flux-schnell" + print(f"Init model in {quantized_mode}") model = Flux(configs[name].params).to(torch.bfloat16) - if quantized_mode == "nf4": #nf4 is now useful - print("Loading checkpoint in nf4") - json_path = os.path.join(folder_paths.base_path, "custom_nodes", "ComfyUI_StoryDiffusion", "utils", "config.json") + if quantized_mode == "nf4": #nf4 is now useful. + json_path = os.path.join(folder_paths.base_path, "custom_nodes", "ComfyUI_StoryDiffusion", "utils", + "config.json") + test=False + if test: + from accelerate.utils import set_module_tensor_to_device, compute_module_sizes + from accelerate import init_empty_weights + from .convert_nf4_flux import _replace_with_bnb_linear, create_quantized_param, \ + check_quantized_param + import safetensors.torch + import gc + from diffusers import FluxTransformer2DModel + dtype = torch.bfloat16 + is_torch_e4m3fn_available = hasattr(torch, "float8_e4m3fn") + original_state_dict = safetensors.torch.load_file(ckpt_path) + print("Start a quantization process...") + with init_empty_weights(): + config = FluxTransformer2DModel.load_config(json_path) + model = FluxTransformer2DModel.from_config(config).to(dtype) + expected_state_dict_keys = list(model.state_dict().keys()) + + _replace_with_bnb_linear(model, "nf4") + + for param_name, param in original_state_dict.items(): + if param_name not in expected_state_dict_keys: + continue + + is_param_float8_e4m3fn = is_torch_e4m3fn_available and param.dtype == torch.float8_e4m3fn + if torch.is_floating_point(param) and not is_param_float8_e4m3fn: + param = param.to(dtype) + + if not check_quantized_param(model, param_name): + set_module_tensor_to_device(model, param_name, device=0, value=param) + else: + create_quantized_param( + model, param, param_name, target_device=0, state_dict=original_state_dict, + pre_quantized=True + ) + + del original_state_dict + gc.collect() + + else: sd = load_sft(ckpt_path, device='cpu') with open(json_path) as f: quantization_map = json.load(f) + print("Start a quantization process...") - from optimum.quanto import requantize requantize(model, sd, quantization_map) - print("Model is quantized!") + print("Model is quantized!") else: #fp8 # ckpt_path = 'models/flux-dev-fp8.safetensors' if ( @@ -172,27 +208,20 @@ def load_flow_model_quintized(name: str, ckpt_path,quantized_mode,device: str = else: json_path = os.path.join(folder_paths.base_path, "custom_nodes", "ComfyUI_StoryDiffusion", "utils", "config.json") - print("Loading checkpoint in fp8") # load_sft doesn't support torch.device sd = load_sft(ckpt_path, device='cpu') with open(json_path) as f: quantization_map = json.load(f) print("Start a quantization process...") - from optimum.quanto import requantize requantize(model, sd, quantization_map) print("Model is quantized!") return model def load_flow_model(name: str,ckpt_path, device: str = "cuda", hf_download: bool = True): # Loading Flux - print("Init model") - if "dev" in name: - name="flux-dev" - else: - name="flux-schnell" + print("Init model in fp16") if ckpt_path is not None: # load_sft doesn't support torch.device - print("Loading checkpoint in fp16") with torch.device(device): model = Flux(configs[name].params).to(torch.bfloat16) sd = load_sft(ckpt_path, device=str(device)) @@ -201,24 +230,21 @@ def load_flow_model(name: str,ckpt_path, device: str = "cuda", hf_download: bool return model -def load_t5(name,quantized_mode,device = "cuda", max_length: int = 512) -> HFEmbedder: +def load_t5(name,clip_cf,quantized_mode,device = "cuda", max_length: int = 512) -> HFEmbedder: # max length 64, 128, 256 and 512 should work (if your sequence is short enough) #return HFEmbedder("xlabs-ai/xflux_text_encoders", max_length=max_length, torch_dtype=torch.bfloat16).to(device) - return HFEmbedder(f"{name}/text_encoder_2", f"{name}/tokenizer_2",is_clip=False,quantized_mode=quantized_mode,max_length=max_length, torch_dtype=torch.bfloat16).to(device) + return HFEmbedder(f"{name}/text_encoder_2", f"{name}/tokenizer_2",clip_cf,is_clip=False,quantized_mode=quantized_mode,max_length=max_length, torch_dtype=torch.bfloat16).to(device) -def load_clip(name,quantized_mode,device= "cuda") -> HFEmbedder: - return HFEmbedder(f"{name}/text_encoder",f"{name}/tokenizer",is_clip=True,quantized_mode=quantized_mode,max_length=77, torch_dtype=torch.bfloat16).to(device) +def load_clip(name,clip_cf,quantized_mode,device= "cuda") -> HFEmbedder: + return HFEmbedder(f"{name}/text_encoder",f"{name}/tokenizer",clip_cf,is_clip=True,quantized_mode=quantized_mode,max_length=77, torch_dtype=torch.bfloat16).to(device) #return HFEmbedder("openai/clip-vit-large-patch14", max_length=77, torch_dtype=torch.bfloat16).to(device) -def load_ae(name: str, device: str = "cuda", hf_download: bool = True) -> AutoEncoder: - ckpt_path = os.path.join(name,"ae.safetensors") # ckpt_path = configs[name].ae_path +def load_ae(name: str,vae_cf, device: str = "cuda", hf_download: bool = True) -> AutoEncoder: + ckpt_path=vae_cf + #ckpt_path = os.path.join(name,"ae.safetensors") # ckpt_path = configs[name].ae_path - if "dev" in name: - name = "flux-dev" - else: - name = "flux-schnell" if ( not os.path.exists(ckpt_path) and configs[name].repo_id is not None diff --git a/README.md b/README.md index a5fbb0d..4e9c09a 100644 --- a/README.md +++ b/README.md @@ -1,37 +1,38 @@ -# ComfyUI_StoryDiffusion -You can using StoryDiffusion in ComfyUI. +
+

ComfyUI_StoryDiffusion：You can using StoryDiffusion in ComfyUI.

* [中文说明](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/README-CN.md) * StoryDiffusion origin From: [link](https://github.com/HVision-NKU/StoryDiffusion) ---&--- MS-Diffusion origin From: [link](https://github.com/MS-Diffusion/MS-Diffusion)---&---StoryMakerr from From:[StoryMaker](https://github.com/RedAIGC/StoryMaker) - ## Updates: -**2024/09/24** -* Now if using Kolor's "ip-adapter" or "face ID", you can choose the monolithic model of clip_vision (such as :"clip-vit-large-patch14.safetensors") to load the image encoder. The change this brings is that Kolor's local directory can delete all files in the "clip-vit-large-patch14-336" and "Kolors-IP-Adapter-Plus" folders. Of course, because comfyUI defaults to clip image processing of "224" size , while Kolor defaults to 336 size, there will be a loss of accuracy and quality. Please refer to the image comparison in readme for details. -* Another change is that we now need to port the model of "ip_adapter_plus_general.bin" in "kolor-ipadapter" to the "comfyUI/models/photomaker" directory; +**2024/09/27** +* make easy and comfy -**Previous updates：** -* For the convenience of use, the layout of the node has been adjusted again,delete "id number"(base character lines now),delete " model_type"(base input image [img2img function] or not [txt2img function] now), 'clip_vision' , 'controlnet' , 'character prompt' ,'image','control_image' now in model loader node now. -* Now,using story_maker or pulid_flux function ,need choice clip vision model (use 'clip_vision' menu); -* add StoryMaker from From: [StoryMaker](https://github.com/RedAIGC/StoryMaker) to make dual role..or normal img2img,as detailed in the following text 3.5 ,fill "maker,dual" in easy function to using StoryMaker for dual role; -* Add " PulID FLUX " function, In my testing, a minimum of 12GB of VRAM can run normally,now you can fill "X:/xxx/xxx/black-forest-labs/FLUX.1-dev",and fill easy function "pilid,fp8,cpu"(if you Vram>30G,can remove cpu,and using Kijai/flux-fp8,if you Vram>45G,can remove fp8,cpu), although it is a bit slow if using cpu! ,Of course, some models need to be prepared, as detailed in the following text; -* Add kolor FaceId function, now you can fill "xxx:/xxx/xxx/Kwai-Kolors/Kolors",and fill easy function "face",Of course, some models need to be prepared, as detailed in the following text; -* Add diffusers'img2img codes( Not commit diffusers yet),Now you can using flux img2img function. in flux img2img,"guidance_scale" is usually 3.5 ,you can change ip-adapter_strength's number to Control the noise of the output image, the closer the number is to 1, the less it looks like the original image, and the closer it is to 0, the more it looks like the original image. Correspondingly, your generated step count is a multiple of this value, which means that if you enter 50 steps, the actual number of steps taken is 50 * 0.8 (0.8 is the value of change ip-adapter_strength) #you can see exmaple img -* AWPortrait-FL-fp8.safetensors is support if using fp8 mode,.. -* using img crop to fix ms_diffusion only using square's error; -* change W and H global names,it cause some error; -* fix runway error,when loader single model. -* The loading speed of the NF4 model is many times faster than FP8, so I recommend using the NF4 model to run Flux. I have included the workflow of NF4 in the example， -* Add an "easy_function" for debugging new function. This time, I have added support for "auraface" in "Photomake V2". You can enter "auraface" into the "easy_function" to test this method -* support Flux ,1.only using fp8 repo,fill local "X:/xxx/xxx/black-forest-labs/FLUX.1-dev" and ckpt_name="none";if save .pt,fill easy function "save",2. using unet,fill local repo and choice a fp8 Unet(like Kijai/flux-fp8, AWPortrait-FL-fp8.safetensors. *.pt), 3. using fn4,need download weights at [link](https://huggingface.co/sayakpaul/flux.1-dev-nf4/tree/main),put weight in "comfyui/models/checkpoints/";fill local "X:/xxx/xxx/black-forest-labs/FLUX.1-dev" -* Now clip checkpoints no need diffusers_repo,you can using "clip_g.safetensors" or other base from "CLIP-ViT-bigG-14-laion2B-39B-b160k"; -* 2 role in 1 img now using [A]...[B]... mode, -* Support "kolors" text2img and "kolors"ipadapter img2img,using repo like :"xxx:/xxx/xxx/Kwai-Kolors/Kolors" (Please refer to the end of the article for detailed file combinations) -* support photomaker V2; -* ControlNet now uses community models. -* The base model now has only two options: using repo input or selecting the community model... -* Introducing Controlnet for dual character co image, supporting multi image introduction, -* Add the function of saving and loading character models +## Function introduction +**story-diffusion** +* Support img2img & txt2img，All you need to do is select an SDXL model in“ckpt_name”menu to get started, support sdxl lora; +* If using img2img,You can choose between the v1 or v2 version of PhotoMaker checkpoints(Automatic downloads); phtomaker v2 need insightface and the models that go with it; + +**ms-diffusion** +* Enabled when 2 characters are required to appear in the same image, When [roleA] and [roleB] appear in a scene prompt at the same time, it will be automatically enabled; +* Of course the "ms_adapter.bin" model is also required, and the "clip_vision_g.safetensors"I is selected in the clip_vision; +* MS supports ControlNet, and a dual-role prompt requires a preprocessed ControlNet picture. + +**story-maker** +* "story-maker“ is similar to "story-diffusion", currently only supports img2img, which features the ability to migrate the character's clothing and supports dual characters with the same picture; +* To turn on this function, you need to enter 'maker' in easy-function; Then select an sdxl model and select the "clip_vision_H.safetensors" model in the clip-vision,The companion “mask.bin” model and“insightface"model are automatically downloaded; +* If you enter 'maker,dual', the function of using 'story-diffusion' in the front section and using 'story-maker' in the same picture for both characters will be enabled +* The method requires a mask, so “RMBG-1.4” is built-in, which can be downloaded automatically； + +**kolor** +* With kolor, you need to enter the local path of kolor in the repo_id, using '/' splitting the directory;You can use "clip-vit-large-patch14.safetensors", or the image encoder in the repo; +* Kolor supports img2img (IPadapeter and FaceID), txt2img,The matching model will be automatically downloaded, and the details can be found in the README model content; +* Kolor supports prompt input in all Chinese characters, note that the character name needs to be changed to ['张三']； +* using kolor FaceId function, need fill easy function "face", + +**Flux and PULID-FLUX** +* Flux supports img2img and txt2img, and supports FP8 and NF4 (recommended) quantization models；To enable it, enter the local path of flux diffuser in 'repo_id' and select the corresponding model in 'ckpt-name';example fill "X:/xxx/xxx/black-forest-labs/FLUX.1-dev"; +* PULID-FLUX needs to connect to the dual clip nodes of comfy in clip, and select 'EVA02_CLIP_L_336_psz14_s6B.ptin ' clip-vision, 'ae.safetensors' in vae, fill in 'pulid, fp8, cpu' in 'easy-function' , if your VRAM is about 24G, you can try to remove the cpu;The accompanying 'insightface' model will be automatically downloaded; 1.Installation ----- @@ -45,7 +46,7 @@ git clone https://github.com/smthemex/ComfyUI_StoryDiffusion.git ``` pip install -r requirements.txt ``` -if using photomakerV2,need: +if using photomaker V2，pulid-flux，kolor，story-make: ``` pip install insightface==0.7.3 or new ``` @@ -214,7 +215,7 @@ downlaod nf4 model [link](https://huggingface.co/sayakpaul/flux.1-dev-nf4/tree/ | ├── rename nf4 ckpt ``` -**3.3.3.4 using flux pulid,repo_id+ckpt_name** . +**3.3.3.4 using flux pulid,clip+ckpt_name** . torch must > 0.24.0 optimum-quanto must >=0.2.4 ``` @@ -225,6 +226,7 @@ DIAMONIK7777/antelopev2 auto downlaod....[https://huggingface.co/DIAMONIK7777/an "pulid_flux_v0.9.0.safetensors" download from [link](https://huggingface.co/guozinan/PuLID/tree/main) fp8 using flux1-dev-fp8.safetensors from [link](https://huggingface.co/Kijai/flux-fp8/tree/main) make sure ae.safetensors in you FLUX.1-dev dir,example: + ``` ├── ComfyUI/models/photomaker/ | ├── pulid_flux_v0.9.0.safetensors @@ -232,6 +234,9 @@ make sure ae.safetensors in you FLUX.1-dev dir,example: | ├── EVA02_CLIP_L_336_psz14_s6B.pt ├── ComfyUI/models/checkpoints/ | ├── flux1-dev-fp8.safetensors +├── ComfyUI/models/clip/ +| ├── t5xxl_fp8_e4m3fn.safetensors +| ├── clip_l.safetensors ``` **3.5 if using storymake..** @@ -252,51 +257,36 @@ RMBG-1.4 from [link](https://huggingface.co/briaai/RMBG-1.4/tree/main)#自动 ``` 4 Example ---- -img2img mode only use storymaker 只用storymaker生成(包括双角色),最新 (Latest version) -![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/using_storymake_dual.png) +**story-make** +img2img 纯storymaker生成，非最新示例 (outdated version examples) +![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/maker2role.png) -img2img mode use storymaker for dual role only,只用storymaker生成双角色,单角色用diffusion生成,最新(Latest version) -![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/using_storymake_dual_only.png) +**flux-pulid** +img2img mode use flux pulid 12G Vram,cpu Flux使用PULID功能,最新示例(Latest version) +![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/flux.png) -img2img mode use flux pulid 12G Vram,cpu Flux使用PULID功能,最新(Latest version) -![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/pulid_flux.png) +**kolor-face** +img2img kolor face，参数输入没变化，非最新示例 (outdated version examples) +![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/kolor.png) -img2img mode use nf4 flux 开启flux nf4模式,速度最快,示例图为旧节点(outdated version examples) -![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/flux_img2img.png) -![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/flux_img2img2role.png) - -txt2img mode use NF4 FLUX 开启flux nf4模式,速度最快,示例图为旧节点 (outdated version examples) +**flux-nf4** +* txt2img mode use NF4 FLUX 开启flux nf4模式,速度最快，非最新示例 (outdated version examples) ![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/nf4.png) +* flux img2img 图生图，注意用ip的参数来控制噪声（多样性），非最新示例 (outdated version examples) +![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/flux_img2img.png) -img2img mode use auraface photomake V2 开启v2模式,示例图为旧节点 (outdated version examples) -![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/newest.png) - -img2img mode kolors ip-adapter using monolithic model 可图使用单体模型来解码图片 最新 (Latest version) -![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/kolor_using_comfyclip.png) - -img2img model use kolors ip adapter,and using chinese prompt 开启kolor模式,使用中文提示词,示例图为旧节点 (outdated version examples) -![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/kolor_ipadapter_use_chinese.png) -![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/kolor_ipadapter.png) - -img2img model use kolors ip adapter face id (outdated version examples) -![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/kolor_faceid.png) - -img2img sdxl mode, uses photomakeV1 (outdated version examples) -![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/photomakev1.png) - -img2img sdxl mode, uses photomakeV2 (outdated version examples) -![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/photomakev2.png) - -txt2img using lora and comic node (outdated version examples) -![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/txt2img_lora_comic.png) - -img2img2role in 1 image (Outdated version examples) +**ms-diffusion** +* img2img2role in 1 image，双角色同图，非最新示例 (Outdated version examples) ![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/2rolein1img.png) - -ControlNet added dual role co frame (Role 1 and Role 2) (Outdated version examples) +* ControlNet added dual role co frame (Role 1 and Role 2) (Outdated version examples) ![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/controlnet.png) -Translate the text into other language examples, and the translation nodes in the diagram can be replaced with any translation node. Outdated version examples +**story-diffusion** +* photomake v2 in img2img normal 最基础的story流程，非最新示例 (outdated version examples) +![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/phtomakev2.png) +* txt2img using lora and comic node (outdated version examples) +![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/txt2img_lora_comic.png) +* Translate the text into other language examples, and the translation nodes in the diagram can be replaced with any translation node. Outdated version examples ![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/trans1.png) Function Description of Nodes @@ -304,6 +294,7 @@ Function Description of Nodes **** * image: optional,The interface that must be linked to the image generation diagram. For dual roles, please follow the example and use the built-in image batch node in comfyUI; * controlnet:only ms function support; +* clip:Connect the two clip nodes of Comfyui（using flux-pulid） * character prompt: The prompt for the character, [character name] must be at the beginning. If using the graphic mode, the keyword "img" must be added, such as a man img;if using chinese prompt, need["角色名"] or ['角色名'] * repo_id: using diffuser models ; * ckpt_name: using community SDLX model selection; @@ -323,17 +314,16 @@ Function Description of Nodes **** * model: The interface that must be linked; * Scene prompts: The prompt for the scene description, [character name], must start at the beginning. It is best for both characters to appear once in the first two lines. [NC] At the beginning, the character does not appear (suitable for non character scenes). When [character A] and [character B], MS diffusion's dual character mode is enabled, and and the spaces before and after it cannot be ignored# Used for segmented prompt, rendering the entire segment, but only outputting the prompt after #; -* split prompt: The symbol for splitting the prompt, which does not take effect when it is empty. It is used to normalize paragraphs when the prompt is external. For example, when you pass in 10 lines of text, the hyphen may not be correct, but using a hyphen, such as ";", can effectively distinguish each line. * negative prompt: only effective when img_style is No_style; * seed/steps/cfg: suitable for commonly used functions in comfyUI; * ip-adapter_strength: img2img controls the weight of ip-adapter in graph generation,only using in kolors; * style_strength'ratio: Style weight control, which controls from which step the style takes effect. When the style consistency is not good, you can try increasing or decreasing this parameter; * role-scale: only effective when two characters are in the same image, controlling the weight of the characters in the image; -* Mask_threshold: It is only effective when two roles are in the same picture, and controls the position of the role in the picture (MS system automatically assigns the role position according to prompt, so appropriate role position information description can be added to prompt); +* Mask_threshold: It is only effective when two roles are in the same picture, and controls the position of the role in the picture (MS system automatically assigns the role position according to prompt, so appropriate role position information description can be added to prompt)(ms-diffusion only); * Start_step: Only effective when two characters are in the same image, controlling the number of starting steps for the character's position in the image * Save_character: Whether to save the character weights of the current character, file in/ Under ComfyUI_StoryDiffusion/weights/pt, use time as the file name; -* Controllet_scale: control net weight; -* guidance_list: contrlol role's position; +* Controllet_scale: control net weight,(ms-diffusion only); +* guidance_list: contrlol role's position(ms-diffusion only); **** * Fonts list: The puzzle node supports custom fonts (place the font file in the fonts directory. fonts/you_font. ttf); @@ -350,11 +340,38 @@ Function Description of Nodes * Playground v2.5 can be effective on txt2img, and there is no Playground v2.5 style Lora available when accelerated Lora can be used; * Role-scale, mask_threshold, and start_step mainly regulate the randomness and style consistency of two characters in the same frame; * The consistency of style can be adjusted between ip-adapter_strength and style_strength'ratio in img2img; -* Preprocess translation text nodes, please refer to the example diagram for usage methods. (Pay attention to changing the font for Chinese or other East Asian characters); * By default, use the ";" at the end of each paragraph to divide the paragraph. After translation into Chinese, there is a chance that it will be translated as ";", so remember to change it to ";", otherwise it will be a sentence. -* The process of generating images using PhotosMaker requires the IMG keyword in the character prompt column. You can use keywords such as a woman IMG, a man IMG, etc; +* Preprocess translation text nodes, please refer to the example diagram for usage methods. (Pay attention to changing the font for Chinese or other East Asian characters); +* By default, use the ";" at the end of each paragraph to divide the paragraph. After translation into Chinese, there is a chance that it will be translated as ";", so remember to change it to ";", otherwise it will be a sentence. +* The process of generating images using PhotosMaker requires the "img" keyword in the character prompt column. You can use keywords such as a woman "img" , a man img , etc; * No characters appear in the image, add [NC] in front of the scene prompt; * Segmented prompt, using #, such as AAAA # BBBB, will generate AAAA content, but the text will only display BBBB +**Previous updates** +* Now if using Kolor's "ip-adapter" or "face ID", you can choose the monolithic model of clip_vision (such as :"clip-vit-large-patch14.safetensors") to load the image encoder. The change this brings is that Kolor's local directory can delete all files in the "clip-vit-large-patch14-336" and "Kolors-IP-Adapter-Plus" folders. Of course, because comfyUI defaults to clip image processing of "224" size , while Kolor defaults to 336 size, there will be a loss of accuracy and quality. Please refer to the image comparison in readme for details. +* Another change is that we now need to port the model of "ip_adapter_plus_general.bin" in "kolor-ipadapter" to the "comfyUI/models/photomaker" directory; +* For the convenience of use, the layout of the node has been adjusted again,delete "id number"(base character lines now),delete " model_type"(base input image [img2img function] or not [txt2img function] now), 'clip_vision' , 'controlnet' , 'character prompt' ,'image','control_image' now in model loader node now. +* Now,using story_maker or pulid_flux function ,need choice clip vision model (use 'clip_vision' menu); +* add StoryMaker from From: [StoryMaker](https://github.com/RedAIGC/StoryMaker) to make dual role..or normal img2img,as detailed in the following text 3.5 ,fill "maker,dual" in easy function to using StoryMaker for dual role; +* Add " PulID FLUX " function, In my testing, a minimum of 12GB of VRAM can run normally,now you can fill "X:/xxx/xxx/black-forest-labs/FLUX.1-dev",and fill easy function "pilid,fp8,cpu"(if you Vram>30G,can remove cpu,and using Kijai/flux-fp8,if you Vram>45G,can remove fp8,cpu), although it is a bit slow if using cpu! ,Of course, some models need to be prepared, as detailed in the following text; +* Add kolor FaceId function, now you can fill "xxx:/xxx/xxx/Kwai-Kolors/Kolors",and fill easy function "face",Of course, some models need to be prepared, as detailed in the following text; +* Add diffusers'img2img codes( Not commit diffusers yet),Now you can using flux img2img function. in flux img2img,"guidance_scale" is usually 3.5 ,you can change ip-adapter_strength's number to Control the noise of the output image, the closer the number is to 1, the less it looks like the original image, and the closer it is to 0, the more it looks like the original image. Correspondingly, your generated step count is a multiple of this value, which means that if you enter 50 steps, the actual number of steps taken is 50 * 0.8 (0.8 is the value of change ip-adapter_strength) #you can see exmaple img +* AWPortrait-FL-fp8.safetensors is support if using fp8 mode,.. +* using img crop to fix ms_diffusion only using square's error; +* change W and H global names,it cause some error; +* fix runway error,when loader single model. +* The loading speed of the NF4 model is many times faster than FP8, so I recommend using the NF4 model to run Flux. I have included the workflow of NF4 in the example， +* Add an "easy_function" for debugging new function. This time, I have added support for "auraface" in "Photomake V2". You can enter "auraface" into the "easy_function" to test this method +* support Flux ,1.only using fp8 repo,fill local "X:/xxx/xxx/black-forest-labs/FLUX.1-dev" and ckpt_name="none";if save .pt,fill easy function "save",2. using unet,fill local repo and choice a fp8 Unet(like Kijai/flux-fp8, AWPortrait-FL-fp8.safetensors. *.pt), 3. using fn4,need download weights at [link](https://huggingface.co/sayakpaul/flux.1-dev-nf4/tree/main),put weight in "comfyui/models/checkpoints/";fill local "X:/xxx/xxx/black-forest-labs/FLUX.1-dev" +* Now clip checkpoints no need diffusers_repo,you can using "clip_g.safetensors" or other base from "CLIP-ViT-bigG-14-laion2B-39B-b160k"; +* 2 role in 1 img now using [A]...[B]... mode, +* Support "kolors" text2img and "kolors"ipadapter img2img,using repo like :"xxx:/xxx/xxx/Kwai-Kolors/Kolors" (Please refer to the end of the article for detailed file combinations) +* support photomaker V2; +* ControlNet now uses community models. +* The base model now has only two options: using repo input or selecting the community model... +* Introducing Controlnet for dual character co image, supporting multi image introduction, +* Add the function of saving and loading character models + + Citation ------ diff --git a/StoryMaker/ip_adapter/attention_processor.py b/StoryMaker/ip_adapter_/attention_processor.py similarity index 100% rename from StoryMaker/ip_adapter/attention_processor.py rename to StoryMaker/ip_adapter_/attention_processor.py diff --git a/StoryMaker/ip_adapter/attention_processor_faceid.py b/StoryMaker/ip_adapter_/attention_processor_faceid.py similarity index 100% rename from StoryMaker/ip_adapter/attention_processor_faceid.py rename to StoryMaker/ip_adapter_/attention_processor_faceid.py diff --git a/StoryMaker/ip_adapter/ip_adapter.py b/StoryMaker/ip_adapter_/ip_adapter.py similarity index 99% rename from StoryMaker/ip_adapter/ip_adapter.py rename to StoryMaker/ip_adapter_/ip_adapter.py index 0033268..6c08a5c 100644 --- a/StoryMaker/ip_adapter/ip_adapter.py +++ b/StoryMaker/ip_adapter_/ip_adapter.py @@ -12,7 +12,7 @@ if is_torch2_available(): from .attention_processor import IPAttnProcessor2_0 as IPAttnProcessor, AttnProcessor2_0 as AttnProcessor, CNAttnProcessor2_0 as CNAttnProcessor else: from .attention_processor import IPAttnProcessor, AttnProcessor, CNAttnProcessor -from .resampler import Resampler +from .resampler_ import Resampler class ImageProjModel(torch.nn.Module): diff --git a/StoryMaker/ip_adapter/ip_adapter_faceid.py b/StoryMaker/ip_adapter_/ip_adapter_faceid.py similarity index 99% rename from StoryMaker/ip_adapter/ip_adapter_faceid.py rename to StoryMaker/ip_adapter_/ip_adapter_faceid.py index ea1f489..79d57dd 100644 --- a/StoryMaker/ip_adapter/ip_adapter_faceid.py +++ b/StoryMaker/ip_adapter_/ip_adapter_faceid.py @@ -21,7 +21,7 @@ if is_torch2_available() and (not USE_DAFAULT_ATTN): ) else: from .attention_processor_faceid import LoRAAttnProcessor, LoRAIPAttnProcessor -from .resampler import PerceiverAttention, FeedForward, Resampler +from .resampler_ import PerceiverAttention, FeedForward, Resampler class FacePerceiverResampler(torch.nn.Module): def __init__( diff --git a/StoryMaker/ip_adapter/resampler.py b/StoryMaker/ip_adapter_/resampler_.py similarity index 100% rename from StoryMaker/ip_adapter/resampler.py rename to StoryMaker/ip_adapter_/resampler_.py diff --git a/StoryMaker/ip_adapter/utils.py b/StoryMaker/ip_adapter_/utils.py similarity index 100% rename from StoryMaker/ip_adapter/utils.py rename to StoryMaker/ip_adapter_/utils.py diff --git a/StoryMaker/pipeline_sdxl_storymaker.py b/StoryMaker/pipeline_sdxl_storymaker.py index 7f8c9f4..a039ce5 100644 --- a/StoryMaker/pipeline_sdxl_storymaker.py +++ b/StoryMaker/pipeline_sdxl_storymaker.py @@ -48,12 +48,12 @@ from diffusers.utils.import_utils import is_xformers_available from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection from insightface.utils import face_align -from .ip_adapter.resampler import Resampler -from .ip_adapter.utils import is_torch2_available -from .ip_adapter.ip_adapter_faceid import faceid_plus +from .ip_adapter_.resampler_ import Resampler +from .ip_adapter_.utils import is_torch2_available +from .ip_adapter_.ip_adapter_faceid import faceid_plus -from .ip_adapter.attention_processor import IPAttnProcessor2_0 as IPAttnProcessor, AttnProcessor2_0 as AttnProcessor -from .ip_adapter.attention_processor_faceid import LoRAIPAttnProcessor2_0 as LoRAIPAttnProcessor, LoRAAttnProcessor2_0 as LoRAAttnProcessor +from .ip_adapter_.attention_processor import IPAttnProcessor2_0 as IPAttnProcessor, AttnProcessor2_0 as AttnProcessor +from .ip_adapter_.attention_processor_faceid import LoRAIPAttnProcessor2_0 as LoRAIPAttnProcessor, LoRAAttnProcessor2_0 as LoRAAttnProcessor logger = logging.get_logger(__name__) # pylint: disable=invalid-name @@ -216,8 +216,7 @@ class StableDiffusionXLStoryMakerPipeline(StableDiffusionXLPipeline): clip_img = self.clip_image_processor(images=ref_img, return_tensors="pt").pixel_values return clip_img, clip_face, torch.from_numpy(face_info.normed_embedding).unsqueeze(0) - def _encode_prompt_image_emb(self, image, image_2, mask_image, mask_image_2, face_info, face_info_2, cloth, cloth_2, \ - device, num_images_per_prompt, dtype, do_classifier_free_guidance): + def _encode_prompt_image_emb(self, image, image_2, mask_image, mask_image_2, face_info, face_info_2, cloth, cloth_2,device, num_images_per_prompt, dtype, do_classifier_free_guidance): crop_list = []; face_list = []; id_list = [] if image is not None: clip_img, clip_face, face_emb = self.crop_image(image, mask_image, face_info) @@ -252,9 +251,12 @@ class StableDiffusionXLStoryMakerPipeline(StableDiffusionXLPipeline): if do_classifier_free_guidance: prompt_image_emb = self.image_proj_model(id_embeds, clip_image_embeds, clip_face_embeds) B, C, D = prompt_image_emb.shape - prompt_image_emb = prompt_image_emb.view(1, B*C, D) + #prompt_image_emb = prompt_image_emb.view(1, B*C, D) + prompt_image_emb = prompt_image_emb.view(1, B * C, D) + neg_emb = self.image_proj_model(torch.zeros_like(id_embeds), torch.zeros_like(clip_image_embeds), torch.zeros_like(clip_face_embeds)) - neg_emb = neg_emb.view(1, B*C, D) + #neg_emb = neg_emb.view(1, B*C, D) + neg_emb = neg_emb.view(1, B * C, D) prompt_image_emb = torch.cat([neg_emb, prompt_image_emb], dim=0) #torch.Size([2, 40, 2048]) #print(prompt_image_emb.shape) else: @@ -264,8 +266,9 @@ class StableDiffusionXLStoryMakerPipeline(StableDiffusionXLPipeline): #print(f'prompt_image_emb: {prompt_image_emb.shape}')#torch.Size([2, 40, 2048]) bs_embed, seq_len, _ = prompt_image_emb.shape - prompt_image_emb = prompt_image_emb.repeat(1, num_images_per_prompt, 1) + + #prompt_image_emb = prompt_image_emb.repeat(1, num_images_per_prompt, 1) prompt_image_emb = prompt_image_emb.view(bs_embed * num_images_per_prompt, seq_len, -1) return prompt_image_emb.to(device=device, dtype=dtype) @@ -498,7 +501,7 @@ class StableDiffusionXLStoryMakerPipeline(StableDiffusionXLPipeline): batch_size = len(prompt) else: batch_size = prompt_embeds.shape[0] - #print(batch_size) + #print(f"batch_size is {batch_size}") device = self.unet.device # pdb.set_trace() # 3.1 Encode input prompt @@ -598,8 +601,8 @@ class StableDiffusionXLStoryMakerPipeline(StableDiffusionXLPipeline): prompt_embeds = prompt_embeds.to(device) add_text_embeds = add_text_embeds.to(device) add_time_ids = add_time_ids.to(device).repeat(batch_size * num_images_per_prompt, 1) - # print(prompt_embeds.shape, prompt_image_emb.shape) torch.Size([4, 77, 2048])? torch.Size([2, 40, 2048]) - prompt_image_emb=torch.cat((prompt_image_emb,) * batch_size, dim=0) + #print(prompt_embeds.shape,"and", prompt_image_emb.shape) # torch.Size([4, 77, 2048]) ? torch.Size([2, 40, 2048]) + #prompt_image_emb = torch.cat([prompt_image_emb] * batch_size, dim=0) #print(prompt_embeds.shape, prompt_image_emb.shape) encoder_hidden_states = torch.cat([prompt_embeds, prompt_image_emb], dim=1) @@ -696,4 +699,4 @@ class StableDiffusionXLStoryMakerPipeline(StableDiffusionXLPipeline): if not return_dict: return (image,) - return StableDiffusionXLPipelineOutput(images=image) + return StableDiffusionXLPipelineOutput(images=image) \ No newline at end of file diff --git a/Storydiffusion_node.py b/Storydiffusion_node.py index 9ff7435..2cd2d46 100644 --- a/Storydiffusion_node.py +++ b/Storydiffusion_node.py @@ -16,6 +16,7 @@ from PIL import ImageFont from safetensors.torch import load_file +import ComfyUI.comfy.model_sampling from .PuLID.pulid.utils import resize_numpy_image_long from .ip_adapter.attention_processor import IPAttnProcessor2_0 import sys @@ -57,6 +58,7 @@ import folder_paths from comfy.utils import common_upscale from comfy.model_management import cleanup_models from comfy.clip_vision import load as clip_load +from comfy.model_management import total_vram global total_count, attn_count, cur_step, mask1024, mask4096, attn_procs, unet global sa32, sa64 @@ -691,16 +693,17 @@ def process_generation( height, load_chars, lora, - trigger_words,photomake_mode,use_kolor,use_flux,make_dual_only,kolor_face,pulid,story_maker,app_face, pipeline_mask,role_scale + trigger_words,photomake_mode,use_kolor,use_flux,make_dual_only,kolor_face,pulid,story_maker, + input_id_emb_s_dict, input_id_img_s_dict,input_id_emb_un_dict, input_id_cloth_dict,role_scale,control_img,empty_emb_zero,miX_mode ): # Corrected font_choice usage if len(general_prompt.splitlines()) >= 3: raise "Support for more than three characters is temporarily unavailable due to VRAM limitations, but this issue will be resolved soon." # _model_type = "Photomaker" if _model_type == "Using Ref Images" else "original" - if not use_kolor and not use_flux: + if not use_kolor and not use_flux and not story_maker: if model_type == "img2img" and "img" not in general_prompt: - raise 'Please choice img2img typle,and add the triger word " img " behind the class word you want to customize, such as: man img or woman img' + raise 'if using normal SDXL img2img ,need add the triger word " img " behind the class word you want to customize, such as: man img or woman img' global total_length, attn_procs, cur_model_type global write @@ -709,11 +712,12 @@ def process_generation( #load_chars = load_character_files_on_running(unet, character_files=char_files) prompts_origin = prompt_array.splitlines() + prompts_origin=[i.strip() for i in prompts_origin] + prompts_origin=[i for i in prompts_origin if '[' in i] #删除空行 + #print(prompts_origin) prompts = [prompt for prompt in prompts_origin if not len(extract_content_from_brackets(prompt))>=2] # 剔除双角色 - if use_kolor: - add_trigger_words = "," + trigger_words + " " + "风格" + ";" - else: - add_trigger_words = "," + trigger_words + " " + "style" + ";" + + add_trigger_words = " " + trigger_words + " style " if lora : if lora in lora_lightning_list: prompts = remove_punctuation_from_strings(prompts) @@ -757,7 +761,9 @@ def process_generation( prompts = [ prompt.rpartition("#")[0] if "#" in prompt else prompt for prompt in prompts ] - + img_mode=False + if kolor_face or pulid or (story_maker and not make_dual_only) or model_type=="img2img": + img_mode=True # id_prompts = prompts[:id_length] ( character_index_dict, @@ -765,15 +771,19 @@ def process_generation( replace_prompts, ref_indexs_dict, ref_totals, - ) = process_original_prompt(character_dict, prompts, id_length) - + ) = process_original_prompt(character_dict, prompts, id_length,img_mode) + #print(character_index_dict,invert_character_index_dict,replace_prompts,ref_indexs_dict,ref_totals) + #character_index_dict：{'[Taylor]': [0, 3], '[sam]': [1, 2]},if 1 role {'[Taylor]': [0, 1, 2]} + #invert_character_index_dict:{0: ['[Taylor]'], 1: ['[sam]'], 2: ['[sam]'], 3: ['[Taylor]']},if 1 role {0: ['[Taylor]'], 1: ['[Taylor]'], 2: ['[Taylor]']} + #ref_indexs_dict:{'[Taylor]': [0, 3], '[sam]': [1, 2]},if 1 role {'[Taylor]': [0]} + #ref_totals: [0, 3, 1, 2] if 1 role [0] if model_type == "img2img": # _upload_images = [_upload_images] input_id_images_dict = {} if len(upload_images) != len(character_dict.keys()): raise f"You upload images({len(upload_images)}) is not equal to the number of characters({len(character_dict.keys())})!" for ind, img in enumerate(upload_images): - input_id_images_dict[character_list[ind]] = [img] # 已经pil转化了 不用load + input_id_images_dict[character_list[ind]] = [img] # 已经pil转化了 不用load {a:[img],b:[img]} # input_id_images_dict[character_list[ind]] = [load_image(img)] # real_prompts = prompts[id_length:] # if device == "cuda": @@ -781,8 +791,6 @@ def process_generation( write = True cur_step = 0 attn_count = 0 - #id_prompts_no_using, negative_prompt = apply_style(style_name, ["id_prompts"], negative_prompt) - #setup_seed(seed_) total_results = [] id_images = [] results_dict = {} @@ -790,17 +798,19 @@ def process_generation( global cur_character if not load_chars: - for character_key in character_dict.keys():# 先生成角色对应第一句场景提示词的图片 + for character_key in character_dict.keys():# 先生成角色对应第一句场景提示词的图片,图生图是批次生成 + character_key_str=character_key cur_character = [character_key] ref_indexs = ref_indexs_dict[character_key] current_prompts = [replace_prompts[ref_ind] for ref_ind in ref_indexs] - setup_seed(seed_) + if model_type == "txt2img": + setup_seed(seed_) generator = torch.Generator(device=device).manual_seed(seed_) cur_step = 0 cur_positive_prompts, cur_negative_prompt = apply_style( style_name, current_prompts, negative_prompt ) - print(f"sampler:{cur_positive_prompts}") + print(f"Sampler {character_key_str} 's cur_positive_prompts :{cur_positive_prompts}") if model_type == "txt2img": if use_flux: id_images = pipe( @@ -834,27 +844,38 @@ def process_generation( cur_negative_prompt = cur_negative_prompt * len(cur_positive_prompts) if len( cur_negative_prompt) != len(cur_positive_prompts) else cur_negative_prompt if kolor_face: - img_=input_id_images_dict[character_key][0] - face_info = app_face.get_faceinfo_one_img(img_) - face_bbox_square = face_bbox_to_square(face_info["bbox"]) - crop_image = img_.crop(face_bbox_square) - crop_image = crop_image.resize((336, 336)) - crop_image = [crop_image]*len(cur_positive_prompts) - face_embeds = torch.from_numpy(np.array([face_info["embedding"]])) - face_embeds=torch.cat((face_embeds,)*len(cur_positive_prompts),dim=0) + crop_image=input_id_img_s_dict[character_key_str][0] + face_embeds=input_id_emb_s_dict[character_key_str][0] face_embeds = face_embeds.to(device, dtype=torch.float16) - id_images = pipe( - prompt=cur_positive_prompts, - negative_prompt=cur_negative_prompt, - height=height, - width=width, - num_inference_steps=_num_steps, - guidance_scale=guidance_scale, - num_images_per_prompt=1, - generator= generator, - face_crop_image=crop_image, - face_insightface_embeds=face_embeds, - ).images + if id_length > 1: + id_images=[] + for index, i in enumerate(cur_positive_prompts): + id_image = pipe( + prompt=i, + negative_prompt=cur_negative_prompt[index], + height=height, + width=width, + num_inference_steps=_num_steps, + guidance_scale=guidance_scale, + num_images_per_prompt=1, + generator=generator, + face_crop_image=crop_image, + face_insightface_embeds=face_embeds, + ).images + id_images.append(id_image) + else: + id_images = pipe( + prompt=cur_positive_prompts, + negative_prompt=cur_negative_prompt, + height=height, + width=width, + num_inference_steps=_num_steps, + guidance_scale=guidance_scale, + num_images_per_prompt=1, + generator=generator, + face_crop_image=crop_image, + face_insightface_embeds=face_embeds, + ).images else: pipe.set_ip_adapter_scale([_Ip_Adapter_Strength]) id_images = pipe( @@ -870,24 +891,42 @@ def process_generation( ).images elif use_flux: if pulid: - id_image = resize_numpy_image_long(input_id_images_dict[character_key][0], 1024) - use_true_cfg = abs(role_scale - 1.0) > 1e-2 - id_embeddings, uncond_id_embeddings = pipe.pulid_model.get_id_embedding(id_image,cal_uncond=use_true_cfg) + id_embeddings=input_id_emb_s_dict[character_key_str][0] + uncond_id_embeddings=input_id_emb_un_dict[character_key_str][0] strength = _Ip_Adapter_Strength - id_images= pipe.generate_image( - prompt=cur_positive_prompts, - seed=seed_, - start_step=2, - num_steps=_num_steps, - height=height, - width=width, - id_embeddings=id_embeddings, - uncond_id_embeddings=uncond_id_embeddings, - id_weight=strength, - guidance=guidance_scale, - true_cfg=role_scale, - max_sequence_length=128, - ) + if id_length > 1: + id_images = [] + for index, i in enumerate(cur_positive_prompts): + id_image = pipe.generate_image( + prompt=i, + seed=seed_, + start_step=2, + num_steps=_num_steps, + height=height, + width=width, + id_embeddings=id_embeddings, + uncond_id_embeddings=uncond_id_embeddings, + id_weight=strength, + guidance=guidance_scale, + true_cfg=role_scale, + max_sequence_length=128, + ) + id_images.append(id_image) + else: + id_images = pipe.generate_image( + prompt=cur_positive_prompts, + seed=seed_, + start_step=2, + num_steps=_num_steps, + height=height, + width=width, + id_embeddings=id_embeddings, + uncond_id_embeddings=uncond_id_embeddings, + id_weight=strength, + guidance=guidance_scale, + true_cfg=role_scale, + max_sequence_length=128, + ) else: strength = _Ip_Adapter_Strength if _Ip_Adapter_Strength != 1 else 0.9 id_images = pipe( @@ -903,42 +942,53 @@ def process_generation( guidance_scale=guidance_scale, generator=generator, ).images + elif story_maker and not make_dual_only: img = input_id_images_dict[character_key][0] - mask_image = pipeline_mask(img, return_mask=True) .convert('RGB')# outputs a pillow mask - print(type(mask_image)) - mask_image.save("asdad.png") - face_info = app_face.get(cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR)) - face_info = \ - sorted(face_info, key=lambda x: (x['bbox'][2] - x['bbox'][0]) * (x['bbox'][3] - x['bbox'][1]))[ - -1] # only use the maximum face + #print(character_key_str,input_id_images_dict) + mask_image = input_id_img_s_dict[character_key_str][0] + face_info=input_id_emb_s_dict[character_key_str][0] + cloth_info=None + if isinstance(control_img,torch.Tensor): + cloth_info=input_id_cloth_dict[character_key_str][0] cur_negative_prompt = [cur_negative_prompt] cur_negative_prompt = cur_negative_prompt * len(cur_positive_prompts) if len( cur_negative_prompt) != len(cur_positive_prompts) else cur_negative_prompt - id_images = pipe( - image=img, - mask_image=mask_image, - face_info=face_info, - prompt=cur_positive_prompts, - negative_prompt=cur_negative_prompt, - ip_adapter_scale=_Ip_Adapter_Strength, lora_scale=0.8, - num_inference_steps=_num_steps, - guidance_scale=guidance_scale, - height=height, width=width, - generator=generator, - ).images + if id_length>1: + id_images = [] + for index, i in enumerate(cur_positive_prompts): + id_image = pipe( + image=img, + mask_image=mask_image, + face_info=face_info, + prompt=i, + negative_prompt=cur_negative_prompt[index], + ip_adapter_scale=_Ip_Adapter_Strength, lora_scale=0.8, + num_inference_steps=_num_steps, + guidance_scale=guidance_scale, + height=height, width=width, + generator=generator, + cloth=cloth_info, + ).images + id_images.append(id_image) + else: + id_images = pipe( + image=img, + mask_image=mask_image, + face_info=face_info, + prompt=cur_positive_prompts, + negative_prompt=cur_negative_prompt, + ip_adapter_scale=_Ip_Adapter_Strength, lora_scale=0.8, + num_inference_steps=_num_steps, + guidance_scale=guidance_scale, + height=height, width=width, + generator=generator, + cloth=cloth_info, + ).images + else: if photomake_mode == "v2": - # 提取id - # print("v2 mode load_chars", cur_positive_prompts, negative_prompt, character_key) - img = input_id_images_dict[character_key][0] # input_id_images_dict {'[Taylor]': [pil], '[Lecun]': [pil]} - img = np.array(img) - img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) - from .utils.insightface_package import analyze_faces - faces = analyze_faces(app_face, img, ) - id_embed_list = [torch.from_numpy((faces[0]['embedding']))] - id_embeds = torch.stack(id_embed_list) - + id_embeds = input_id_emb_s_dict[character_key_str][0] id_images = pipe( cur_positive_prompts, input_id_images=input_id_images_dict[character_key], @@ -972,8 +1022,18 @@ def process_generation( p_num+=1 # total_results = id_images + total_results # yield total_results - for ind, img in enumerate(id_images): - results_dict[ref_indexs[ind]] = img + if story_maker and not make_dual_only and id_length>1 and model_type == "img2img": + for index,ind in enumerate(character_index_dict[character_key]): + results_dict[ref_totals[ind]] = id_images[index] + elif pulid and id_length>1 and model_type == "img2img": + for index,ind in enumerate(character_index_dict[character_key]): + results_dict[ref_totals[ind]] = id_images[index] + elif kolor_face and id_length>1 and model_type == "img2img": + for index,ind in enumerate(character_index_dict[character_key]): + results_dict[ref_totals[ind]] = id_images[index] + else: + for ind, img in enumerate(id_images): + results_dict[ref_indexs[ind]] = img # real_images = [] yield [results_dict[ind] for ind in results_dict.keys()] write = False @@ -984,21 +1044,22 @@ def process_generation( else: real_prompts_inds = [ind for ind in range(len(prompts))] - real_prompt_no, negative_prompt_style = apply_style_positive(style_name, "real_prompt") negative_prompt=str(negative_prompt)+str(negative_prompt_style) - - for real_prompts_ind in real_prompts_inds: # 非角色流程 + #print(f"real_prompts_inds is {real_prompts_inds}") + for real_prompts_ind in real_prompts_inds: # real_prompt = replace_prompts[real_prompts_ind] cur_character = get_ref_character(prompts[real_prompts_ind], character_dict) - #print(cur_character) - setup_seed(seed_) + + if model_type=="txt2img": + setup_seed(seed_) + generator = torch.Generator(device=device).manual_seed(seed_) + if len(cur_character) > 1 and model_type == "img2img": raise "Temporarily Not Support Multiple character in Ref Image Mode!" - generator = torch.Generator(device=device).manual_seed(seed_) cur_step = 0 real_prompt ,negative_prompt_style_no= apply_style_positive(style_name, real_prompt) - print(f"sampler:{real_prompt}") + print(f"Sample real_prompt : {real_prompt}") if model_type == "txt2img": # print(results_dict,real_prompts_ind) if use_flux: @@ -1027,15 +1088,9 @@ def process_generation( empty_img = Image.new('RGB', (height, width), (255, 255, 255)) if use_kolor: if kolor_face: - #print(real_prompt,negative_prompt) - img_1= input_id_images_dict[cur_character[0]] if real_prompts_ind not in nc_indexs else empty_img - #print(img_1) - face_info = app_face.get_faceinfo_one_img(img_1) - face_bbox_square = face_bbox_to_square(face_info["bbox"]) - crop_image = img_1.crop(face_bbox_square) - crop_image = crop_image.resize((336, 336)) - crop_image = [crop_image] - face_embeds = torch.from_numpy(np.array([face_info["embedding"]])) + empty_image = Image.new('RGB', (336, 336), (255, 255, 255)) + crop_image=input_id_img_s_dict[cur_character[0]] if real_prompts_ind not in nc_indexs else empty_image + face_embeds=input_id_emb_s_dict[cur_character[0]][0] if real_prompts_ind not in nc_indexs else empty_emb_zero face_embeds = face_embeds.to(device, dtype=torch.float16) results_dict[real_prompts_ind] = pipe( prompt=real_prompt, @@ -1048,7 +1103,6 @@ def process_generation( generator= generator, face_crop_image=crop_image, face_insightface_embeds=face_embeds, - nc_flag=True if real_prompts_ind in nc_indexs else False, # nc_flag，用索引标记，主要控制非角色人物的生成，默认false ).images[0] else: results_dict[real_prompts_ind] = pipe( @@ -1067,13 +1121,10 @@ def process_generation( generator=generator, nc_flag=True if real_prompts_ind in nc_indexs else False, # nc_flag，用索引标记，主要控制非角色人物的生成，默认false ).images[0] - elif use_flux: if pulid: - img_id = input_id_images_dict[cur_character[0]] if real_prompts_ind not in nc_indexs else empty_img - id_image = resize_numpy_image_long(img_id, 1024) - use_true_cfg = abs(role_scale - 1.0) > 1e-2 - id_embeddings, uncond_id_embeddings = pipe.pulid_model.get_id_embedding(id_image,cal_uncond=use_true_cfg) + id_embeddings=input_id_emb_s_dict[cur_character[0]][0] if real_prompts_ind not in nc_indexs else empty_emb_zero + uncond_id_embeddings=input_id_emb_un_dict[cur_character[0]][0] if real_prompts_ind not in nc_indexs else empty_emb_zero strength = _Ip_Adapter_Strength results_dict[real_prompts_ind] =pipe.generate_image( prompt=real_prompt, @@ -1086,7 +1137,7 @@ def process_generation( uncond_id_embeddings=uncond_id_embeddings, id_weight=strength, guidance=guidance_scale, - true_cfg=role_scale, + true_cfg=1.0, max_sequence_length=128, ) else: @@ -1109,15 +1160,13 @@ def process_generation( generator=generator, ).images[0] elif story_maker and not make_dual_only: - img_2= input_id_images_dict[cur_character[0]] if real_prompts_ind not in nc_indexs else input_id_images_dict[character_list[0]] - img_2=img_2[0] - mask_image = pipeline_mask(img_2, return_mask=True) - mask_image.save("A2.png") - mask_image = mask_image.convert('RGB') # outputs a pillow mask - face_info = app_face.get(cv2.cvtColor(np.array(img_2), cv2.COLOR_RGB2BGR)) - face_info = \ - sorted(face_info, key=lambda x: (x['bbox'][2] - x['bbox'][0]) * (x['bbox'][3] - x['bbox'][1]))[ - -1] # only use the maximum face + cloth_info=None + if isinstance(control_img, torch.Tensor): + cloth_info = input_id_cloth_dict[cur_character[0]][0] + mask_image=input_id_img_s_dict[cur_character[0]][0] + img_2= input_id_images_dict[cur_character[0]][0] if real_prompts_ind not in nc_indexs else empty_img + face_info=input_id_emb_s_dict[cur_character[0]][0] if real_prompts_ind not in nc_indexs else empty_emb_zero + results_dict[real_prompts_ind] = pipe( image=img_2, mask_image=mask_image, @@ -1129,30 +1178,13 @@ def process_generation( guidance_scale=guidance_scale, height=height, width=width, generator=generator, + cloth=cloth_info, ).images[0] else: if photomake_mode == "v2": # V2版本必须要有id_embeds，只能用input_id_images作为风格参考 - img = ( - input_id_images_dict[cur_character[0]] - if real_prompts_ind not in nc_indexs - else input_id_images_dict[character_list[0]] - ), - # print(img,type(img[0][0])) - - img = np.array(img[0][0]) - img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) - from .utils.insightface_package import analyze_faces - faces = analyze_faces(app_face, img, ) - nc_flag = True if real_prompts_ind in nc_indexs else False # 有nc为ture，无则false - if not nc_flag: - id_embed_list = [torch.from_numpy((faces[0]['embedding']))] - else: - id_embed_list = [torch.zeros_like(torch.from_numpy((faces[0]['embedding'])))] - id_embeds = torch.stack(id_embed_list) - - # print(real_prompts_ind, real_prompt,"v2 mode") - + print(cur_character) + id_embeds = input_id_emb_s_dict[cur_character[0]][0] if real_prompts_ind not in nc_indexs else empty_emb_zero results_dict[real_prompts_ind] = pipe( real_prompt, input_id_images=( @@ -1168,7 +1200,7 @@ def process_generation( negative_prompt=negative_prompt, generator=generator, id_embeds=id_embeds, - nc_flag=True if real_prompts_ind in nc_indexs else False, + nc_flag= True if real_prompts_ind in nc_indexs else False, ).images[0] else: #print(real_prompts_ind, real_prompt, "v1 mode", ) @@ -1392,7 +1424,7 @@ def msdiffusion_main(image_1, image_2, prompts_dual, width, height, steps, seed, ) # 添加Lora trigger - add_trigger_words = "," + trigger_words + " " + "style" + ";" + add_trigger_words = " " + trigger_words + " style " if lora: prompts_dual = remove_punctuation_from_strings(prompts_dual) if lora not in lora_lightning_list: # 加速lora不需要trigger @@ -1489,7 +1521,8 @@ class Storydiffusion_Model_Loader: "easy_function":("STRING", {"default": ""}), }, "optional": {"image": ("IMAGE",), - "control_image": ("IMAGE",), }, + "control_image": ("IMAGE",), + "clip":("CLIP",),}, } RETURN_TYPES = ("STORY_DICT", ) @@ -1506,12 +1539,14 @@ class Storydiffusion_Model_Loader: repo = get_instance_path(model_path) return repo def story_model_loader(self,character_prompt, repo_id, ckpt_name, vae_id, character_weights, lora, lora_scale,controlnet_model, clip_vision,trigger_words, scheduler, - sa32_degree, sa64_degree, width, height, photomake_mode, easy_function,**kwargs - ): + sa32_degree, sa64_degree, width, height, photomake_mode, easy_function,**kwargs): + + clip=kwargs.get("clip") + id_number=len(character_prompt.splitlines()) if id_number > 2: id_number=2 - print(f"run in id number {id_number}") + print(f"run in id number ： {id_number}") image = kwargs.get("image") if isinstance(image,torch.Tensor): batch_num,_,_,_=image.size() @@ -1536,7 +1571,24 @@ class Storydiffusion_Model_Loader: control_image=kwargs.get("control_image") if not isinstance(control_image, torch.Tensor): raise "if using controlnet,need input a image in control_image" - + + photomaker_path = os.path.join(photomaker_dir, f"photomaker-{photomake_mode}.bin") + if photomake_mode=="v1": + if not os.path.exists(photomaker_path): + photomaker_path = hf_hub_download( + repo_id="TencentARC/PhotoMaker", + filename="photomaker-v1.bin", + local_dir=photomaker_dir, + ) + else: + if not os.path.exists(photomaker_path): + photomaker_path = hf_hub_download( + repo_id="TencentARC/PhotoMaker-V2", + filename="photomaker-v2.bin", + local_dir=photomaker_dir, + ) + photomake_mode_=photomake_mode + if clip_vision=="none": clip_vision_path=None else: @@ -1558,6 +1610,7 @@ class Storydiffusion_Model_Loader: auraface=True else: auraface = False + if "nf4" in easy_function: NF4=True else: @@ -1577,10 +1630,17 @@ class Storydiffusion_Model_Loader: filename="ipa-faceid-plus.bin", local_dir=photomaker_dir, ) + photomake_mode="" else: kolor_face = False face_ckpt="" + if "schnell" in easy_function: + flux_pulid_name="flux-schnell" + else: + flux_pulid_name = "flux-dev" + + if "pulid" in easy_function: pulid=True pulid_ckpt=os.path.join(photomaker_dir, "pulid_flux_v0.9.0.safetensors") @@ -1590,6 +1650,7 @@ class Storydiffusion_Model_Loader: filename="pulid_flux_v0.9.0.safetensors", local_dir=photomaker_dir, ) + photomake_mode = "" else: pulid = False pulid_ckpt="" @@ -1603,9 +1664,11 @@ class Storydiffusion_Model_Loader: aggressive_offload=True else: aggressive_offload = False + make_dual_only = False if "maker" in easy_function: story_maker = True + photomake_mode = "" if "dual" in easy_function: make_dual_only = True if not clip_vision_path: @@ -1621,23 +1684,6 @@ class Storydiffusion_Model_Loader: else: story_maker = False face_adapter="" - - - photomaker_path = os.path.join(photomaker_dir, f"photomaker-{photomake_mode}.bin") - if photomake_mode=="v1": - if not os.path.exists(photomaker_path): - photomaker_path = hf_hub_download( - repo_id="TencentARC/PhotoMaker", - filename="photomaker-v1.bin", - local_dir=photomaker_dir, - ) - else: - if not os.path.exists(photomaker_path): - photomaker_path = hf_hub_download( - repo_id="TencentARC/PhotoMaker-V2", - filename="photomaker-v2.bin", - local_dir=photomaker_dir, - ) if lora != "none": lora_path = folder_paths.get_full_path("loras", lora) @@ -1675,8 +1721,10 @@ class Storydiffusion_Model_Loader: repo_id.replace("\\","/") if repo_id.rsplit("/")[-1].lower() in "kwai-kolors/kolors": use_kolor=True + photomake_mode = "" if repo_id.rsplit("/")[-1] .lower() in "black-forest-labs/flux.1-dev,black-forest-labs/flux.1-schnell": use_flux = True + photomake_mode = "" if use_kolor: if model_type=="img2img" and not kolor_face: @@ -1687,6 +1735,7 @@ class Storydiffusion_Model_Loader: filename="ip_adapter_plus_general.bin", local_dir=photomaker_dir, ) + photomake_mode = "" if not repo_id and not ckpt_path: raise "you need choice a model or repo_id" @@ -1712,11 +1761,30 @@ class Storydiffusion_Model_Loader: pipe.load_storymaker_adapter(image_encoder, face_adapter, scale=0.8, lora_scale=0.8) pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config) else: + photomake_mode_1 = photomake_mode_ pipe = load_models(ckpt_path, model_type=model_type, single_files=True, use_safetensors=True, - photomake_mode=photomake_mode, photomaker_path=photomaker_path, lora=lora, + photomake_mode=photomake_mode_1, photomaker_path=photomaker_path, lora=lora, lora_path=lora_path, trigger_words=trigger_words, lora_scale=lora_scale) set_attention_processor(pipe.unet, id_length, is_ipadapter=False) + elif "flux" in ckpt_path.lower(): + use_flux=True + if pulid: + from .PuLID.app_flux import FluxGenerator + if not clip_vision_path: + raise "need 'EVA02_CLIP_L_336_psz14_s6B.pt' in comfyUI/models/clip_vision" + if NF4: + quantized_mode = "nf4" + if vae_id == "none": + raise "Now,using pulid must choice ae from comfyUI vae menu" + else: + vae_path = folder_paths.get_full_path("vae", vae_id) + pipe = FluxGenerator(flux_pulid_name, ckpt_path, "cuda", offload=True, + aggressive_offload=aggressive_offload, pretrained_model=pulid_ckpt, + quantized_mode=quantized_mode, clip_vision_path=clip_vision_path, clip_cf=clip, + vae_cf=vae_path) + else: + raise "need choice a SDXL checkpoint" else: pipe = load_models(ckpt_path, model_type=model_type, single_files=True, use_safetensors=True, photomake_mode=photomake_mode, photomaker_path=photomaker_path, lora=lora, @@ -1890,9 +1958,13 @@ class Storydiffusion_Model_Loader: raise "need 'EVA02_CLIP_L_336_psz14_s6B.pt' in comfyUI/models/clip_vision" if NF4: quantized_mode="nf4" - pipe = FluxGenerator(repo_id, ckpt_path, "cuda", offload=True, + if vae_id=="none": + raise "Now,using pulid must choice ae from comfyUI vae menu" + else: + vae_path = folder_paths.get_full_path("vae", vae_id) + pipe = FluxGenerator(flux_pulid_name, ckpt_path, "cuda", offload=True, aggressive_offload=aggressive_offload, pretrained_model=pulid_ckpt, - quantized_mode=quantized_mode,clip_vision_path=clip_vision_path) + quantized_mode=quantized_mode,clip_vision_path=clip_vision_path,clip_cf=clip,vae_cf=vae_path) else: if NF4: # https://github.com/huggingface/diffusers/issues/9165 @@ -2013,16 +2085,37 @@ class Storydiffusion_Model_Loader: # if device != "mps": # pipe.enable_model_cpu_offload() torch.cuda.empty_cache() - if story_maker: - from insightface.app import FaceAnalysis - app_face = FaceAnalysis(name='buffalo_l', root='./', - providers=['CUDAExecutionProvider', 'CPUExecutionProvider']) - app_face.prepare(ctx_id=0, det_size=(640, 640)) - from transformers import pipeline - pipeline_mask = pipeline("image-segmentation", model="briaai/RMBG-1.4", trust_remote_code=True) - # pillow_mask = pipe(image_path, return_mask=True) # outputs a pillow mask - elif use_kolor: - if kolor_face: + # need get emb + character_name_dict_, character_list_ = character_to_dict(character_prompt, lora, trigger_words) + #print(character_list_) + #global input_id_img_s_dict,input_id_emb_s_dict,input_id_emb_un_dict,input_id_cloth_dict + miX_mode = False + if model_type=="img2img": + d1, _, _, _ = image.size() + if d1 == 1: + image_load = [nomarl_upscale(image, width, height)] + else: + img_list = list(torch.chunk(image, chunks=d1)) + image_load = [nomarl_upscale(img, width, height) for img in img_list] + + if photomake_mode == "v2": + from .utils.insightface_package import FaceAnalysis2, analyze_faces + if auraface: + from huggingface_hub import snapshot_download + snapshot_download( + "fal/AuraFace-v1", + local_dir="models/auraface", + ) + app_face = FaceAnalysis2(name="auraface", + providers=["CUDAExecutionProvider", "CPUExecutionProvider"], root=".", + allowed_modules=['detection', 'recognition']) + else: + app_face = FaceAnalysis2(providers=['CUDAExecutionProvider'], + allowed_modules=['detection', 'recognition']) + app_face.prepare(ctx_id=0, det_size=(640, 640)) + pipeline_mask = None + app_face_ = None + elif kolor_face: from .kolors.models.sample_ipadapter_faceid_plus import FaceInfoGenerator from huggingface_hub import snapshot_download snapshot_download( @@ -2030,33 +2123,155 @@ class Storydiffusion_Model_Loader: local_dir='models/antelopev2', ) app_face = FaceInfoGenerator(root_dir=".") + pipeline_mask=None + app_face_ = None + elif story_maker: + from insightface.app import FaceAnalysis + from transformers import pipeline + pipeline_mask = pipeline("image-segmentation", model="briaai/RMBG-1.4", + trust_remote_code=True) + if make_dual_only: # 前段用story 双人用maker + if photomake_mode_ == "v2": + from .utils.insightface_package import FaceAnalysis2 + if auraface: + from huggingface_hub import snapshot_download + snapshot_download( + "fal/AuraFace-v1", + local_dir="models/auraface", + ) + app_face = FaceAnalysis2(name="auraface", + providers=["CUDAExecutionProvider", "CPUExecutionProvider"], + root=".", + allowed_modules=['detection', 'recognition']) + else: + app_face = FaceAnalysis2(providers=['CUDAExecutionProvider'], + allowed_modules=['detection', 'recognition']) + app_face.prepare(ctx_id=0, det_size=(640, 640)) + app_face_ = FaceAnalysis(name='buffalo_l', root='./', + providers=['CUDAExecutionProvider', 'CPUExecutionProvider']) + app_face_.prepare(ctx_id=0, det_size=(640, 640)) + else: + app_face = FaceAnalysis(name='buffalo_l', root='./', + providers=['CUDAExecutionProvider', 'CPUExecutionProvider']) + app_face.prepare(ctx_id=0, det_size=(640, 640)) + app_face_ = None + else: + app_face = FaceAnalysis(name='buffalo_l', root='./', + providers=['CUDAExecutionProvider', 'CPUExecutionProvider']) + app_face.prepare(ctx_id=0, det_size=(640, 640)) + app_face_ = None else: app_face=None - pipeline_mask = None - elif photomake_mode == "v2": - from .utils.insightface_package import FaceAnalysis2, analyze_faces - if auraface: - from huggingface_hub import snapshot_download - snapshot_download( - "fal/AuraFace-v1", - local_dir="models/auraface", - ) - app_face = FaceAnalysis2(name="auraface", - providers=["CUDAExecutionProvider", "CPUExecutionProvider"], root=".", - allowed_modules=['detection', 'recognition']) + pipeline_mask=None + app_face_=None + input_id_emb_s_dict = {} + input_id_img_s_dict = {} + input_id_emb_un_dict ={} + for ind, img in enumerate(image_load): + if photomake_mode == "v2": + from .utils.insightface_package import analyze_faces + img = np.array(img) + img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) + faces = analyze_faces(app_face, img,) + id_embed_list = torch.from_numpy((faces[0]['embedding'])) + crop_image=img + uncond_id_embeddings = None + elif kolor_face: + device = ( + "cuda" + if torch.cuda.is_available() + else "mps" if torch.backends.mps.is_available() else "cpu" + ) + face_info = app_face.get_faceinfo_one_img(img) + face_bbox_square = face_bbox_to_square(face_info["bbox"]) + crop_image = img.crop(face_bbox_square) + crop_image = crop_image.resize((336, 336)) + face_embeds = torch.from_numpy(np.array([face_info["embedding"]])) + id_embed_list = face_embeds.to(device, dtype=torch.float16) + uncond_id_embeddings = None + elif story_maker: + if make_dual_only: #前段用story 双人用maker + if photomake_mode_=="v2": + from .utils.insightface_package import analyze_faces + img = np.array(img) + img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) + faces = analyze_faces(app_face, img, ) + id_embed_list = torch.from_numpy((faces[0]['embedding'])) + crop_image = pipeline_mask(img, return_mask=True).convert('RGB') # outputs a pillow mask + face_info = app_face_.get(cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR)) + uncond_id_embeddings = \ + sorted(face_info, + key=lambda x: (x['bbox'][2] - x['bbox'][0]) * (x['bbox'][3] - x['bbox'][1]))[ + -1] # only use the maximum face + photomake_mode="v2" + miX_mode=True + # make+v2模式下，emb存v2的向量，corp 和 unemb 存make的向量 + else: #V1不需要调用emb + crop_image = pipeline_mask(img, return_mask=True).convert('RGB') # outputs a pillow mask + face_info = app_face.get(cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR)) + id_embed_list = \ + sorted(face_info, + key=lambda x: (x['bbox'][2] - x['bbox'][0]) * (x['bbox'][3] - x['bbox'][1]))[ + -1] # only use the maximum face + uncond_id_embeddings = None + else: #全程用maker + crop_image = pipeline_mask(img, return_mask=True).convert('RGB') # outputs a pillow mask + #timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + #crop_image.copy().save(os.path.join(folder_paths.get_output_directory(),f"{timestamp}_mask.png")) + face_info = app_face.get(cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR)) + id_embed_list = \ + sorted(face_info, + key=lambda x: (x['bbox'][2] - x['bbox'][0]) * (x['bbox'][3] - x['bbox'][1]))[ + -1] # only use the maximum face + + uncond_id_embeddings = None + elif pulid: + id_image = resize_numpy_image_long(img, 1024) + use_true_cfg = abs(1.0 - 1.0) > 1e-2 + id_embed_list, uncond_id_embeddings = pipe.pulid_model.get_id_embedding(id_image,cal_uncond=use_true_cfg) + crop_image = img + else: + id_embed_list=None + uncond_id_embeddings = None + crop_image = None + input_id_img_s_dict[character_list_[ind]] = [crop_image] + input_id_emb_s_dict[character_list_[ind]] = [id_embed_list] + input_id_emb_un_dict[character_list_[ind]] = [uncond_id_embeddings] + + if story_maker or kolor_face or photomake_mode == "v2" : + del app_face + torch.cuda.empty_cache() + if story_maker: + del pipeline_mask + torch.cuda.empty_cache() + + if isinstance(control_image,torch.Tensor) and story_maker: + e1, _, _, _ = control_image.size() + if e1 == 1: + cn_image_load = [nomarl_upscale(control_image, width, height)] + else: + img_list = list(torch.chunk(control_image, chunks=e1)) + cn_image_load = [nomarl_upscale(img, width, height) for img in img_list] + input_id_cloth_dict = {} + for ind, img in enumerate(cn_image_load): + input_id_cloth_dict[character_list_[ind]] = [img] else: - app_face = FaceAnalysis2(providers=['CUDAExecutionProvider'], - allowed_modules=['detection', 'recognition']) - app_face.prepare(ctx_id=0, det_size=(640, 640)) - pipeline_mask = None + input_id_cloth_dict={} else: - app_face=None - pipeline_mask=None + input_id_emb_s_dict = {} + input_id_img_s_dict = {} + input_id_emb_un_dict ={} + input_id_cloth_dict = {} + #print(input_id_img_s_dict) + role_name_list = [i for i in character_name_dict_.keys()] + #print( role_name_list) model={"pipe":pipe,"use_flux":use_flux,"use_kolor":use_kolor,"photomake_mode":photomake_mode,"trigger_words":trigger_words,"lora_scale":lora_scale, "load_chars":load_chars,"repo_id":repo_id,"lora_path":lora_path,"ckpt_path":ckpt_path,"model_type":model_type, "lora": lora, - "scheduler":scheduler,"auraface":auraface,"width":width,"height":height,"kolor_face":kolor_face,"pulid":pulid,"story_maker":story_maker, + "scheduler":scheduler,"width":width,"height":height,"kolor_face":kolor_face,"pulid":pulid,"story_maker":story_maker, "make_dual_only":make_dual_only,"face_adapter":face_adapter,"clip_vision_path":clip_vision_path, - "controlnet_path":controlnet_path,"character_prompt":character_prompt,"image":image,"control_image":control_image,"app_face":app_face,"pipeline_mask":pipeline_mask} + "controlnet_path":controlnet_path,"character_prompt":character_prompt,"image":image,"control_image":control_image, + "input_id_emb_s_dict":input_id_emb_s_dict,"input_id_img_s_dict":input_id_img_s_dict, + "input_id_emb_un_dict":input_id_emb_un_dict,"input_id_cloth_dict":input_id_cloth_dict,"role_name_list":role_name_list,"miX_mode":miX_mode} return (model,) @@ -2071,7 +2286,6 @@ class Storydiffusion_Sampler: "model": ("STORY_DICT",), "scene_prompts": ("STRING", {"multiline": True, "default": "[Taylor] wake up in the bed ;\n[Taylor] have breakfast by the window;\n[Lecun] drving a car;\n[Lecun] is working."}), - "split_prompt": ("STRING", {"default": ""}), "negative_prompt": ("STRING", {"multiline": True, "default": "bad anatomy, bad hands, missing fingers, extra fingers, " "three hands, three legs, bad arms, missing legs, " @@ -2123,7 +2337,7 @@ class Storydiffusion_Sampler: bottom = (height + new_height) / 2 return img.crop((left, top, right, bottom)) - def story_sampler(self, model,scene_prompts,split_prompt, negative_prompt, img_style, seed, steps, + def story_sampler(self, model,scene_prompts, negative_prompt, img_style, seed, steps, cfg, ip_adapter_strength, style_strength_ratio, role_scale, mask_threshold, start_step,save_character,controlnet_scale,guidance_list,): # get value from dict @@ -2139,7 +2353,6 @@ class Storydiffusion_Sampler: ckpt_path=model.get("ckpt_path") lora=model.get("lora") lora_scale =model.get("lora_scale") - auraface=model.get("auraface") scheduler=model.get("scheduler") height=model.get("height") width = model.get("width") @@ -2154,30 +2367,35 @@ class Storydiffusion_Sampler: character_prompt=model.get("character_prompt") control_image=model.get("control_image") image=model.get("image") - pipeline_mask=model.get("pipeline_mask") - app_face=model.get("app_face") + + input_id_emb_s_dict = model.get("input_id_emb_s_dict") + input_id_img_s_dict = model.get("input_id_img_s_dict") + input_id_emb_un_dict = model.get("input_id_emb_un_dict") + input_id_cloth_dict = model.get("input_id_cloth_dict") + role_name_list=model.get("role_name_list") + miX_mode=model.get("miX_mode") + + #print(input_id_emb_s_dict,input_id_img_s_dict,input_id_emb_un_dict,role_name_list) #'[Taylor]',['[Taylor]'] + + empty_emb_zero = None + if pulid or kolor_face or photomake_mode=="v2": + empty_emb_zero = torch.zeros_like(input_id_emb_s_dict[role_name_list[0]][0]).to(device) + # 格式化文字内容 - if split_prompt: - scene_prompts.replace("\n", "").replace(split_prompt, ";\n").strip() - character_prompt.replace("\n", "").replace(split_prompt, ";\n").strip() - else: - scene_prompts.strip() - character_prompt.strip() - if "\n" not in scene_prompts: - scene_prompts.replace(";", ";\n").strip() - if "\n" in character_prompt: - if character_prompt.count("\n") > 1: - character_prompt.replace("\n", "").replace("[", "\n[").strip() - if character_prompt.count("\n") > 1: - character_prompt.replace("\n", "").replace("[", "\n[", 2).strip() # 多行角色在这里强行转为双角色 - + scene_prompts.strip() + character_prompt.strip() # 从角色列表获取角色方括号信息 char_origin = character_prompt.splitlines() + char_origin=[i for i in char_origin if "[" in i] + #print(char_origin) char_describe = char_origin # [A a men...,B a girl ] char_origin = [char.split("]")[0] + "]" for char in char_origin] - + #print(char_origin) # 判断是否有双角色prompt，如果有，获取双角色列表及对应的位置列表， prompts_origin = scene_prompts.splitlines() + prompts_origin=[i.strip() for i in prompts_origin] + prompts_origin = [i for i in prompts_origin if "[" in i] + #print(prompts_origin) positions_dual = [index for index, prompt in enumerate(prompts_origin) if len(extract_content_from_brackets(prompt))>=2] #改成单句中双方括号方法，利于MS组句，[A]... [B]...[C] prompts_dual = [prompt for prompt in prompts_origin if len(extract_content_from_brackets(prompt))>=2] @@ -2205,7 +2423,8 @@ class Storydiffusion_Sampler: height, load_chars, lora, - trigger_words,photomake_mode,use_kolor,use_flux,make_dual_only,kolor_face,pulid,story_maker,app_face, pipeline_mask,role_scale) + trigger_words,photomake_mode,use_kolor,use_flux,make_dual_only, + kolor_face,pulid,story_maker,input_id_emb_s_dict, input_id_img_s_dict,input_id_emb_un_dict, input_id_cloth_dict,role_scale,control_image,empty_emb_zero,miX_mode) else: if story_maker: @@ -2221,7 +2440,8 @@ class Storydiffusion_Sampler: height, load_chars, lora, - trigger_words,photomake_mode,use_kolor,use_flux,make_dual_only,kolor_face,pulid,story_maker,app_face, pipeline_mask,role_scale) + trigger_words,photomake_mode,use_kolor,use_flux,make_dual_only,kolor_face, + pulid,story_maker,input_id_emb_s_dict, input_id_img_s_dict,input_id_emb_un_dict, input_id_cloth_dict,role_scale,control_image,empty_emb_zero,miX_mode) for value in gen: print(type(value)) @@ -2271,26 +2491,28 @@ class Storydiffusion_Sampler: image_encoder = clip_load(clip_vision_path) pipe.load_storymaker_adapter(image_encoder, face_adapter, scale=0.8, lora_scale=0.8) pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config) - - mask_image_1 = pipeline_mask(image_a, return_mask=True).convert('RGB') # applies mask on input and returns a pillow image - mask_image_2 = pipeline_mask(image_b, return_mask=True).convert('RGB') # applies mask on input and returns a pillow image - #mask_image_1.save("asdad1.png") - - face_info_1 = app_face.get(cv2.cvtColor(np.array(image_a), cv2.COLOR_RGB2BGR)) - face_info_1 = \ - sorted(face_info_1, key=lambda x: (x['bbox'][2] - x['bbox'][0]) * (x['bbox'][3] - x['bbox'][1]))[ - -1] # only use the maximum face - face_info_2 = app_face.get(cv2.cvtColor(np.array(image_b), cv2.COLOR_RGB2BGR)) - face_info_2 = \ - sorted(face_info_2, key=lambda x: (x['bbox'][2] - x['bbox'][0]) * (x['bbox'][3] - x['bbox'][1]))[ - -1] # only use the maximum face - # get n p prompt + mask_image_1=input_id_img_s_dict[role_name_list[0]][0] + mask_image_2=input_id_img_s_dict[role_name_list[1]][0] + + if photomake_mode=="v2": + face_info_1 = input_id_emb_un_dict[role_name_list[0]][0] + face_info_2 = input_id_emb_un_dict[role_name_list[1]][0] + else: + face_info_1 = input_id_emb_s_dict[role_name_list[0]][0] + face_info_2 = input_id_emb_s_dict[role_name_list[1]][0] + + cloth_info_1=None + cloth_info_2=None + if isinstance(control_image,torch.Tensor): + cloth_info_1 = input_id_cloth_dict[role_name_list[0]][0] + cloth_info_2 = input_id_cloth_dict[role_name_list[1]][0] + prompts_dual, negative_prompt = apply_style( img_style, prompts_dual, negative_prompt ) # 添加Lora trigger - add_trigger_words = "," + trigger_words + " " + "style" + ";" + add_trigger_words = " " + trigger_words+ " style " if lora: prompts_dual = remove_punctuation_from_strings(prompts_dual) if lora not in lora_lightning_list: # 加速lora不需要trigger @@ -2303,7 +2525,8 @@ class Storydiffusion_Sampler: prompts_dual = [item.replace("[", " ", ).replace("]", " ", ) for item in prompts_dual] image_dual = [] - setup_seed(seed) + if model_type=="txt2img": + setup_seed(seed) generator = torch.Generator(device='cuda').manual_seed(seed) for i,prompt in enumerate(prompts_dual): output = pipe( @@ -2316,6 +2539,8 @@ class Storydiffusion_Sampler: guidance_scale=cfg, height=height, width=width, generator=generator, + cloth=cloth_info_1, + cloth_2=cloth_info_2 ).images[0] image_dual.append(output) else: #using ms diffusion diff --git a/examples/flux.png b/examples/flux.png new file mode 100644 index 0000000..703da89 Binary files /dev/null and b/examples/flux.png differ diff --git a/examples/flux_img2img2role.png b/examples/flux_img2img2role.png deleted file mode 100644 index f0c4b60..0000000 Binary files a/examples/flux_img2img2role.png and /dev/null differ diff --git a/examples/flux_pulid_fp8_12GVR.png b/examples/flux_pulid_fp8_12GVR.png deleted file mode 100644 index f106f08..0000000 Binary files a/examples/flux_pulid_fp8_12GVR.png and /dev/null differ diff --git a/examples/kolor.png b/examples/kolor.png new file mode 100644 index 0000000..d62c342 Binary files /dev/null and b/examples/kolor.png differ diff --git a/examples/kolor_faceid.png b/examples/kolor_faceid.png deleted file mode 100644 index 21dec23..0000000 Binary files a/examples/kolor_faceid.png and /dev/null differ diff --git a/examples/kolor_ipadapter.png b/examples/kolor_ipadapter.png deleted file mode 100644 index 0cfe4a2..0000000 Binary files a/examples/kolor_ipadapter.png and /dev/null differ diff --git a/examples/kolor_ipadapter_use_chinese.png b/examples/kolor_ipadapter_use_chinese.png deleted file mode 100644 index d6c5cfa..0000000 Binary files a/examples/kolor_ipadapter_use_chinese.png and /dev/null differ diff --git a/examples/kolor_using_comfyclip.png b/examples/kolor_using_comfyclip.png deleted file mode 100644 index 442a28d..0000000 Binary files a/examples/kolor_using_comfyclip.png and /dev/null differ diff --git a/examples/kolor_using_configeclip.png b/examples/kolor_using_configeclip.png deleted file mode 100644 index 9baf1c5..0000000 Binary files a/examples/kolor_using_configeclip.png and /dev/null differ diff --git a/examples/maker2role.png b/examples/maker2role.png new file mode 100644 index 0000000..9c4a116 Binary files /dev/null and b/examples/maker2role.png differ diff --git a/examples/new_0922.json b/examples/new_0922.json deleted file mode 100644 index 613990b..0000000 --- a/examples/new_0922.json +++ /dev/null @@ -1,240 +0,0 @@ -{ - "last_node_id": 52, - "last_link_id": 60, - "nodes": [ - { - "id": 43, - "type": "SaveImage", - "pos": { - "0": 9880, - "1": -1580 - }, - "size": { - "0": 576.3123168945312, - "1": 565.4249267578125 - }, - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 59, - "label": "images" - } - ], - "outputs": [], - "properties": {}, - "widgets_values": [ - "ComfyUI" - ] - }, - { - "id": 49, - "type": "LoadImage", - "pos": { - "0": 8684, - "1": -1319 - }, - "size": { - "0": 315, - "1": 314 - }, - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 60 - ], - "slot_index": 0, - "shape": 3, - "label": "IMAGE" - }, - { - "name": "MASK", - "type": "MASK", - "links": null, - "shape": 3, - "label": "MASK" - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "4.jpg", - "image" - ] - }, - { - "id": 51, - "type": "Storydiffusion_Model_Loader", - "pos": { - "0": 9015, - "1": -1570 - }, - "size": { - "0": 391.1097412109375, - "1": 541.8019409179688 - }, - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [ - { - "name": "image", - "type": "IMAGE", - "link": 60, - "label": "image" - }, - { - "name": "control_image", - "type": "IMAGE", - "link": null, - "label": "control_image" - } - ], - "outputs": [ - { - "name": "model", - "type": "STORY_DICT", - "links": [ - 58 - ], - "slot_index": 0, - "shape": 3, - "label": "model" - } - ], - "properties": { - "Node name for S&R": "Storydiffusion_Model_Loader" - }, - "widgets_values": [ - "[Taylor] a woman img, wearing a white T-shirt, blue loose hair.", - "F:/test/ComfyUI/models/diffusers/black-forest-labs/FLUX.1-dev", - "flux1-kj-dev-fp8.safetensors", - "none", - "none", - "none", - 0.8, - "none", - "EVA02_CLIP_L_336_psz14_s6B.pt", - "best quality", - "Euler", - 0.5, - 0.5, - 768, - 768, - "v1", - "pulid,fp8,cpu" - ] - }, - { - "id": 52, - "type": "Storydiffusion_Sampler", - "pos": { - "0": 9435, - "1": -1579 - }, - "size": [ - 435.2590231270369, - 632.1462531527177 - ], - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "STORY_DICT", - "link": 58, - "label": "model" - } - ], - "outputs": [ - { - "name": "image", - "type": "IMAGE", - "links": [ - 59 - ], - "slot_index": 0, - "shape": 3, - "label": "image" - }, - { - "name": "prompt_array", - "type": "STRING", - "links": null, - "shape": 3, - "label": "prompt_array" - } - ], - "properties": { - "Node name for S&R": "Storydiffusion_Sampler" - }, - "widgets_values": [ - "[Taylor] have lunch ;\n", - "", - "bad anatomy, bad hands, missing fingers, extra fingers, three hands, three legs, bad arms, missing legs, missing arms, poorly drawn face, bad face, fused face, cloned face, three crus, fused feet, fused thigh, extra crus, ugly fingers, horn,amputation, disconnected limbs", - "No_style", - 1242473506, - "randomize", - 20, - 7, - 0.5, - 20, - 0.8, - 0.5, - 5, - false, - 0.8, - "0., 0.25, 0.4, 0.75;0.6, 0.25, 1., 0.75" - ] - } - ], - "links": [ - [ - 58, - 51, - 0, - 52, - 0, - "STORY_DICT" - ], - [ - 59, - 52, - 0, - 43, - 0, - "IMAGE" - ], - [ - 60, - 49, - 0, - 51, - 0, - "IMAGE" - ] - ], - "groups": [], - "config": {}, - "extra": { - "ds": { - "scale": 0.8769226950000392, - "offset": [ - -8631.83743113032, - 1742.8190057917652 - ] - } - }, - "version": 0.4 -} \ No newline at end of file diff --git a/examples/newest.png b/examples/newest.png deleted file mode 100644 index 5f80860..0000000 Binary files a/examples/newest.png and /dev/null differ diff --git a/examples/photomakev1.png b/examples/photomakev1.png deleted file mode 100644 index 21c2321..0000000 Binary files a/examples/photomakev1.png and /dev/null differ diff --git a/examples/photomakev2.png b/examples/photomakev2.png deleted file mode 100644 index 4e95916..0000000 Binary files a/examples/photomakev2.png and /dev/null differ diff --git a/examples/phtomakev2.png b/examples/phtomakev2.png new file mode 100644 index 0000000..b8c8d3d Binary files /dev/null and b/examples/phtomakev2.png differ diff --git a/examples/pulid_flux.png b/examples/pulid_flux.png deleted file mode 100644 index 6c28091..0000000 Binary files a/examples/pulid_flux.png and /dev/null differ diff --git a/examples/using_storymake_dual.png b/examples/using_storymake_dual.png deleted file mode 100644 index 636b929..0000000 Binary files a/examples/using_storymake_dual.png and /dev/null differ diff --git a/examples/using_storymake_dual_only.png b/examples/using_storymake_dual_only.png deleted file mode 100644 index 2e1197d..0000000 Binary files a/examples/using_storymake_dual_only.png and /dev/null differ diff --git a/examples/using_storymake_dual_onlyA.png b/examples/using_storymake_dual_onlyA.png deleted file mode 100644 index 47ea644..0000000 Binary files a/examples/using_storymake_dual_onlyA.png and /dev/null differ diff --git a/pyproject.toml b/pyproject.toml index a92df89..6e2d0a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,9 @@ [project] name = "comfyui_storydiffusion" description = "you can using sotry-diffusion in comfyui" -version = "1.0.1" +version = "1.1.0" license = { file = "LICENSE" } -dependencies = ["ip_adapter @ git+https://github.com/tencent-ailab/IP-Adapter.git@main", "spaces", "peft"] +dependencies = ["diffusers", "spaces", "peft"] [project.urls] Repository = "https://github.com/smthemex/ComfyUI_StoryDiffusion" diff --git a/utils/gradio_utils.py b/utils/gradio_utils.py index ed55259..63e14c1 100644 --- a/utils/gradio_utils.py +++ b/utils/gradio_utils.py @@ -486,7 +486,7 @@ def get_cur_id_list(real_prompt,character_dict,character_index_dict): real_prompt = real_prompt.replace(keys,character_dict[keys]) return list_arr,real_prompt -def process_original_prompt(character_dict,prompts,id_length): +def process_original_prompt(character_dict,prompts,id_length,img_mode): replace_prompts = [] character_index_dict = {} invert_character_index_dict = {} @@ -506,19 +506,22 @@ def process_original_prompt(character_dict,prompts,id_length): replace_prompts.append(cur_prompt) ref_index_dict = {} ref_totals = [] - # print(character_index_dict,66666) for character_key in character_index_dict.keys(): if character_key not in character_index_dict: raise f"{character_key} not have prompt description, please remove it" index_list = character_index_dict[character_key] index_list = [index for index in index_list if len(invert_character_index_dict[index]) == 1] - # print(index_list,55555) - if len(index_list) < id_length: - raise f"{character_key} not have enough prompt description, need no less than {id_length}, but you give {len(index_list)}" + if not img_mode: + if len(index_list) < id_length: + raise f"{character_key} not have enough prompt description, need no less than {id_length}, but you give {len(index_list)}" ref_index_dict[character_key] = index_list[:id_length] ref_totals = ref_totals + index_list[:id_length] return character_index_dict,invert_character_index_dict,replace_prompts,ref_index_dict,ref_totals +#character_index_dict：{'[Taylor]': [0, 3], '[sam]': [1, 2]},if 1 role {'[Taylor]': [0, 1, 2]} +#invert_character_index_dict:{0: ['[Taylor]'], 1: ['[sam]'], 2: ['[sam]'], 3: ['[Taylor]']},if 1 role {0: ['[Taylor]'], 1: ['[Taylor]'], 2: ['[Taylor]']} +#ref_indexs_dict:{'[Taylor]': [0, 3], '[sam]': [1, 2]},if 1 role {'[Taylor]': [0]} +#ref_totals: [0, 3, 1, 2] if 1 role [0] def get_ref_character(real_prompt,character_dict): list_arr = [] diff --git a/utils/pipeline.py b/utils/pipeline.py index 3e677cd..fc1c71b 100644 --- a/utils/pipeline.py +++ b/utils/pipeline.py @@ -12,8 +12,7 @@ from huggingface_hub.utils import validate_hf_hub_args from transformers import CLIPImageProcessor, CLIPTokenizer from diffusers import StableDiffusionXLPipeline try: - from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl import StableDiffusionXLPipelineOutput, \ - rescale_noise_cfg + from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl import StableDiffusionXLPipelineOutput,rescale_noise_cfg except: try: from diffusers.pipelines.stable_diffusion_xl import StableDiffusionXLPipelineOutput,rescale_noise_cfg diff --git a/utils/style_template.py b/utils/style_template.py index bfa58e6..79964e3 100644 --- a/utils/style_template.py +++ b/utils/style_template.py @@ -1,52 +1,52 @@ style_list = [ { "name": "No_style", - "prompt": "{prompt},8k, RAW photo, best quality,", + "prompt": "{prompt} 8k,RAW", "negative_prompt": "worst quality", }, { "name": "Realistic", - "prompt": "{prompt}, (photorealistic:1.4),8k,RAW photo, best quality, masterpiece,dramatic shadows,uhd, high quality,dramatic,cinematic", + "prompt": "{prompt} (photorealistic:1.4),8k,RAW photo, best quality, masterpiece,dramatic shadows,uhd, high quality,dramatic,cinematic", "negative_prompt": "(worst quality, low quality, normal quality, lowres), (bad teeth, deformed teeth, deformed lips), (bad proportions), (deformed eyes, bad eyes), (deformed face, ugly face), (deformed hands,fused fingers), morbid, mutilated, mutation, disfigured", }, { "name": "Japanese_Anime", - "prompt": "{prompt}.(Anime Style, Manga Style:1.3)anime artwork illustrating,best quality,webtoon,manhua,Linear compositions", + "prompt": "{prompt} (Anime Style, Manga Style:1.3)anime artwork illustrating,best quality,webtoon,manhua,Linear compositions", "negative_prompt": "lowres,text, error, extra digit, fewer digits, cropped, worst quality, low quality, normal quality, jpeg artifacts, signature, watermark, username, blurry", }, { "name": "Digital_Oil_Painting", - "prompt": "{prompt} . (Extremely Detailed Oil Painting:1.2), glow effects, godrays,render, 8k, octane render, cinema 4d, blender, dark, atmospheric 4k ultra detailed, cinematic sensual,humorous illustration,", + "prompt": "{prompt} (Extremely Detailed Oil Painting:1.2), glow effects, godrays,render, 8k, octane render, cinema 4d, blender, dark, atmospheric 4k ultra detailed, cinematic sensual,humorous illustration ", "negative_prompt": "anime, cartoon, graphic, text, crayon, graphite, abstract, glitch, deformed, mutated, ugly, disfigured, lowres, error, extra digit, fewer digits, cropped, worst quality, low quality, normal quality, jpeg artifacts, signature, watermark, username, blurry", }, { "name": "Pixar_Disney_Character", - "prompt": "Create a Disney Pixar 3D style illustration on {prompt} . The scene is vibrant, motivational, filled with vivid colors and a sense of wonder.", + "prompt": "Create a Disney Pixar 3D style illustration on {prompt} The scene is vibrant, motivational, filled with vivid colors and a sense of wonder.", "negative_prompt": "lowres, text, bad eyes, bad legs, error,extra digit, fewer digits, cropped, worst quality, low quality, normal quality, jpeg artifacts, signature, watermark, blurry, grayscale, noisy, sloppy, messy, grainy,ultra textured,", }, { "name": "Photographic", - "prompt": "cinematic photo {prompt} . Hyperrealistic, Hyperdetailed, detailed skin,soft lighting, best quality, ultra realistic, 8k, golden ratio, Intricate, High Detail, film photography", + "prompt": "cinematic photo {prompt} Hyperrealistic, Hyperdetailed, detailed skin,soft lighting, best quality, ultra realistic, 8k, golden ratio, Intricate, High Detail, film photography", "negative_prompt": "drawing, painting, crayon, sketch, graphite, impressionist, noisy, blurry, soft, deformed, ugly, lowres, text, error, extra digit, fewer digits, cropped, worst quality, low quality,signature, watermark, username", }, { "name": "Comic_book", - "prompt": "comic {prompt} . graphic illustration, comic art, graphic novel art, vibrant, highly detailed", + "prompt": "comic {prompt} graphic illustration, comic art, graphic novel art, vibrant, highly detailed", "negative_prompt": "photograph, deformed, glitch, noisy, realistic, stock photo, lowres,text, error,extra digit, fewer digits, cropped, worst quality, low quality, normal quality,signature, watermark, username, blurry", }, { "name": "Line_art", - "prompt": "line art drawing {prompt} . professional, sleek, modern, minimalist, graphic, line art, vector graphics", + "prompt": "line art drawing {prompt} professional, sleek, modern, minimalist, graphic, line art, vector graphics", "negative_prompt": "photorealistic, 35mm film, deformed, glitch, blurry, noisy, off-center, deformed, cross-eyed, closed eyes,ugly, disfigured, mutated, realism, realistic, impressionism, expressionism,acrylic, lowres, text, error,extra digit, fewer digits, worst quality, low quality, normal quality, jpeg artifacts, signature, watermark, username", }, { "name": "Black_and_White_Film_Noir", - "prompt": "{prompt} . (b&w, Monochromatic, Film Photography:1.3), film noir, analog style, soft lighting, subsurface scattering,heavy shadow, masterpiece, best quality,", + "prompt": "{prompt} (b&w, Monochromatic, Film Photography:1.3), film noir, analog style, soft lighting, subsurface scattering,heavy shadow, masterpiece, best quality,", "negative_prompt": "anime, photorealistic, 35mm film, deformed, glitch, blurry, noisy, off-center, deformed, cross-eyed, closed eyes,ugly, disfigured, mutated, impressionism, expressionism,acrylic, lowres,text, error, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality, normal quality, jpeg artifacts, signature, watermark, username", }, { "name": "Isometric_Rooms", - "prompt": "Tiny cute isometric {prompt} .in a cutaway box, soft smooth lighting, soft colors, 100mm lens, 3d blender render", + "prompt": "Tiny cute isometric {prompt} in a cutaway box, soft smooth lighting, soft colors, 100mm lens, 3d blender render", "negative_prompt": "photorealistic,35mm film,deformed,glitch,blurry,noisy,off-center,deformed,cross-eyed,closed eyes,ugly,disfigured,mutated,realism,realistic,impressionism,expressionism,acrylic,lowres,text,error,extra digit,fewer digits,cropped, worst quality, low quality, normal quality, jpeg artifacts, signature,watermark, username,", }, ]