add new 1

PuLID/app_flux.py

@@ -8,29 +8,31 @@ from .flux.util import load_ae, load_clip, load_flow_model, load_t5, load_flow_m
 from .pulid.pipeline_flux import PuLIDPipeline
 
 
-def get_models(name: str,ckpt_path, device: torch.device, offload: bool,quantized_mode):
-    t5 = load_t5(name,quantized_mode,device, max_length=128)
-    clip = load_clip(name,quantized_mode,device)
+def get_models(name: str,ckpt_path,vae_cf, clip_cf,device: torch.device, offload: bool,quantized_mode):
+    t5 = load_t5(name,clip_cf,quantized_mode,device, max_length=128)
+    clip = load_clip(name,clip_cf,quantized_mode,device)
     if quantized_mode=="fp8" or quantized_mode=="nf4":
         model = load_flow_model_quintized(name, ckpt_path,quantized_mode,device="cpu" if offload else device)
     else:
         model = load_flow_model(name, ckpt_path,device="cpu" if offload else device)
     model.eval()
-    ae = load_ae(name, device="cpu" if offload else device)
+    ae = load_ae(name,vae_cf, device="cpu" if offload else device)
     return model, ae, t5, clip
 
 
 class FluxGenerator:
-    def __init__(self, model_name: str, ckpt_path,device: str, offload: bool,aggressive_offload: bool, pretrained_model,quantized_mode,clip_vision_path):
+    def __init__(self, model_name: str, ckpt_path,device: str, offload: bool,aggressive_offload: bool, pretrained_model,quantized_mode,clip_vision_path,clip_cf,vae_cf):
         self.device = torch.device(device)
         self.offload = offload
         self.model_name = model_name
         self.aggressive_offload= aggressive_offload
         self.ckpt_path=ckpt_path
         self.clip_vision_path=clip_vision_path
+        self.clip_cf=clip_cf
+        self.vae_cf=vae_cf
         self.quantized_mode=quantized_mode
         self.model, self.ae, self.t5, self.clip = get_models(
-            model_name,self.ckpt_path,
+            model_name,self.ckpt_path,self.vae_cf,self.clip_cf,
             device=self.device,
             offload=self.offload,quantized_mode=self.quantized_mode,
         )
@@ -55,7 +57,7 @@ class FluxGenerator:
             timestep_to_start_cfg=1,
             max_sequence_length=128,
     ):
-        self.t5.max_length = max_sequence_length
+        #self.t5.max_length = max_sequence_length
 
         seed = int(seed)
         if seed == -1:
@@ -69,9 +71,9 @@ class FluxGenerator:
             raise "prompt must be list or str"
 
         img_output = []
-        for i,prompt in enumerate(prompt_list):
+        for i in prompt_list:
             opts = SamplingOptions(
-                prompt=prompt,
+                text=i,
                 width=width,
                 height=height,
                 num_steps=num_steps,
@@ -81,7 +83,8 @@ class FluxGenerator:
             
             if opts.seed is None:
                 opts.seed = torch.Generator(device="cpu").seed()
-            print(f"Generating '{opts.prompt}' with seed {opts.seed}")
+            opts.text = opts.text if isinstance(opts.text,str) else str(opts.text)
+            print(f"Generating '{opts.text}' with seed {opts.seed}")
             t0 = time.perf_counter()
             
             use_true_cfg = abs(true_cfg - 1.0) > 1e-2
@@ -103,7 +106,7 @@ class FluxGenerator:
             
             if self.offload:
                 self.t5, self.clip = self.t5.to(self.device), self.clip.to(self.device)
-            inp = prepare(t5=self.t5, clip=self.clip, img=x, prompt=opts.prompt)
+            inp = prepare(t5=self.t5, clip=self.clip, img=x, prompt=opts.text)
             inp_neg = prepare(t5=self.t5, clip=self.clip, img=x, prompt=neg_prompt) if use_true_cfg else None
             
             # offload TEs to CPU, load model to gpu
@@ -156,4 +159,4 @@ class FluxGenerator:
             img = Image.fromarray((127.5 * (x + 1.0)).cpu().byte().numpy())
             # return img, str(opts.seed), self.pulid_model.debug_img_list
             img_output.append(img)
-        return img_output
+        return img_output

PuLID/flux/modules/conditioner.py

@@ -3,41 +3,47 @@ from transformers import CLIPTextModel, CLIPTokenizer, T5EncoderModel, T5Tokeniz
 
 
 class HFEmbedder(nn.Module):
-    def __init__(self, version: str, tokenizer_path,is_clip,quantized_mode, max_length: int, **hf_kwargs):
+    def __init__(self, version: str, tokenizer_path,clip_cf,is_clip,quantized_mode, max_length: int, **hf_kwargs):
         super().__init__()
         #self.is_clip = version.startswith("openai")
         self.is_clip=is_clip
         self.quantized_mode=quantized_mode
+        self.clip_cf=clip_cf
         self.max_length = max_length
-        self.output_key = "pooler_output" if self.is_clip else "last_hidden_state"
+        #self.output_key = "pooler_output" if self.is_clip else "last_hidden_state"
+        self.output_key = "pooled_output" if self.is_clip else "cond"
 
-        if self.is_clip:
-            self.tokenizer: CLIPTokenizer = CLIPTokenizer.from_pretrained(tokenizer_path, max_length=max_length)
-            self.hf_module: CLIPTextModel = CLIPTextModel.from_pretrained(version, **hf_kwargs)
-        else:
-            self.tokenizer: T5Tokenizer = T5Tokenizer.from_pretrained(tokenizer_path, max_length=max_length)
-            self.hf_module: T5EncoderModel = T5EncoderModel.from_pretrained(version, **hf_kwargs)
-        if self.quantized_mode=="fp8":
-            from optimum.quanto import freeze, qfloat8, quantize
-            quantize(self.hf_module, weights=qfloat8)
-            freeze(self.hf_module)
-            
-        self.hf_module = self.hf_module.eval().requires_grad_(False)
+        # if self.is_clip:
+        #     self.tokenizer: CLIPTokenizer = CLIPTokenizer.from_pretrained(tokenizer_path, max_length=max_length)
+        #     self.hf_module: CLIPTextModel = CLIPTextModel.from_pretrained(version, **hf_kwargs)
+        # else:
+        #     self.tokenizer: T5Tokenizer = T5Tokenizer.from_pretrained(tokenizer_path, max_length=max_length)
+        #     self.hf_module: T5EncoderModel = T5EncoderModel.from_pretrained(version, **hf_kwargs)
+        # if self.quantized_mode=="fp8":
+        #     from optimum.quanto import freeze, qfloat8, quantize
+        #     quantize(self.hf_module, weights=qfloat8)
+        #     freeze(self.hf_module)
+        #
+        # self.hf_module = self.hf_module.eval().requires_grad_(False)
 
     def forward(self, text: list[str]) -> Tensor:
-        batch_encoding = self.tokenizer(
-            text,
-            truncation=True,
-            max_length=self.max_length,
-            return_length=False,
-            return_overflowing_tokens=False,
-            padding="max_length",
-            return_tensors="pt",
-        )
+        # batch_encoding = self.tokenizer(
+        #     text,
+        #     truncation=True,
+        #     max_length=self.max_length,
+        #     return_length=False,
+        #     return_overflowing_tokens=False,
+        #     padding="max_length",
+        #     return_tensors="pt",
+        # )
 
-        outputs = self.hf_module(
-            input_ids=batch_encoding["input_ids"].to(self.hf_module.device),
-            attention_mask=None,
-            output_hidden_states=False,
-        )
-        return outputs[self.output_key]
+        # outputs = self.hf_module(
+        #     input_ids=batch_encoding["input_ids"].to(self.hf_module.device),
+        #     attention_mask=None,
+        #     output_hidden_states=False,
+        # )
+        tokens = self.clip_cf.tokenize(text)
+        outputs = self.clip_cf.encode_from_tokens(tokens, return_pooled=True, return_dict=True)
+        # cond = outputs.pop("cond")
+        # pooled_output=outputs.pop("pooled_output")
+        return outputs.pop(self.output_key)

PuLID/flux/sampling.py

@@ -43,8 +43,8 @@ def prepare(t5: HFEmbedder, clip: HFEmbedder, img: Tensor, prompt: str) -> dict[
     img_ids[..., 2] = img_ids[..., 2] + torch.arange(w // 2)[None, :]
     img_ids = repeat(img_ids, "h w c -> b (h w) c", b=bs)
 
-    if isinstance(prompt, str):
-        prompt = [prompt]
+    # if isinstance(prompt, str):
+    #     prompt = [prompt]
     txt = t5(prompt)
     if txt.shape[0] == 1 and bs > 1:
         txt = repeat(txt, "1 ... -> bs ...", bs=bs)
@@ -56,10 +56,10 @@ def prepare(t5: HFEmbedder, clip: HFEmbedder, img: Tensor, prompt: str) -> dict[
 
     return {
         "img": img,
-        "img_ids": img_ids.to(img.device),
-        "txt": txt.to(img.device),
-        "txt_ids": txt_ids.to(img.device),
-        "vec": vec.to(img.device),
+        "img_ids": img_ids.to(img.device,torch.bfloat16),
+        "txt": txt.to(img.device,torch.bfloat16),
+        "txt_ids": txt_ids.to(img.device,torch.bfloat16),
+        "vec": vec.to(img.device,torch.bfloat16),
     }
 
 

PuLID/flux/util.py

@@ -2,7 +2,6 @@ import os
 from dataclasses import dataclass
 
 import torch
-#from .transformer_flux import FluxTransformer2DModel
 from huggingface_hub import hf_hub_download
 from safetensors.torch import load_file as load_sft
 import json
@@ -13,7 +12,7 @@ import folder_paths
 
 @dataclass
 class SamplingOptions:
-    prompt: str
+    text: str
     width: int
     height: int
     num_steps: int
@@ -144,23 +143,60 @@ def print_load_warning(missing: list[str], unexpected: list[str]) -> None:
 
 # from XLabs-AI https://github.com/XLabs-AI/x-flux/blob/1f8ef54972105ad9062be69fe6b7f841bce02a08/src/flux/util.py#L330
 def load_flow_model_quintized(name: str, ckpt_path,quantized_mode,device: str = "cuda", hf_download: bool = True):
+    from optimum.quanto import requantize,qint4,quantize,freeze
     # Loading Flux
-    print("Init model")
-    if "dev" in name:
-        name = "flux-dev"
-    else:
-        name = "flux-schnell"
+    print(f"Init model in {quantized_mode}")
     model = Flux(configs[name].params).to(torch.bfloat16)
-    if quantized_mode == "nf4":  #nf4 is now useful
-            print("Loading checkpoint in nf4")
-            json_path = os.path.join(folder_paths.base_path, "custom_nodes", "ComfyUI_StoryDiffusion", "utils", "config.json")
+    if quantized_mode == "nf4":  #nf4 is now useful.
+        json_path = os.path.join(folder_paths.base_path, "custom_nodes", "ComfyUI_StoryDiffusion", "utils",
+                                 "config.json")
+        test=False
+        if  test:
+            from accelerate.utils import set_module_tensor_to_device, compute_module_sizes
+            from accelerate import init_empty_weights
+            from .convert_nf4_flux import _replace_with_bnb_linear, create_quantized_param, \
+                check_quantized_param
+            import safetensors.torch
+            import gc
+            from diffusers import FluxTransformer2DModel
+            dtype = torch.bfloat16
+            is_torch_e4m3fn_available = hasattr(torch, "float8_e4m3fn")
+            original_state_dict = safetensors.torch.load_file(ckpt_path)
+            print("Start a quantization process...")
+            with init_empty_weights():
+                config = FluxTransformer2DModel.load_config(json_path)
+                model = FluxTransformer2DModel.from_config(config).to(dtype)
+                expected_state_dict_keys = list(model.state_dict().keys())
+            
+            _replace_with_bnb_linear(model, "nf4")
+            
+            for param_name, param in original_state_dict.items():
+                if param_name not in expected_state_dict_keys:
+                    continue
+                
+                is_param_float8_e4m3fn = is_torch_e4m3fn_available and param.dtype == torch.float8_e4m3fn
+                if torch.is_floating_point(param) and not is_param_float8_e4m3fn:
+                    param = param.to(dtype)
+                
+                if not check_quantized_param(model, param_name):
+                    set_module_tensor_to_device(model, param_name, device=0, value=param)
+                else:
+                    create_quantized_param(
+                        model, param, param_name, target_device=0, state_dict=original_state_dict,
+                        pre_quantized=True
+                    )
+            
+            del original_state_dict
+            gc.collect()
+
+        else:
             sd = load_sft(ckpt_path, device='cpu')
             with open(json_path) as f:
                 quantization_map = json.load(f)
+            
             print("Start a quantization process...")
-            from optimum.quanto import requantize
             requantize(model, sd, quantization_map)
-            print("Model is quantized!")
+        print("Model is quantized!")
     else: #fp8
         # ckpt_path = 'models/flux-dev-fp8.safetensors'
         if (
@@ -172,27 +208,20 @@ def load_flow_model_quintized(name: str, ckpt_path,quantized_mode,device: str =
         else:
             json_path = os.path.join(folder_paths.base_path, "custom_nodes", "ComfyUI_StoryDiffusion", "utils",
                                      "config.json")
-        print("Loading checkpoint in fp8")
         # load_sft doesn't support torch.device
         sd = load_sft(ckpt_path, device='cpu')
         with open(json_path) as f:
             quantization_map = json.load(f)
         print("Start a quantization process...")
-        from optimum.quanto import requantize
         requantize(model, sd, quantization_map)
         print("Model is quantized!")
     return model
 
 def load_flow_model(name: str,ckpt_path, device: str = "cuda", hf_download: bool = True):
     # Loading Flux
-    print("Init model")
-    if "dev" in name:
-        name="flux-dev"
-    else:
-        name="flux-schnell"
+    print("Init model in fp16")
     if ckpt_path is not None:
         # load_sft doesn't support torch.device
-        print("Loading checkpoint in fp16")
         with torch.device(device):
             model = Flux(configs[name].params).to(torch.bfloat16)
         sd = load_sft(ckpt_path, device=str(device))
@@ -201,24 +230,21 @@ def load_flow_model(name: str,ckpt_path, device: str = "cuda", hf_download: bool
     return model
 
 
-def load_t5(name,quantized_mode,device = "cuda", max_length: int = 512) -> HFEmbedder:
+def load_t5(name,clip_cf,quantized_mode,device = "cuda", max_length: int = 512) -> HFEmbedder:
     # max length 64, 128, 256 and 512 should work (if your sequence is short enough)
     #return HFEmbedder("xlabs-ai/xflux_text_encoders", max_length=max_length, torch_dtype=torch.bfloat16).to(device)
-    return HFEmbedder(f"{name}/text_encoder_2", f"{name}/tokenizer_2",is_clip=False,quantized_mode=quantized_mode,max_length=max_length, torch_dtype=torch.bfloat16).to(device)
+    return HFEmbedder(f"{name}/text_encoder_2", f"{name}/tokenizer_2",clip_cf,is_clip=False,quantized_mode=quantized_mode,max_length=max_length, torch_dtype=torch.bfloat16).to(device)
 
 
-def load_clip(name,quantized_mode,device= "cuda") -> HFEmbedder:
-    return HFEmbedder(f"{name}/text_encoder",f"{name}/tokenizer",is_clip=True,quantized_mode=quantized_mode,max_length=77, torch_dtype=torch.bfloat16).to(device)
+def load_clip(name,clip_cf,quantized_mode,device= "cuda") -> HFEmbedder:
+    return HFEmbedder(f"{name}/text_encoder",f"{name}/tokenizer",clip_cf,is_clip=True,quantized_mode=quantized_mode,max_length=77, torch_dtype=torch.bfloat16).to(device)
     #return HFEmbedder("openai/clip-vit-large-patch14", max_length=77, torch_dtype=torch.bfloat16).to(device)
 
 
-def load_ae(name: str, device: str = "cuda", hf_download: bool = True) -> AutoEncoder:
-    ckpt_path = os.path.join(name,"ae.safetensors")   # ckpt_path = configs[name].ae_path
+def load_ae(name: str,vae_cf, device: str = "cuda", hf_download: bool = True) -> AutoEncoder:
+    ckpt_path=vae_cf
+    #ckpt_path = os.path.join(name,"ae.safetensors")   # ckpt_path = configs[name].ae_path
 
-    if "dev" in name:
-        name = "flux-dev"
-    else:
-        name = "flux-schnell"
     if (
         not os.path.exists(ckpt_path)
         and configs[name].repo_id is not None

README.md

@@ -1,37 +1,38 @@
-# ComfyUI_StoryDiffusion
-You can using StoryDiffusion in ComfyUI.
+<div align="center">
+<h1> ComfyUI_StoryDiffusion：You can using StoryDiffusion in ComfyUI.</h1>
 
 * [中文说明](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/README-CN.md)  
 * StoryDiffusion origin From: [link](https://github.com/HVision-NKU/StoryDiffusion)  ---&---  MS-Diffusion origin From: [link](https://github.com/MS-Diffusion/MS-Diffusion)---&---StoryMakerr from From:[StoryMaker](https://github.com/RedAIGC/StoryMaker)
 
-
 ## Updates:
-**2024/09/24**   
-* Now if using Kolor's "ip-adapter" or "face ID", you can choose the monolithic model of clip_vision (such as :"clip-vit-large-patch14.safetensors") to load the image encoder. The change this brings is that Kolor's local directory can delete all files in the "clip-vit-large-patch14-336" and "Kolors-IP-Adapter-Plus" folders. Of course, because comfyUI defaults to clip image processing of "224" size , while Kolor defaults to 336 size, there will be a loss of accuracy and quality. Please refer to the image comparison in readme for details.
-* Another change is that we now need to port the model of "ip_adapter_plus_general.bin" in "kolor-ipadapter" to the "comfyUI/models/photomaker" directory;  
+**2024/09/27**   
+* make easy and comfy
   
-**Previous updates：**  
-* For the convenience of use, the layout of the node has been adjusted again,delete "id number"(base character lines now),delete " model_type"(base input image [img2img function] or not [txt2img function] now), 'clip_vision' , 'controlnet' , 'character prompt' ,'image','control_image' now in model loader node now.   
-* Now,using story_maker or pulid_flux function ,need choice clip vision model (use 'clip_vision' menu);   
-* add StoryMaker from From: [StoryMaker](https://github.com/RedAIGC/StoryMaker) to make dual role..or normal img2img,as detailed in the following text 3.5 ,fill "maker,dual" in easy function to using StoryMaker for dual role; 
-* Add " PulID FLUX " function, In my testing, a minimum of 12GB of VRAM can run normally,now you can fill "X:/xxx/xxx/black-forest-labs/FLUX.1-dev",and fill easy function "pilid,fp8,cpu"(if you Vram>30G,can remove cpu,and using Kijai/flux-fp8,if you Vram>45G,can remove fp8,cpu), although it is a bit slow if using cpu! ,Of course, some models need to be prepared, as detailed in the following text;
-* Add kolor FaceId function, now you can fill "xxx:/xxx/xxx/Kwai-Kolors/Kolors",and fill easy function "face",Of course, some models need to be prepared, as detailed in the following text; 
-* Add diffusers'img2img codes( Not commit diffusers yet),Now you can using flux img2img function. in flux img2img,"guidance_scale" is usually 3.5 ,you can change ip-adapter_strength's number to Control the noise of the output image, the closer the number is to 1, the less it looks like the original image, and the closer it is to 0, the more it looks like the original image. Correspondingly, your generated step count is a multiple of this value, which means that if you enter 50 steps, the actual number of steps taken is 50 * 0.8 (0.8 is the value of change ip-adapter_strength) #you can see exmaple img
-* AWPortrait-FL-fp8.safetensors is support if using fp8 mode,..  
-* using img crop to fix ms_diffusion only using square's error;
-* change W and H global names,it cause some error;
-* fix runway error,when loader single model. 
-* The loading speed of the NF4 model is many times faster than FP8, so I recommend using the NF4 model to run Flux. I have included the workflow of NF4 in the example，
-* Add an "easy_function" for debugging new function. This time, I have added support for "auraface" in "Photomake V2". You can enter "auraface" into the "easy_function" to test this method
-* support Flux ,1.only using fp8 repo,fill local "X:/xxx/xxx/black-forest-labs/FLUX.1-dev" and ckpt_name="none";if save .pt,fill easy function "save",2. using unet,fill local repo and choice a fp8 Unet(like Kijai/flux-fp8, AWPortrait-FL-fp8.safetensors. *.pt), 3. using fn4,need download weights at [link](https://huggingface.co/sayakpaul/flux.1-dev-nf4/tree/main),put weight in "comfyui/models/checkpoints/";fill local "X:/xxx/xxx/black-forest-labs/FLUX.1-dev"  
-* Now clip checkpoints no need diffusers_repo,you can using "clip_g.safetensors" or other base from "CLIP-ViT-bigG-14-laion2B-39B-b160k";       
-* 2 role in 1 img now using [A]...[B]... mode,  
-* Support "kolors" text2img and "kolors"ipadapter img2img,using repo like :"xxx:/xxx/xxx/Kwai-Kolors/Kolors"  (Please refer to the end of the article for detailed file combinations)  
-* support photomaker V2;  
-* ControlNet now uses community models.   
-* The base model now has only two options: using repo input or selecting the community model...  
-* Introducing Controlnet for dual character co image, supporting multi image introduction, 
-* Add the function of saving and loading character models   
+## Function introduction  
+**story-diffusion**  
+* Support img2img & txt2img，All you need to do is select an SDXL model in“ckpt_name”menu  to get started, support sdxl lora;  
+* If using img2img,You can choose between the v1 or v2 version of PhotoMaker checkpoints(Automatic downloads);  phtomaker v2 need insightface and the models that go with it;
+
+**ms-diffusion**  
+* Enabled when 2 characters are required to appear in the same image, When [roleA] and [roleB] appear in a scene prompt at the same time, it will be automatically enabled;
+* Of course the "ms_adapter.bin" model is also required, and the "clip_vision_g.safetensors"I is selected in the clip_vision;
+* MS supports ControlNet, and a dual-role prompt requires a preprocessed ControlNet picture.
+
+**story-maker**
+* "story-maker“ is similar to "story-diffusion", currently only supports img2img, which features the ability to migrate the character's clothing and supports dual characters with the same picture;
+* To turn on this function, you need to enter 'maker' in easy-function; Then select an sdxl model and select the "clip_vision_H.safetensors" model in the clip-vision,The companion “mask.bin” model and“insightface"model are automatically downloaded;   
+* If you enter 'maker,dual', the function of using 'story-diffusion'  in the front section and using 'story-maker' in the same picture for both characters will be enabled
+* The method requires a mask, so “RMBG-1.4” is built-in, which can be downloaded automatically；
+
+**kolor**
+* With kolor, you need to enter the local path of kolor in the repo_id, using '/' splitting the directory;You can use "clip-vit-large-patch14.safetensors", or the image encoder in the repo;
+* Kolor supports img2img (IPadapeter and FaceID), txt2img,The matching model will be automatically downloaded, and the details can be found in the README model content; 
+* Kolor supports prompt input in all Chinese characters, note that the character name needs to be changed to ['张三']；
+* using kolor FaceId function, need fill easy function "face",
+
+**Flux and PULID-FLUX**
+* Flux supports img2img and txt2img, and supports FP8 and NF4 (recommended) quantization models；To enable it, enter the local path of flux diffuser in 'repo_id' and select the corresponding model in 'ckpt-name';example fill "X:/xxx/xxx/black-forest-labs/FLUX.1-dev"; 
+* PULID-FLUX needs to connect to the dual clip nodes of comfy in clip, and select 'EVA02_CLIP_L_336_psz14_s6B.ptin ' clip-vision, 'ae.safetensors' in vae, fill in 'pulid, fp8, cpu' in 'easy-function' , if your VRAM is about 24G, you can try to remove the cpu;The accompanying 'insightface' model will be automatically downloaded; 
 
 1.Installation
 -----
@@ -45,7 +46,7 @@ git clone https://github.com/smthemex/ComfyUI_StoryDiffusion.git
 ```
 pip install -r requirements.txt
 ```
-if using photomakerV2,need:  
+if using photomaker V2，pulid-flux，kolor，story-make:  
 ```
 pip install insightface==0.7.3  or new   
 ```
@@ -214,7 +215,7 @@ downlaod nf4 model  [link](https://huggingface.co/sayakpaul/flux.1-dev-nf4/tree/
 |             ├── rename nf4 ckpt
 ```
 
-**3.3.3.4 using flux pulid,repo_id+ckpt_name**     .   
+**3.3.3.4 using flux pulid,clip+ckpt_name**     .   
 torch must > 0.24.0   
 optimum-quanto must >=0.2.4 
 ```
@@ -225,6 +226,7 @@ DIAMONIK7777/antelopev2 auto downlaod....[https://huggingface.co/DIAMONIK7777/an
 "pulid_flux_v0.9.0.safetensors" download from [link](https://huggingface.co/guozinan/PuLID/tree/main)     
 fp8 using flux1-dev-fp8.safetensors  from [link](https://huggingface.co/Kijai/flux-fp8/tree/main)       
 make sure ae.safetensors in you FLUX.1-dev dir,example:    
+
 ```
 ├── ComfyUI/models/photomaker/
 |             ├── pulid_flux_v0.9.0.safetensors
@@ -232,6 +234,9 @@ make sure ae.safetensors in you FLUX.1-dev dir,example:
 |             ├── EVA02_CLIP_L_336_psz14_s6B.pt
 ├── ComfyUI/models/checkpoints/
 |             ├── flux1-dev-fp8.safetensors
+├── ComfyUI/models/clip/
+|             ├── t5xxl_fp8_e4m3fn.safetensors
+|             ├── clip_l.safetensors
 ```
 
 **3.5 if using storymake..**    
@@ -252,51 +257,36 @@ RMBG-1.4 from  [link](https://huggingface.co/briaai/RMBG-1.4/tree/main)#自动
 ```
 4 Example
 ----
-img2img mode only use storymaker  只用storymaker生成(包括双角色),最新 (Latest version) 
-![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/using_storymake_dual.png)
+**story-make**
+img2img  纯storymaker生成，非最新示例 (outdated version examples)   
+![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/maker2role.png)
 
-img2img mode use storymaker for dual role only,只用storymaker生成双角色,单角色用diffusion生成,最新(Latest version) 
-![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/using_storymake_dual_only.png)
+**flux-pulid**
+img2img mode use flux pulid  12G Vram,cpu  Flux使用PULID功能,最新示例(Latest version) 
+![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/flux.png)
 
-img2img mode use flux pulid  12G Vram,cpu  Flux使用PULID功能,最新(Latest version) 
-![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/pulid_flux.png)
+**kolor-face**
+img2img kolor face，参数输入没变化，非最新示例  (outdated version examples)   
+![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/kolor.png)
 
-img2img mode use nf4 flux  开启flux nf4模式,速度最快,示例图为旧节点(outdated version examples)  
-![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/flux_img2img.png)
-![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/flux_img2img2role.png)
-
-txt2img mode use NF4 FLUX 开启flux nf4模式,速度最快,示例图为旧节点 (outdated version examples)        
+**flux-nf4**
+* txt2img mode use NF4 FLUX 开启flux nf4模式,速度最快，非最新示例 (outdated version examples)        
 ![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/nf4.png)
+* flux img2img 图生图，注意用ip的参数来控制噪声（多样性），非最新示例 (outdated version examples)   
+![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/flux_img2img.png)
 
-img2img mode use auraface photomake V2  开启v2模式,示例图为旧节点 (outdated version examples)        
-![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/newest.png)
-
-img2img mode  kolors ip-adapter using monolithic model   可图使用单体模型来解码图片 最新 (Latest version) 
-![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/kolor_using_comfyclip.png)
-
-img2img model use kolors ip adapter,and using chinese prompt 开启kolor模式,使用中文提示词,示例图为旧节点 (outdated version examples)   
-![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/kolor_ipadapter_use_chinese.png)
-![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/kolor_ipadapter.png)
-
-img2img model use kolors ip adapter face id  (outdated version examples)   
-![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/kolor_faceid.png)
-
-img2img sdxl mode, uses photomakeV1 (outdated version examples)   
-![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/photomakev1.png)
-
-img2img sdxl  mode, uses photomakeV2 (outdated version examples)   
-![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/photomakev2.png)
-
-txt2img using lora and comic node (outdated version examples)   
-![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/txt2img_lora_comic.png)
-
-img2img2role in 1 image (Outdated version examples)   
+**ms-diffusion**
+* img2img2role in 1 image，双角色同图，非最新示例 (Outdated version examples)   
 ![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/2rolein1img.png)
-
-ControlNet added dual role co frame (Role 1 and Role 2) (Outdated version examples)  
+* ControlNet added dual role co frame (Role 1 and Role 2) (Outdated version examples)  
 ![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/controlnet.png)
 
-Translate the text into other language examples, and the translation nodes in the diagram can be replaced with any translation node. Outdated version examples     
+**story-diffusion**
+* photomake v2 in img2img normal 最基础的story流程，非最新示例 (outdated version examples)   
+![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/phtomakev2.png)
+* txt2img using lora and comic node (outdated version examples)   
+![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/txt2img_lora_comic.png)
+* Translate the text into other language examples, and the translation nodes in the diagram can be replaced with any translation node. Outdated version examples     
 ![](https://github.com/smthemex/ComfyUI_StoryDiffusion/blob/main/examples/trans1.png)
 
 Function Description of Nodes  
@@ -304,6 +294,7 @@ Function Description of Nodes
 **<Storydiffusion_Model_Loader>**    
 * image: optional,The interface that must be linked to the image generation diagram. For dual roles, please follow the example and use the built-in image batch node in comfyUI; 
 * controlnet:only ms function support;  
+* clip:Connect the two clip nodes of Comfyui（using flux-pulid）
 * character prompt: The prompt for the character, [character name] must be at the beginning. If using the graphic mode, the keyword "img" must be added, such as a man img;if using  chinese prompt, need["角色名"] or ['角色名']  
 * repo_id: using diffuser models ;     
 * ckpt_name:  using  community SDLX model selection;   
@@ -323,17 +314,16 @@ Function Description of Nodes
 **<Storydiffusion_Sampler>**      
 * model: The interface that must be linked;   
 * Scene prompts: The prompt for the scene description, [character name], must start at the beginning. It is best for both characters to appear once in the first two lines. [NC] At the beginning, the character does not appear (suitable for non character scenes). When [character A] and [character B], MS diffusion's dual character mode is enabled, and and the spaces before and after it cannot be ignored# Used for segmented prompt, rendering the entire segment, but only outputting the prompt after #;    
-* split prompt: The symbol for splitting the prompt, which does not take effect when it is empty. It is used to normalize paragraphs when the prompt is external. For example, when you pass in 10 lines of text, the hyphen may not be correct, but using a hyphen, such as ";", can effectively distinguish each line.     
 * negative prompt: only effective when img_style is No_style;      
 * seed/steps/cfg: suitable for commonly used functions in comfyUI;     
 * ip-adapter_strength: img2img controls the weight of ip-adapter in graph generation,only using in kolors;   
 * style_strength'ratio: Style weight control, which controls from which step the style takes effect. When the style consistency is not good, you can try increasing or decreasing this parameter;   
 * role-scale: only effective when two characters are in the same image, controlling the weight of the characters in the image;   
-* Mask_threshold: It is only effective when two roles are in the same picture, and controls the position of the role in the picture (MS system automatically assigns the role position according to prompt, so appropriate role position information description can be added to prompt);   
+* Mask_threshold: It is only effective when two roles are in the same picture, and controls the position of the role in the picture (MS system automatically assigns the role position according to prompt, so appropriate role position information description can be added to prompt)(ms-diffusion only);   
 * Start_step: Only effective when two characters are in the same image, controlling the number of starting steps for the character's position in the image   
 * Save_character: Whether to save the character weights of the current character, file in/ Under ComfyUI_StoryDiffusion/weights/pt, use time as the file name;  
-* Controllet_scale: control net weight;   
-* guidance_list: contrlol role's position;     
+* Controllet_scale: control net weight,(ms-diffusion only);   
+* guidance_list: contrlol role's position(ms-diffusion only);     
 
 **<Comic_Type>**        
 * Fonts list: The puzzle node supports custom fonts (place the font file in the fonts directory. fonts/you_font. ttf);   
@@ -350,11 +340,38 @@ Function Description of Nodes
 * Playground v2.5 can be effective on txt2img, and there is no Playground v2.5 style Lora available when accelerated Lora can be used;   
 * Role-scale, mask_threshold, and start_step mainly regulate the randomness and style consistency of two characters in the same frame;   
 * The consistency of style can be adjusted between ip-adapter_strength and style_strength'ratio in img2img;   
-* Preprocess translation text nodes, please refer to the example diagram for usage methods. (Pay attention to changing the font for Chinese or other East Asian characters);    * By default, use the ";" at the end of each paragraph to divide the paragraph. After translation into Chinese, there is a chance that it will be translated as ";", so remember to change it to ";", otherwise it will be a sentence.   
-* The process of generating images using PhotosMaker requires the IMG keyword in the character prompt column. You can use keywords such as a woman IMG, a man IMG, etc;   
+* Preprocess translation text nodes, please refer to the example diagram for usage methods. (Pay attention to changing the font for Chinese or other East Asian characters);    
+* By default, use the ";" at the end of each paragraph to divide the paragraph. After translation into Chinese, there is a chance that it will be translated as ";", so remember to change it to ";", otherwise it will be a sentence.   
+* The process of generating images using PhotosMaker requires the "img" keyword in the character prompt column. You can use keywords such as a woman "img" , a man img , etc;   
 * No characters appear in the image, add [NC] in front of the scene prompt;   
 * Segmented prompt, using #, such as AAAA # BBBB, will generate AAAA content, but the text will only display BBBB   
 
+**Previous updates**
+* Now if using Kolor's "ip-adapter" or "face ID", you can choose the monolithic model of clip_vision (such as :"clip-vit-large-patch14.safetensors") to load the image encoder. The change this brings is that Kolor's local directory can delete all files in the "clip-vit-large-patch14-336" and "Kolors-IP-Adapter-Plus" folders. Of course, because comfyUI defaults to clip image processing of "224" size , while Kolor defaults to 336 size, there will be a loss of accuracy and quality. Please refer to the image comparison in readme for details.
+* Another change is that we now need to port the model of "ip_adapter_plus_general.bin" in "kolor-ipadapter" to the "comfyUI/models/photomaker" directory;  
+* For the convenience of use, the layout of the node has been adjusted again,delete "id number"(base character lines now),delete " model_type"(base input image [img2img function] or not [txt2img function] now), 'clip_vision' , 'controlnet' , 'character prompt' ,'image','control_image' now in model loader node now.   
+* Now,using story_maker or pulid_flux function ,need choice clip vision model (use 'clip_vision' menu);   
+* add StoryMaker from From: [StoryMaker](https://github.com/RedAIGC/StoryMaker) to make dual role..or normal img2img,as detailed in the following text 3.5 ,fill "maker,dual" in easy function to using StoryMaker for dual role; 
+* Add " PulID FLUX " function, In my testing, a minimum of 12GB of VRAM can run normally,now you can fill "X:/xxx/xxx/black-forest-labs/FLUX.1-dev",and fill easy function "pilid,fp8,cpu"(if you Vram>30G,can remove cpu,and using Kijai/flux-fp8,if you Vram>45G,can remove fp8,cpu), although it is a bit slow if using cpu! ,Of course, some models need to be prepared, as detailed in the following text;
+* Add kolor FaceId function, now you can fill "xxx:/xxx/xxx/Kwai-Kolors/Kolors",and fill easy function "face",Of course, some models need to be prepared, as detailed in the following text; 
+* Add diffusers'img2img codes( Not commit diffusers yet),Now you can using flux img2img function. in flux img2img,"guidance_scale" is usually 3.5 ,you can change ip-adapter_strength's number to Control the noise of the output image, the closer the number is to 1, the less it looks like the original image, and the closer it is to 0, the more it looks like the original image. Correspondingly, your generated step count is a multiple of this value, which means that if you enter 50 steps, the actual number of steps taken is 50 * 0.8 (0.8 is the value of change ip-adapter_strength) #you can see exmaple img
+* AWPortrait-FL-fp8.safetensors is support if using fp8 mode,..  
+* using img crop to fix ms_diffusion only using square's error;
+* change W and H global names,it cause some error;
+* fix runway error,when loader single model. 
+* The loading speed of the NF4 model is many times faster than FP8, so I recommend using the NF4 model to run Flux. I have included the workflow of NF4 in the example，
+* Add an "easy_function" for debugging new function. This time, I have added support for "auraface" in "Photomake V2". You can enter "auraface" into the "easy_function" to test this method
+* support Flux ,1.only using fp8 repo,fill local "X:/xxx/xxx/black-forest-labs/FLUX.1-dev" and ckpt_name="none";if save .pt,fill easy function "save",2. using unet,fill local repo and choice a fp8 Unet(like Kijai/flux-fp8, AWPortrait-FL-fp8.safetensors. *.pt), 3. using fn4,need download weights at [link](https://huggingface.co/sayakpaul/flux.1-dev-nf4/tree/main),put weight in "comfyui/models/checkpoints/";fill local "X:/xxx/xxx/black-forest-labs/FLUX.1-dev"  
+* Now clip checkpoints no need diffusers_repo,you can using "clip_g.safetensors" or other base from "CLIP-ViT-bigG-14-laion2B-39B-b160k";       
+* 2 role in 1 img now using [A]...[B]... mode,  
+* Support "kolors" text2img and "kolors"ipadapter img2img,using repo like :"xxx:/xxx/xxx/Kwai-Kolors/Kolors"  (Please refer to the end of the article for detailed file combinations)  
+* support photomaker V2;  
+* ControlNet now uses community models.   
+* The base model now has only two options: using repo input or selecting the community model...  
+* Introducing Controlnet for dual character co image, supporting multi image introduction, 
+* Add the function of saving and loading character models   
+
+
 Citation
 ------
 

StoryMaker/ip_adapter_/ip_adapter.py

@@ -12,7 +12,7 @@ if is_torch2_available():
     from .attention_processor import IPAttnProcessor2_0 as IPAttnProcessor, AttnProcessor2_0 as AttnProcessor, CNAttnProcessor2_0 as CNAttnProcessor
 else:
     from .attention_processor import IPAttnProcessor, AttnProcessor, CNAttnProcessor
-from .resampler import Resampler
+from .resampler_ import Resampler
 
 
 class ImageProjModel(torch.nn.Module):

StoryMaker/ip_adapter_/ip_adapter_faceid.py

@@ -21,7 +21,7 @@ if is_torch2_available() and (not USE_DAFAULT_ATTN):
     )
 else:
     from .attention_processor_faceid import LoRAAttnProcessor, LoRAIPAttnProcessor
-from .resampler import PerceiverAttention, FeedForward, Resampler
+from .resampler_ import PerceiverAttention, FeedForward, Resampler
 
 class FacePerceiverResampler(torch.nn.Module):
     def __init__(

StoryMaker/pipeline_sdxl_storymaker.py

@@ -48,12 +48,12 @@ from diffusers.utils.import_utils import is_xformers_available
 from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection
 from insightface.utils import face_align
 
-from .ip_adapter.resampler import Resampler
-from .ip_adapter.utils import is_torch2_available
-from .ip_adapter.ip_adapter_faceid import faceid_plus
+from .ip_adapter_.resampler_ import Resampler
+from .ip_adapter_.utils import is_torch2_available
+from .ip_adapter_.ip_adapter_faceid import faceid_plus
 
-from .ip_adapter.attention_processor import IPAttnProcessor2_0 as IPAttnProcessor, AttnProcessor2_0 as AttnProcessor
-from .ip_adapter.attention_processor_faceid import LoRAIPAttnProcessor2_0 as LoRAIPAttnProcessor, LoRAAttnProcessor2_0 as LoRAAttnProcessor
+from .ip_adapter_.attention_processor import IPAttnProcessor2_0 as IPAttnProcessor, AttnProcessor2_0 as AttnProcessor
+from .ip_adapter_.attention_processor_faceid import LoRAIPAttnProcessor2_0 as LoRAIPAttnProcessor, LoRAAttnProcessor2_0 as LoRAAttnProcessor
 
 logger = logging.get_logger(__name__)  # pylint: disable=invalid-name
 
@@ -216,8 +216,7 @@ class StableDiffusionXLStoryMakerPipeline(StableDiffusionXLPipeline):
         clip_img = self.clip_image_processor(images=ref_img, return_tensors="pt").pixel_values
         return clip_img, clip_face, torch.from_numpy(face_info.normed_embedding).unsqueeze(0)
 
-    def _encode_prompt_image_emb(self, image, image_2, mask_image, mask_image_2, face_info, face_info_2, cloth, cloth_2, \
-                                 device, num_images_per_prompt, dtype, do_classifier_free_guidance):
+    def _encode_prompt_image_emb(self, image, image_2, mask_image, mask_image_2, face_info, face_info_2, cloth, cloth_2,device, num_images_per_prompt, dtype, do_classifier_free_guidance):
         crop_list = []; face_list = [];  id_list = []
         if image is not None:
             clip_img, clip_face, face_emb = self.crop_image(image, mask_image, face_info)
@@ -252,9 +251,12 @@ class StableDiffusionXLStoryMakerPipeline(StableDiffusionXLPipeline):
         if do_classifier_free_guidance:
             prompt_image_emb = self.image_proj_model(id_embeds, clip_image_embeds, clip_face_embeds)
             B, C, D = prompt_image_emb.shape
-            prompt_image_emb = prompt_image_emb.view(1, B*C, D)
+            #prompt_image_emb = prompt_image_emb.view(1, B*C, D)
+            prompt_image_emb = prompt_image_emb.view(1, B * C, D)
+            
             neg_emb = self.image_proj_model(torch.zeros_like(id_embeds), torch.zeros_like(clip_image_embeds), torch.zeros_like(clip_face_embeds))
-            neg_emb = neg_emb.view(1, B*C, D)
+            #neg_emb = neg_emb.view(1, B*C, D)
+            neg_emb = neg_emb.view(1, B * C, D)
             prompt_image_emb = torch.cat([neg_emb, prompt_image_emb], dim=0) #torch.Size([2, 40, 2048])
             #print(prompt_image_emb.shape)
         else:
@@ -264,8 +266,9 @@ class StableDiffusionXLStoryMakerPipeline(StableDiffusionXLPipeline):
         
         #print(f'prompt_image_emb: {prompt_image_emb.shape}')#torch.Size([2, 40, 2048])
         bs_embed, seq_len, _ = prompt_image_emb.shape
-        
         prompt_image_emb = prompt_image_emb.repeat(1, num_images_per_prompt, 1)
+        
+        #prompt_image_emb = prompt_image_emb.repeat(1, num_images_per_prompt, 1)
         prompt_image_emb = prompt_image_emb.view(bs_embed * num_images_per_prompt, seq_len, -1)
         
         return prompt_image_emb.to(device=device, dtype=dtype)
@@ -498,7 +501,7 @@ class StableDiffusionXLStoryMakerPipeline(StableDiffusionXLPipeline):
             batch_size = len(prompt)
         else:
             batch_size = prompt_embeds.shape[0]
-        #print(batch_size)
+        #print(f"batch_size is {batch_size}")
         device = self.unet.device
         # pdb.set_trace()
         # 3.1 Encode input prompt
@@ -598,8 +601,8 @@ class StableDiffusionXLStoryMakerPipeline(StableDiffusionXLPipeline):
         prompt_embeds = prompt_embeds.to(device)
         add_text_embeds = add_text_embeds.to(device)
         add_time_ids = add_time_ids.to(device).repeat(batch_size * num_images_per_prompt, 1)
-       # print(prompt_embeds.shape, prompt_image_emb.shape) torch.Size([4, 77, 2048])?  torch.Size([2, 40, 2048])
-        prompt_image_emb=torch.cat((prompt_image_emb,) * batch_size, dim=0)
+        #print(prompt_embeds.shape,"and", prompt_image_emb.shape) # torch.Size([4, 77, 2048]) ?  torch.Size([2, 40, 2048])
+        #prompt_image_emb = torch.cat([prompt_image_emb] * batch_size, dim=0)
         #print(prompt_embeds.shape, prompt_image_emb.shape)
         encoder_hidden_states = torch.cat([prompt_embeds, prompt_image_emb], dim=1)
 
@@ -696,4 +699,4 @@ class StableDiffusionXLStoryMakerPipeline(StableDiffusionXLPipeline):
         if not return_dict:
             return (image,)
 
-        return StableDiffusionXLPipelineOutput(images=image)
+        return StableDiffusionXLPipelineOutput(images=image)

examples/new_0922.json

@@ -1,240 +0,0 @@
-{
-  "last_node_id": 52,
-  "last_link_id": 60,
-  "nodes": [
-    {
-      "id": 43,
-      "type": "SaveImage",
-      "pos": {
-        "0": 9880,
-        "1": -1580
-      },
-      "size": {
-        "0": 576.3123168945312,
-        "1": 565.4249267578125
-      },
-      "flags": {},
-      "order": 3,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "images",
-          "type": "IMAGE",
-          "link": 59,
-          "label": "images"
-        }
-      ],
-      "outputs": [],
-      "properties": {},
-      "widgets_values": [
-        "ComfyUI"
-      ]
-    },
-    {
-      "id": 49,
-      "type": "LoadImage",
-      "pos": {
-        "0": 8684,
-        "1": -1319
-      },
-      "size": {
-        "0": 315,
-        "1": 314
-      },
-      "flags": {},
-      "order": 0,
-      "mode": 0,
-      "inputs": [],
-      "outputs": [
-        {
-          "name": "IMAGE",
-          "type": "IMAGE",
-          "links": [
-            60
-          ],
-          "slot_index": 0,
-          "shape": 3,
-          "label": "IMAGE"
-        },
-        {
-          "name": "MASK",
-          "type": "MASK",
-          "links": null,
-          "shape": 3,
-          "label": "MASK"
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "LoadImage"
-      },
-      "widgets_values": [
-        "4.jpg",
-        "image"
-      ]
-    },
-    {
-      "id": 51,
-      "type": "Storydiffusion_Model_Loader",
-      "pos": {
-        "0": 9015,
-        "1": -1570
-      },
-      "size": {
-        "0": 391.1097412109375,
-        "1": 541.8019409179688
-      },
-      "flags": {},
-      "order": 1,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "image",
-          "type": "IMAGE",
-          "link": 60,
-          "label": "image"
-        },
-        {
-          "name": "control_image",
-          "type": "IMAGE",
-          "link": null,
-          "label": "control_image"
-        }
-      ],
-      "outputs": [
-        {
-          "name": "model",
-          "type": "STORY_DICT",
-          "links": [
-            58
-          ],
-          "slot_index": 0,
-          "shape": 3,
-          "label": "model"
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "Storydiffusion_Model_Loader"
-      },
-      "widgets_values": [
-        "[Taylor] a woman img, wearing a white T-shirt, blue loose hair.",
-        "F:/test/ComfyUI/models/diffusers/black-forest-labs/FLUX.1-dev",
-        "flux1-kj-dev-fp8.safetensors",
-        "none",
-        "none",
-        "none",
-        0.8,
-        "none",
-        "EVA02_CLIP_L_336_psz14_s6B.pt",
-        "best quality",
-        "Euler",
-        0.5,
-        0.5,
-        768,
-        768,
-        "v1",
-        "pulid,fp8,cpu"
-      ]
-    },
-    {
-      "id": 52,
-      "type": "Storydiffusion_Sampler",
-      "pos": {
-        "0": 9435,
-        "1": -1579
-      },
-      "size": [
-        435.2590231270369,
-        632.1462531527177
-      ],
-      "flags": {},
-      "order": 2,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "model",
-          "type": "STORY_DICT",
-          "link": 58,
-          "label": "model"
-        }
-      ],
-      "outputs": [
-        {
-          "name": "image",
-          "type": "IMAGE",
-          "links": [
-            59
-          ],
-          "slot_index": 0,
-          "shape": 3,
-          "label": "image"
-        },
-        {
-          "name": "prompt_array",
-          "type": "STRING",
-          "links": null,
-          "shape": 3,
-          "label": "prompt_array"
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "Storydiffusion_Sampler"
-      },
-      "widgets_values": [
-        "[Taylor] have lunch ;\n",
-        "",
-        "bad anatomy, bad hands, missing fingers, extra fingers, three hands, three legs, bad arms, missing legs, missing arms, poorly drawn face, bad face, fused face, cloned face, three crus, fused feet, fused thigh, extra crus, ugly fingers, horn,amputation, disconnected limbs",
-        "No_style",
-        1242473506,
-        "randomize",
-        20,
-        7,
-        0.5,
-        20,
-        0.8,
-        0.5,
-        5,
-        false,
-        0.8,
-        "0., 0.25, 0.4, 0.75;0.6, 0.25, 1., 0.75"
-      ]
-    }
-  ],
-  "links": [
-    [
-      58,
-      51,
-      0,
-      52,
-      0,
-      "STORY_DICT"
-    ],
-    [
-      59,
-      52,
-      0,
-      43,
-      0,
-      "IMAGE"
-    ],
-    [
-      60,
-      49,
-      0,
-      51,
-      0,
-      "IMAGE"
-    ]
-  ],
-  "groups": [],
-  "config": {},
-  "extra": {
-    "ds": {
-      "scale": 0.8769226950000392,
-      "offset": [
-        -8631.83743113032,
-        1742.8190057917652
-      ]
-    }
-  },
-  "version": 0.4
-}

pyproject.toml

@@ -1,9 +1,9 @@
 [project]
 name = "comfyui_storydiffusion"
 description = "you can using sotry-diffusion in comfyui"
-version = "1.0.1"
+version = "1.1.0"
 license = { file = "LICENSE" }
-dependencies = ["ip_adapter @ git+https://github.com/tencent-ailab/IP-Adapter.git@main", "spaces", "peft"]
+dependencies = ["diffusers", "spaces", "peft"]
 
 [project.urls]
 Repository = "https://github.com/smthemex/ComfyUI_StoryDiffusion"

utils/gradio_utils.py

@@ -486,7 +486,7 @@ def get_cur_id_list(real_prompt,character_dict,character_index_dict):
             real_prompt = real_prompt.replace(keys,character_dict[keys])
     return list_arr,real_prompt
 
-def process_original_prompt(character_dict,prompts,id_length):
+def process_original_prompt(character_dict,prompts,id_length,img_mode):
     replace_prompts = []
     character_index_dict = {}
     invert_character_index_dict = {}
@@ -506,19 +506,22 @@ def process_original_prompt(character_dict,prompts,id_length):
                 replace_prompts.append(cur_prompt)
     ref_index_dict = {}
     ref_totals = []
-    # print(character_index_dict,66666)
     for character_key in character_index_dict.keys():
         if character_key not in character_index_dict:
             raise f"{character_key} not have prompt description, please remove it"
         index_list = character_index_dict[character_key]
         index_list = [index for index in index_list if len(invert_character_index_dict[index]) == 1]
-        # print(index_list,55555)
-        if len(index_list) < id_length:
-            raise f"{character_key} not have enough prompt description, need no less than {id_length}, but you give {len(index_list)}"
+        if not img_mode:
+            if len(index_list) < id_length:
+                raise f"{character_key} not have enough prompt description, need no less than {id_length}, but you give {len(index_list)}"
         ref_index_dict[character_key] = index_list[:id_length]
         ref_totals = ref_totals + index_list[:id_length]
     return character_index_dict,invert_character_index_dict,replace_prompts,ref_index_dict,ref_totals
 
+#character_index_dict：{'[Taylor]': [0, 3], '[sam]': [1, 2]},if 1 role {'[Taylor]': [0, 1, 2]}
+#invert_character_index_dict:{0: ['[Taylor]'], 1: ['[sam]'], 2: ['[sam]'], 3: ['[Taylor]']},if 1 role  {0: ['[Taylor]'], 1: ['[Taylor]'], 2: ['[Taylor]']}
+#ref_indexs_dict:{'[Taylor]': [0, 3], '[sam]': [1, 2]},if 1 role {'[Taylor]': [0]}
+#ref_totals: [0, 3, 1, 2]  if 1 role [0]
 
 def get_ref_character(real_prompt,character_dict):
     list_arr = []

utils/pipeline.py

@@ -12,8 +12,7 @@ from huggingface_hub.utils import validate_hf_hub_args
 from transformers import CLIPImageProcessor, CLIPTokenizer
 from diffusers import StableDiffusionXLPipeline
 try:
-    from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl import StableDiffusionXLPipelineOutput, \
-    rescale_noise_cfg
+    from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl import StableDiffusionXLPipelineOutput,rescale_noise_cfg
 except:
     try:
        from diffusers.pipelines.stable_diffusion_xl import StableDiffusionXLPipelineOutput,rescale_noise_cfg

utils/style_template.py

@@ -1,52 +1,52 @@
 style_list = [
     {
         "name": "No_style",
-        "prompt": "{prompt},8k, RAW photo, best quality,",
+        "prompt": "{prompt} 8k,RAW",
         "negative_prompt": "worst quality",
     },
     {
         "name": "Realistic",
-        "prompt": "{prompt}, (photorealistic:1.4),8k,RAW photo, best quality, masterpiece,dramatic shadows,uhd, high quality,dramatic,cinematic",
+        "prompt": "{prompt}  (photorealistic:1.4),8k,RAW photo, best quality, masterpiece,dramatic shadows,uhd, high quality,dramatic,cinematic",
         "negative_prompt": "(worst quality, low quality, normal quality, lowres), (bad teeth, deformed teeth, deformed lips), (bad proportions), (deformed eyes, bad eyes), (deformed face, ugly face), (deformed hands,fused fingers), morbid, mutilated, mutation, disfigured",
     },
     {
         "name": "Japanese_Anime",
-        "prompt": "{prompt}.(Anime Style, Manga Style:1.3)anime artwork illustrating,best quality,webtoon,manhua,Linear compositions",
+        "prompt": "{prompt} (Anime Style, Manga Style:1.3)anime artwork illustrating,best quality,webtoon,manhua,Linear compositions",
         "negative_prompt": "lowres,text, error, extra digit, fewer digits, cropped, worst quality, low quality, normal quality, jpeg artifacts, signature, watermark, username, blurry",
     },
     {
         "name": "Digital_Oil_Painting",
-        "prompt": "{prompt} . (Extremely Detailed Oil Painting:1.2), glow effects, godrays,render, 8k, octane render, cinema 4d, blender, dark, atmospheric 4k ultra detailed, cinematic sensual,humorous illustration,",
+        "prompt": "{prompt}  (Extremely Detailed Oil Painting:1.2), glow effects, godrays,render, 8k, octane render, cinema 4d, blender, dark, atmospheric 4k ultra detailed, cinematic sensual,humorous illustration ",
         "negative_prompt": "anime, cartoon, graphic, text, crayon, graphite, abstract, glitch, deformed, mutated, ugly, disfigured, lowres, error, extra digit, fewer digits, cropped, worst quality, low quality, normal quality, jpeg artifacts, signature, watermark, username, blurry",
     },
     {
         "name": "Pixar_Disney_Character",
-        "prompt": "Create a Disney Pixar 3D style illustration on {prompt} . The scene is vibrant, motivational, filled with vivid colors and a sense of wonder.",
+        "prompt": "Create a Disney Pixar 3D style illustration on {prompt}  The scene is vibrant, motivational, filled with vivid colors and a sense of wonder.",
         "negative_prompt": "lowres, text, bad eyes, bad legs, error,extra digit, fewer digits, cropped, worst quality, low quality, normal quality, jpeg artifacts, signature, watermark, blurry, grayscale, noisy, sloppy, messy, grainy,ultra textured,",
     },
     {
         "name": "Photographic",
-        "prompt": "cinematic photo {prompt} . Hyperrealistic, Hyperdetailed, detailed skin,soft lighting, best quality, ultra realistic, 8k, golden ratio, Intricate, High Detail, film photography",
+        "prompt": "cinematic photo {prompt}  Hyperrealistic, Hyperdetailed, detailed skin,soft lighting, best quality, ultra realistic, 8k, golden ratio, Intricate, High Detail, film photography",
         "negative_prompt": "drawing, painting, crayon, sketch, graphite, impressionist, noisy, blurry, soft, deformed, ugly, lowres, text, error,  extra digit, fewer digits, cropped, worst quality, low quality,signature, watermark, username",
     },
     {
         "name": "Comic_book",
-        "prompt": "comic {prompt} . graphic illustration, comic art, graphic novel art, vibrant, highly detailed",
+        "prompt": "comic {prompt} graphic illustration, comic art, graphic novel art, vibrant, highly detailed",
         "negative_prompt": "photograph, deformed, glitch, noisy, realistic, stock photo, lowres,text, error,extra digit, fewer digits, cropped, worst quality, low quality, normal quality,signature, watermark, username, blurry",
     },
     {
         "name": "Line_art",
-        "prompt": "line art drawing {prompt} . professional, sleek, modern, minimalist, graphic, line art, vector graphics",
+        "prompt": "line art drawing {prompt}  professional, sleek, modern, minimalist, graphic, line art, vector graphics",
         "negative_prompt": "photorealistic, 35mm film, deformed, glitch, blurry, noisy, off-center, deformed, cross-eyed, closed eyes,ugly, disfigured, mutated, realism, realistic, impressionism, expressionism,acrylic, lowres, text, error,extra digit, fewer digits,  worst quality, low quality, normal quality, jpeg artifacts, signature, watermark, username",
     },
     {
         "name": "Black_and_White_Film_Noir",
-        "prompt": "{prompt} . (b&w, Monochromatic, Film Photography:1.3), film noir, analog style, soft lighting, subsurface scattering,heavy shadow, masterpiece, best quality,",
+        "prompt": "{prompt}  (b&w, Monochromatic, Film Photography:1.3), film noir, analog style, soft lighting, subsurface scattering,heavy shadow, masterpiece, best quality,",
         "negative_prompt": "anime, photorealistic, 35mm film, deformed, glitch, blurry, noisy, off-center, deformed, cross-eyed, closed eyes,ugly, disfigured, mutated, impressionism, expressionism,acrylic, lowres,text, error, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality, normal quality, jpeg artifacts, signature, watermark, username",
     },
     {
         "name": "Isometric_Rooms",
-        "prompt": "Tiny cute isometric {prompt} .in a cutaway box, soft smooth lighting, soft colors, 100mm lens, 3d blender render",
+        "prompt": "Tiny cute isometric {prompt} in a cutaway box, soft smooth lighting, soft colors, 100mm lens, 3d blender render",
         "negative_prompt": "photorealistic,35mm film,deformed,glitch,blurry,noisy,off-center,deformed,cross-eyed,closed eyes,ugly,disfigured,mutated,realism,realistic,impressionism,expressionism,acrylic,lowres,text,error,extra digit,fewer digits,cropped, worst quality, low quality, normal quality, jpeg artifacts, signature,watermark, username,",
     },
 ]