From 7d7943fed36388db129a9744850d532df5f8a51a Mon Sep 17 00:00:00 2001 From: maochaojie Date: Thu, 21 Nov 2024 15:41:45 +0800 Subject: [PATCH] modify yaml and workflow --- .../studio/chatbot/models/ace_0.6b_1024.yaml | 1 + .../chatbot/models/ace_0.6b_1024_refiner.yaml | 3 +- .../studio/chatbot/models/ace_0.6b_512.yaml | 3 +- scepter/modules/inference/ace_inference.py | 51 ++++--- scepter/studio/chatbot/chatbot.py | 30 +++-- scepter/studio/chatbot/example.py | 125 ++++++++++-------- scepter/workflow/model_node.py | 33 ++--- 7 files changed, 140 insertions(+), 106 deletions(-) diff --git a/scepter/methods/studio/chatbot/models/ace_0.6b_1024.yaml b/scepter/methods/studio/chatbot/models/ace_0.6b_1024.yaml index ae590af..668dac2 100644 --- a/scepter/methods/studio/chatbot/models/ace_0.6b_1024.yaml +++ b/scepter/methods/studio/chatbot/models/ace_0.6b_1024.yaml @@ -1,5 +1,6 @@ NAME: ACE_0.6B_1024 IS_DEFAULT: False +USE_DYNAMIC_MODEL: False DEFAULT_PARAS: PARAS: # diff --git a/scepter/methods/studio/chatbot/models/ace_0.6b_1024_refiner.yaml b/scepter/methods/studio/chatbot/models/ace_0.6b_1024_refiner.yaml index 7203272..5647bc1 100644 --- a/scepter/methods/studio/chatbot/models/ace_0.6b_1024_refiner.yaml +++ b/scepter/methods/studio/chatbot/models/ace_0.6b_1024_refiner.yaml @@ -1,5 +1,6 @@ NAME: ACE_0.6B_1024_REFINER -IS_DEFAULT: True +IS_DEFAULT: False +USE_DYNAMIC_MODEL: False DEFAULT_PARAS: PARAS: # diff --git a/scepter/methods/studio/chatbot/models/ace_0.6b_512.yaml b/scepter/methods/studio/chatbot/models/ace_0.6b_512.yaml index cc787c9..0872224 100644 --- a/scepter/methods/studio/chatbot/models/ace_0.6b_512.yaml +++ b/scepter/methods/studio/chatbot/models/ace_0.6b_512.yaml @@ -1,5 +1,6 @@ NAME: ACE_0.6B_512 -IS_DEFAULT: False +IS_DEFAULT: True +USE_DYNAMIC_MODEL: False DEFAULT_PARAS: PARAS: # diff --git a/scepter/modules/inference/ace_inference.py b/scepter/modules/inference/ace_inference.py index e0cf84e..fa8b37e 100644 --- a/scepter/modules/inference/ace_inference.py +++ b/scepter/modules/inference/ace_inference.py @@ -87,12 +87,16 @@ class TextEmbedding(nn.Module): class RefinerInference(DiffusionInference): def init_from_cfg(self, cfg): + self.use_dynamic_model = cfg.get('USE_DYNAMIC_MODEL', True) super().init_from_cfg(cfg) self.diffusion = DIFFUSIONS.build(cfg.MODEL.DIFFUSION, logger=self.logger) \ if cfg.MODEL.have('DIFFUSION') else None self.max_seq_length = cfg.MODEL.get("MAX_SEQ_LENGTH", 4096) assert self.diffusion is not None - + if not self.use_dynamic_model: + self.dynamic_load(self.first_stage_model, 'first_stage_model') + self.dynamic_load(self.cond_stage_model, 'cond_stage_model') + self.dynamic_load(self.diffusion_model, 'diffusion_model') @torch.no_grad() def encode_first_stage(self, x, **kwargs): _, dtype = self.get_function_info(self.first_stage_model, 'encode') @@ -152,17 +156,17 @@ class RefinerInference(DiffusionInference): noise.append(noise_) noise, x_shapes = pack_imagelist_into_tensor(noise) if reverse_scale > 0: - self.dynamic_load(self.first_stage_model, 'first_stage_model') + if self.use_dynamic_model: self.dynamic_load(self.first_stage_model, 'first_stage_model') x_samples = [x.unsqueeze(0) for x in x_samples] x_start = self.encode_first_stage(x_samples, **kwargs) - self.dynamic_unload(self.first_stage_model, + if self.use_dynamic_model: self.dynamic_unload(self.first_stage_model, 'first_stage_model', skip_loaded=True) x_start, _ = pack_imagelist_into_tensor(x_start) else: x_start = None # cond stage - self.dynamic_load(self.cond_stage_model, 'cond_stage_model') + if self.use_dynamic_model: self.dynamic_load(self.cond_stage_model, 'cond_stage_model') function_name, dtype = self.get_function_info(self.cond_stage_model) with torch.autocast('cuda', enabled=dtype == 'float16', @@ -170,12 +174,12 @@ class RefinerInference(DiffusionInference): ctx = getattr(get_model(self.cond_stage_model), function_name)(prompt) ctx["x_shapes"] = x_shapes - self.dynamic_unload(self.cond_stage_model, + if self.use_dynamic_model: self.dynamic_unload(self.cond_stage_model, 'cond_stage_model', skip_loaded=True) - self.dynamic_load(self.diffusion_model, 'diffusion_model') + if self.use_dynamic_model: self.dynamic_load(self.diffusion_model, 'diffusion_model') # UNet use input n_prompt function_name, dtype = self.get_function_info( self.diffusion_model) @@ -203,12 +207,12 @@ class RefinerInference(DiffusionInference): x=x_start, **kwargs).float() latent = unpack_tensor_into_imagelist(latent, x_shapes) - self.dynamic_unload(self.diffusion_model, + if self.use_dynamic_model: self.dynamic_unload(self.diffusion_model, 'diffusion_model', skip_loaded=True) - self.dynamic_load(self.first_stage_model, 'first_stage_model') + if self.use_dynamic_model: self.dynamic_load(self.first_stage_model, 'first_stage_model') x_samples = self.decode_first_stage(latent) - self.dynamic_unload(self.first_stage_model, + if self.use_dynamic_model: self.dynamic_unload(self.first_stage_model, 'first_stage_model', skip_loaded=True) return x_samples @@ -227,6 +231,7 @@ class ACEInference(DiffusionInference): def init_from_cfg(self, cfg): self.name = cfg.NAME self.is_default = cfg.get('IS_DEFAULT', False) + self.use_dynamic_model = cfg.get('USE_DYNAMIC_MODEL', True) module_paras = self.load_default(cfg.get('DEFAULT_PARAS', None)) assert cfg.have('MODEL') @@ -250,6 +255,7 @@ class ACEInference(DiffusionInference): # self.refiner_prompt = cfg.get('REFINER_PROMPT', "") self.ace_prompt = cfg.get("ACE_PROMPT", []) if self.refiner_model_cfg: + self.refiner_model_cfg.USE_DYNAMIC_MODEL = self.use_dynamic_model self.refiner_module = RefinerInference(self.logger) self.refiner_module.init_from_cfg(self.refiner_model_cfg) else: @@ -277,6 +283,10 @@ class ACEInference(DiffusionInference): self.size_factor = cfg.get('SIZE_FACTOR', 8) self.decoder_bias = cfg.get('DECODER_BIAS', 0) self.default_n_prompt = cfg.get('DEFAULT_N_PROMPT', '') + if not self.use_dynamic_model: + self.dynamic_load(self.first_stage_model, 'first_stage_model') + self.dynamic_load(self.cond_stage_model, 'cond_stage_model') + self.dynamic_load(self.diffusion_model, 'diffusion_model') @torch.no_grad() def encode_first_stage(self, x, **kwargs): @@ -388,9 +398,9 @@ class ACEInference(DiffusionInference): if use_ace and (not is_txt_image or refiner_scale <= 0): ctx, null_ctx = {}, {} # Get Noise Shape - self.dynamic_load(self.first_stage_model, 'first_stage_model') + if self.use_dynamic_model: self.dynamic_load(self.first_stage_model, 'first_stage_model') x = self.encode_first_stage(image) - self.dynamic_unload(self.first_stage_model, + if self.use_dynamic_model: self.dynamic_unload(self.first_stage_model, 'first_stage_model', skip_loaded=True) noise = [ @@ -406,7 +416,7 @@ class ACEInference(DiffusionInference): ctx['x_mask'] = null_ctx['x_mask'] = cond_mask # Encode Prompt - self.dynamic_load(self.cond_stage_model, 'cond_stage_model') + if self.use_dynamic_model: self.dynamic_load(self.cond_stage_model, 'cond_stage_model') function_name, dtype = self.get_function_info(self.cond_stage_model) cont, cont_mask = getattr(get_model(self.cond_stage_model), function_name)(prompt) @@ -416,14 +426,14 @@ class ACEInference(DiffusionInference): function_name)(n_prompt) null_cont, null_cont_mask = self.cond_stage_embeddings( prompt, edit_image, null_cont, null_cont_mask) - self.dynamic_unload(self.cond_stage_model, + if self.use_dynamic_model: self.dynamic_unload(self.cond_stage_model, 'cond_stage_model', skip_loaded=False) ctx['crossattn'] = cont null_ctx['crossattn'] = null_cont # Encode Edit Images - self.dynamic_load(self.first_stage_model, 'first_stage_model') + if self.use_dynamic_model: self.dynamic_load(self.first_stage_model, 'first_stage_model') edit_image = [to_device(i, strict=False) for i in edit_image] edit_image_mask = [to_device(i, strict=False) for i in edit_image_mask] e_img, e_mask = [], [] @@ -434,14 +444,14 @@ class ACEInference(DiffusionInference): m = [None] * len(u) e_img.append(self.encode_first_stage(u, **kwargs)) e_mask.append([self.interpolate_func(i) for i in m]) - self.dynamic_unload(self.first_stage_model, + if self.use_dynamic_model: self.dynamic_unload(self.first_stage_model, 'first_stage_model', skip_loaded=True) null_ctx['edit'] = ctx['edit'] = e_img null_ctx['edit_mask'] = ctx['edit_mask'] = e_mask # Diffusion Process - self.dynamic_load(self.diffusion_model, 'diffusion_model') + if self.use_dynamic_model: self.dynamic_load(self.diffusion_model, 'diffusion_model') function_name, dtype = self.get_function_info(self.diffusion_model) with torch.autocast('cuda', enabled=dtype in ('float16', 'bfloat16'), @@ -482,15 +492,15 @@ class ACEInference(DiffusionInference): guide_rescale=guide_rescale, return_intermediate=None, **kwargs) - self.dynamic_unload(self.diffusion_model, + if self.use_dynamic_model: self.dynamic_unload(self.diffusion_model, 'diffusion_model', skip_loaded=False) # Decode to Pixel Space - self.dynamic_load(self.first_stage_model, 'first_stage_model') + if self.use_dynamic_model: self.dynamic_load(self.first_stage_model, 'first_stage_model') samples = unpack_tensor_into_imagelist(latent, x_shapes) x_samples = self.decode_first_stage(samples) - self.dynamic_unload(self.first_stage_model, + if self.use_dynamic_model: self.dynamic_unload(self.first_stage_model, 'first_stage_model', skip_loaded=False) x_samples = [x.squeeze(0) for x in x_samples] @@ -509,7 +519,8 @@ class ACEInference(DiffusionInference): x_samples = self.refiner_module.refine(x_samples, reverse_scale = input_refine_scale, prompt= input_refine_prompt, - seed=seed) + seed=seed, + use_dynamic_model=self.use_dynamic_model) imgs = [ torch.clamp((x_i.float() + 1.0) / 2.0 + self.decoder_bias / 255, diff --git a/scepter/studio/chatbot/chatbot.py b/scepter/studio/chatbot/chatbot.py index 7bf29bc..c94c88f 100644 --- a/scepter/studio/chatbot/chatbot.py +++ b/scepter/studio/chatbot/chatbot.py @@ -55,8 +55,10 @@ class ChatBotUI(object): from diffusers.utils import export_to_video except Exception as e: print(f"Import diffusers failed, please install or upgrade diffusers. Error information: {e}") - - cfg = Config(cfg_file=cfg_general_file) + if isinstance(cfg_general_file, str): + cfg = Config(cfg_file=cfg_general_file) + else: + cfg = cfg_general_file cfg.WORK_DIR = os.path.join(root_work_dir, cfg.WORK_DIR) if not FS.exists(cfg.WORK_DIR): FS.make_dir(cfg.WORK_DIR) @@ -75,7 +77,7 @@ class ChatBotUI(object): self.model_choices[model_name] = model_cfg print('Models: ', self.model_choices.keys()) assert len(self.model_choices) > 0 - if self.default_model_name == "": self.default_model_name = self.model_choices.keys()[0] + if self.default_model_name == "": self.default_model_name = list(self.model_choices.keys())[0] self.model_name = self.default_model_name self.pipe = ACEInference() self.pipe.init_from_cfg(self.model_choices[self.default_model_name]) @@ -383,7 +385,7 @@ class ChatBotUI(object): label='Rescale') self.refiner_scale = gr.Slider(minimum=-0.1, maximum=1.0, - value=self.pipe.input.get("refiner_scale", 0.5), + value=self.pipe.input.get("refiner_scale", -1), visible=self.pipe.input.get("refiner_scale", None) is not None, label='Refiner Scale') self.seed = gr.Slider(minimum=-1, @@ -530,6 +532,7 @@ class ChatBotUI(object): lock.acquire() del self.pipe torch.cuda.empty_cache() + torch.cuda.ipc_collect() self.pipe = ACEInference() self.pipe.init_from_cfg(self.model_choices[model_name]) self.model_name = model_name @@ -555,7 +558,7 @@ class ChatBotUI(object): value=self.pipe.input.get("refiner_prompt", ""), visible=self.pipe.input.get("refiner_prompt", None) is not None), gr.Slider( - value=self.pipe.input.get("refiner_scale", 0.5), + value=self.pipe.input.get("refiner_scale", -1), visible=self.pipe.input.get("refiner_scale", None) is not None ), gr.Checkbox( @@ -996,9 +999,13 @@ class ChatBotUI(object): w = int(w / ratio) img = img.resize((w, h)) edit_image.append(img) + if img_mask is not None: + img_mask = img_mask if np.sum(np.array(img_mask)) > 0 else None edit_image_mask.append( img_mask if img_mask is not None else None) edit_task.append(task) + if ref1 is not None: + ref1 = ref1 if np.sum(np.array(ref1)) > 0 else None if ref1 is not None: edit_image.append(ref1) edit_image_mask.append(None) @@ -1034,8 +1041,13 @@ class ChatBotUI(object): img_str = f'' history = [(prompt, f'{pre_info} The generated image is:\n {img_str}')] + + img_id = get_md5(img_b64)[:12] + save_path = os.path.join(self.cache_dir, f'{img_id}.png') + img.convert('RGB').save(save_path) + return self.get_history(history), gr.update(value=''), gr.update( - visible=False), gr.update(value=-1) + visible=False), gr.Image(value=save_path), gr.update(value=-1) with self.eg: self.example_task = gr.Text(label='Task Name', @@ -1061,8 +1073,9 @@ class ChatBotUI(object): self.example_task, self.example_image, self.example_mask, self.example_ref_im1, self.text, self.seed ], - outputs=[self.chatbot, self.text, self.gallery, self.seed], + outputs=[self.chatbot, self.text, self.gallery, self.legacy_image_viewer, self.seed], examples_per_page=4, + cache_examples=False, run_on_click=True) ######################################## @@ -1390,7 +1403,7 @@ class ChatBotUI(object): def run_gr(cfg): with gr.Blocks() as demo: chatbot = ChatBotUI(cfg) - chatbot.create_bot_ui() + chatbot.create_ui() chatbot.set_callbacks() demo.launch(server_name='0.0.0.0', server_port=cfg.args.server_port, @@ -1402,6 +1415,7 @@ if __name__ == '__main__': parser.add_argument('--server_port', dest='server_port', help='', + type=int, default=2345) parser.add_argument('--root_path', dest='root_path', help='', default='') cfg = Config(load=True, parser_ins=parser) diff --git a/scepter/studio/chatbot/example.py b/scepter/studio/chatbot/example.py index b0ef5a0..4c5878e 100644 --- a/scepter/studio/chatbot/example.py +++ b/scepter/studio/chatbot/example.py @@ -3,6 +3,7 @@ import os from scepter.modules.utils.file_system import FS +from PIL import Image def download_image(image, local_path=None): @@ -10,44 +11,56 @@ def download_image(image, local_path=None): local_path = FS.get_from(image, local_path=local_path) return local_path +def blank_image(): + return Image.new('RGBA', (128, 128), (0, 0, 0, 0)) + + def get_examples(cache_dir): print('Downloading Examples ...') + bl_img = blank_image() examples = [ [ 'Facial Editing', download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/e33edc106953.png?raw=true', - os.path.join(cache_dir, 'examples/e33edc106953.png')), None, - None, '{image} let the man smile', 6666 + os.path.join(cache_dir, 'examples/e33edc106953.png')), bl_img, + bl_img, '{image} let the man smile', 6666 ], [ 'Facial Editing', download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/5d2bcc91a3e9.png?raw=true', - os.path.join(cache_dir, 'examples/5d2bcc91a3e9.png')), None, - None, 'let the man in {image} wear sunglasses', 9999 + os.path.join(cache_dir, 'examples/5d2bcc91a3e9.png')), bl_img, + bl_img, 'let the man in {image} wear sunglasses', 9999 + ], + [ + 'Facial Editing', + download_image( + 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/5d2bcc91a3e9.png?raw=true', + os.path.join(cache_dir, 'examples/5d2bcc91a3e9.png')), bl_img, + bl_img, 'let the man in {image} wear sunglasses', 9999 ], [ 'Facial Editing', download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/3a52eac708bd.png?raw=true', - os.path.join(cache_dir, 'examples/3a52eac708bd.png')), None, - None, '{image} red hair', 9999 + os.path.join(cache_dir, 'examples/3a52eac708bd.png')), bl_img, + bl_img, '{image} red hair', 9999 ], [ 'Facial Editing', download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/3f4dc464a0ea.png?raw=true', - os.path.join(cache_dir, 'examples/3f4dc464a0ea.png')), None, - None, '{image} let the man serious', 99999 + os.path.join(cache_dir, 'examples/3f4dc464a0ea.png')), bl_img, + bl_img, '{image} let the man serious', 99999 ], [ 'Controllable Generation', download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/131ca90fd2a9.png?raw=true', os.path.join(cache_dir, - 'examples/131ca90fd2a9.png')), None, None, + 'examples/131ca90fd2a9.png')), bl_img, bl_img, '"A person sits contemplatively on the ground, surrounded by falling autumn leaves. Dressed in a green sweater and dark blue pants, they rest their chin on their hand, exuding a relaxed demeanor. Their stylish checkered slip-on shoes add a touch of flair, while a black purse lies in their lap. The backdrop of muted brown enhances the warm, cozy atmosphere of the scene." , generate the image that corresponds to the given scribble {image}.', 613725 ], @@ -59,7 +72,7 @@ def get_examples(cache_dir): download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/33e9f27c2c48_mask.png?raw=true', os.path.join(cache_dir, - 'examples/33e9f27c2c48_mask.png')), None, + 'examples/33e9f27c2c48_mask.png')), bl_img, 'Put the text "C A T" at the position marked by mask in the {image}', 6666 ], @@ -67,7 +80,7 @@ def get_examples(cache_dir): 'Style Transfer', download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/9e73e7eeef55.png?raw=true', - os.path.join(cache_dir, 'examples/9e73e7eeef55.png')), None, + os.path.join(cache_dir, 'examples/9e73e7eeef55.png')), bl_img, download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/2e02975293d6.png?raw=true', os.path.join(cache_dir, 'examples/2e02975293d6.png')), @@ -81,7 +94,7 @@ def get_examples(cache_dir): download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/f2b22c08be3f_mask.png?raw=true', os.path.join(cache_dir, - 'examples/f2b22c08be3f_mask.png')), None, + 'examples/f2b22c08be3f_mask.png')), bl_img, 'Could the {image} be widened within the space designated by mask, while retaining the original?', 6666 ], @@ -89,57 +102,57 @@ def get_examples(cache_dir): 'Image Segmentation', download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/db3ebaa81899.png?raw=true', - os.path.join(cache_dir, 'examples/db3ebaa81899.png')), None, - None, '{image} Segmentation', 6666 + os.path.join(cache_dir, 'examples/db3ebaa81899.png')), bl_img, + bl_img, '{image} Segmentation', 6666 ], [ 'Depth Estimation', download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/f1927c4692ba.png?raw=true', - os.path.join(cache_dir, 'examples/f1927c4692ba.png')), None, - None, '{image} Depth Estimation', 6666 + os.path.join(cache_dir, 'examples/f1927c4692ba.png')), bl_img, + bl_img, '{image} Depth Estimation', 6666 ], [ 'Pose Estimation', download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/014e5bf3b4d1.png?raw=true', - os.path.join(cache_dir, 'examples/014e5bf3b4d1.png')), None, - None, '{image} distinguish the poses of the figures', 999999 + os.path.join(cache_dir, 'examples/014e5bf3b4d1.png')), bl_img, + bl_img, '{image} distinguish the poses of the figures', 999999 ], [ 'Scribble Extraction', download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/5f59a202f8ac.png?raw=true', - os.path.join(cache_dir, 'examples/5f59a202f8ac.png')), None, - None, 'Generate a scribble of {image}, please.', 6666 + os.path.join(cache_dir, 'examples/5f59a202f8ac.png')), bl_img, + bl_img, 'Generate a scribble of {image}, please.', 6666 ], [ 'Mosaic', download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/3a2f52361eea.png?raw=true', - os.path.join(cache_dir, 'examples/3a2f52361eea.png')), None, - None, 'Adapt {image} into a mosaic representation.', 6666 + os.path.join(cache_dir, 'examples/3a2f52361eea.png')), bl_img, + bl_img, 'Adapt {image} into a mosaic representation.', 6666 ], [ 'Edge map Extraction', download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/b9d1e519d6e5.png?raw=true', - os.path.join(cache_dir, 'examples/b9d1e519d6e5.png')), None, - None, 'Get the edge-enhanced result for {image}.', 6666 + os.path.join(cache_dir, 'examples/b9d1e519d6e5.png')), bl_img, + bl_img, 'Get the edge-enhanced result for {image}.', 6666 ], [ 'Grayscale', download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/c4ebbe2ba29b.png?raw=true', - os.path.join(cache_dir, 'examples/c4ebbe2ba29b.png')), None, - None, 'transform {image} into a black and white one', 6666 + os.path.join(cache_dir, 'examples/c4ebbe2ba29b.png')), bl_img, + bl_img, 'transform {image} into a black and white one', 6666 ], [ 'Contour Extraction', download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/19652d0f6c4b.png?raw=true', os.path.join(cache_dir, - 'examples/19652d0f6c4b.png')), None, None, + 'examples/19652d0f6c4b.png')), bl_img, bl_img, 'Would you be able to make a contour picture from {image} for me?', 6666 ], @@ -148,7 +161,7 @@ def get_examples(cache_dir): download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/249cda2844b7.png?raw=true', os.path.join(cache_dir, - 'examples/249cda2844b7.png')), None, None, + 'examples/249cda2844b7.png')), bl_img, bl_img, 'Following the segmentation outcome in mask of {image}, develop a real-life image using the explanatory note in "a mighty cat lying on the bed”.', 6666 ], @@ -157,7 +170,7 @@ def get_examples(cache_dir): download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/411f6c4b8e6c.png?raw=true', os.path.join(cache_dir, - 'examples/411f6c4b8e6c.png')), None, None, + 'examples/411f6c4b8e6c.png')), bl_img, bl_img, 'use the depth map {image} and the text caption "a cut white cat" to create a corresponding graphic image', 999999 ], @@ -166,7 +179,7 @@ def get_examples(cache_dir): download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/a35c96ed137a.png?raw=true', os.path.join(cache_dir, - 'examples/a35c96ed137a.png')), None, None, + 'examples/a35c96ed137a.png')), bl_img, bl_img, 'help translate this posture schema {image} into a colored image based on the context I provided "A beautiful woman Climbing the climbing wall, wearing a harness and climbing gear, skillfully maneuvering up the wall with her back to the camera, with a safety rope."', 3599999 ], @@ -175,7 +188,7 @@ def get_examples(cache_dir): download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/dcb2fc86f1ce.png?raw=true', os.path.join(cache_dir, - 'examples/dcb2fc86f1ce.png')), None, None, + 'examples/dcb2fc86f1ce.png')), bl_img, bl_img, 'Transform and generate an image using mosaic {image} and "Monarch butterflies gracefully perch on vibrant purple flowers, showcasing their striking orange and black wings in a lush garden setting." description', 6666 ], @@ -184,7 +197,7 @@ def get_examples(cache_dir): download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/4cd4ee494962.png?raw=true', os.path.join(cache_dir, - 'examples/4cd4ee494962.png')), None, None, + 'examples/4cd4ee494962.png')), bl_img, bl_img, 'make this {image} colorful as per the "beautiful sunflowers"', 6666 ], @@ -193,7 +206,7 @@ def get_examples(cache_dir): download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/a47e3a9cd166.png?raw=true', os.path.join(cache_dir, - 'examples/a47e3a9cd166.png')), None, None, + 'examples/a47e3a9cd166.png')), bl_img, bl_img, 'Take the edge conscious {image} and the written guideline "A whimsical animated character is depicted holding a delectable cake adorned with blue and white frosting and a drizzle of chocolate. The character wears a yellow headband with a bow, matching a cozy yellow sweater. Her dark hair is styled in a braid, tied with a yellow ribbon. With a golden fork in hand, she stands ready to enjoy a slice, exuding an air of joyful anticipation. The scene is creatively rendered with a charming and playful aesthetic." and produce a realistic image.', 613725 ], @@ -202,7 +215,7 @@ def get_examples(cache_dir): download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/d890ed8a3ac2.png?raw=true', os.path.join(cache_dir, - 'examples/d890ed8a3ac2.png')), None, None, + 'examples/d890ed8a3ac2.png')), bl_img, bl_img, 'creating a vivid image based on {image} and description "This image features a delicious rectangular tart with a flaky, golden-brown crust. The tart is topped with evenly sliced tomatoes, layered over a creamy cheese filling. Aromatic herbs are sprinkled on top, adding a touch of green and enhancing the visual appeal. The background includes a soft, textured fabric and scattered white flowers, creating an elegant and inviting presentation. Bright red tomatoes in the upper right corner hint at the fresh ingredients used in the dish."', 6666 ], @@ -211,7 +224,7 @@ def get_examples(cache_dir): download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/0844a686a179.png?raw=true', os.path.join(cache_dir, - 'examples/0844a686a179.png')), None, None, + 'examples/0844a686a179.png')), bl_img, bl_img, 'Eliminate noise interference in {image} and maximize the crispness to obtain superior high-definition quality', 6666 ], @@ -223,7 +236,7 @@ def get_examples(cache_dir): download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/fa91b6b7e59b_mask.png?raw=true', os.path.join(cache_dir, - 'examples/fa91b6b7e59b_mask.png')), None, + 'examples/fa91b6b7e59b_mask.png')), bl_img, 'Ensure to overhaul the parts of the {image} indicated by the mask.', 6666 ], @@ -235,7 +248,7 @@ def get_examples(cache_dir): download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/632899695b26_mask.png?raw=true', os.path.join(cache_dir, - 'examples/632899695b26_mask.png')), None, + 'examples/632899695b26_mask.png')), bl_img, 'Refashion the mask portion of {image} in accordance with "A yellow egg with a smiling face painted on it"', 6666 ], @@ -244,7 +257,7 @@ def get_examples(cache_dir): download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/354d17594afe.png?raw=true', os.path.join(cache_dir, - 'examples/354d17594afe.png')), None, None, + 'examples/354d17594afe.png')), bl_img, bl_img, '{image} change the dog\'s posture to walking in the water, and change the background to green plants and a pond.', 6666 ], @@ -253,7 +266,7 @@ def get_examples(cache_dir): download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/38946455752b.png?raw=true', os.path.join(cache_dir, - 'examples/38946455752b.png')), None, None, + 'examples/38946455752b.png')), bl_img, bl_img, '{image} change the color of the dress from white to red and the model\'s hair color red brown to blonde.Other parts remain unchanged', 6669 ], @@ -262,7 +275,7 @@ def get_examples(cache_dir): download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/3ba5202f0cd8.png?raw=true', os.path.join(cache_dir, - 'examples/3ba5202f0cd8.png')), None, None, + 'examples/3ba5202f0cd8.png')), bl_img, bl_img, 'Keep the same facial feature in @3ba5202f0cd8, change the woman\'s clothing from a Blue denim jacket to a white turtleneck sweater and adjust her posture so that she is supporting her chin with both hands. Other aspects, such as background, hairstyle, facial expression, etc, remain unchanged.', 99999 ], @@ -270,22 +283,22 @@ def get_examples(cache_dir): 'Facial Editing', download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/369365b94725.png?raw=true', - os.path.join(cache_dir, 'examples/369365b94725.png')), None, - None, '{image} Make her looking at the camera', 6666 + os.path.join(cache_dir, 'examples/369365b94725.png')), bl_img, + bl_img, '{image} Make her looking at the camera', 6666 ], [ 'Facial Editing', download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/92751f2e4a0e.png?raw=true', - os.path.join(cache_dir, 'examples/92751f2e4a0e.png')), None, - None, '{image} Remove the smile from his face', 9899999 + os.path.join(cache_dir, 'examples/92751f2e4a0e.png')), bl_img, + bl_img, '{image} Remove the smile from his face', 9899999 ], [ 'Remove Text', download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/8530a6711b2e.png?raw=true', - os.path.join(cache_dir, 'examples/8530a6711b2e.png')), None, - None, 'Aim to remove any textual element in {image}', 6666 + os.path.join(cache_dir, 'examples/8530a6711b2e.png')), bl_img, + bl_img, 'Aim to remove any textual element in {image}', 6666 ], [ 'Remove Text', @@ -295,7 +308,7 @@ def get_examples(cache_dir): download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/c4d7fb28f8f6_mask.png?raw=true', os.path.join(cache_dir, - 'examples/c4d7fb28f8f6_mask.png')), None, + 'examples/c4d7fb28f8f6_mask.png')), bl_img, 'Rub out any text found in the mask sector of the {image}.', 6666 ], [ @@ -303,7 +316,7 @@ def get_examples(cache_dir): download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/e2f318fa5e5b.png?raw=true', os.path.join(cache_dir, - 'examples/e2f318fa5e5b.png')), None, None, + 'examples/e2f318fa5e5b.png')), bl_img, bl_img, 'Remove the unicorn in this {image}, ensuring a smooth edit.', 99999 ], @@ -315,7 +328,7 @@ def get_examples(cache_dir): download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/1ae96d8aca00_mask.png?raw=true', os.path.join(cache_dir, 'examples/1ae96d8aca00_mask.png')), - None, 'Discard the contents of the mask area from {image}.', 99999 + bl_img, 'Discard the contents of the mask area from {image}.', 99999 ], [ 'Add Object', @@ -325,22 +338,22 @@ def get_examples(cache_dir): download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/80289f48e511_mask.png?raw=true', os.path.join(cache_dir, - 'examples/80289f48e511_mask.png')), None, + 'examples/80289f48e511_mask.png')), bl_img, 'add a Hot Air Balloon into the {image}, per the mask', 613725 ], [ 'Style Transfer', download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/d725cb2009e8.png?raw=true', - os.path.join(cache_dir, 'examples/d725cb2009e8.png')), None, - None, 'Change the style of {image} to colored pencil style', 99999 + os.path.join(cache_dir, 'examples/d725cb2009e8.png')), bl_img, + bl_img, 'Change the style of {image} to colored pencil style', 99999 ], [ 'Style Transfer', download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/e0f48b3fd010.png?raw=true', - os.path.join(cache_dir, 'examples/e0f48b3fd010.png')), None, - None, 'make {image} to Walt Disney Animation style', 99999 + os.path.join(cache_dir, 'examples/e0f48b3fd010.png')), bl_img, + bl_img, 'make {image} to Walt Disney Animation style', 99999 ], [ 'Try On', @@ -359,8 +372,8 @@ def get_examples(cache_dir): 'Workflow', download_image( 'https://github.com/ali-vilab/ace-page/blob/main/assets/examples/cb85353c004b.png?raw=true', - os.path.join(cache_dir, 'examples/cb85353c004b.png')), None, - None, ' ice cream {image}', 99999 + os.path.join(cache_dir, 'examples/cb85353c004b.png')), bl_img, + bl_img, ' ice cream {image}', 99999 ], ] print('Finish. Start building UI ...') diff --git a/scepter/workflow/model_node.py b/scepter/workflow/model_node.py index 99c6555..171a9ac 100644 --- a/scepter/workflow/model_node.py +++ b/scepter/workflow/model_node.py @@ -39,8 +39,8 @@ class ModelNode: 'mantras': ('CONDITIONING', ), 'tuners': ('CONDITIONING', ), 'controls': ('CONDITIONING', ), - 'image': ('IMAGE',), - 'mask': ('MASK',) + 'image': ('IMAGE', ), + 'mask': ('MASK', ) } } @@ -64,15 +64,17 @@ class ModelNode: image = [TT.ToPILImage()(image.squeeze(0).permute(2, 0, 1))] if mask is not None: mask = [TT.ToPILImage()(mask.squeeze(0))] - data = self.format_parameters(model, model_source, prompt, negative_prompt, - parameters, mantras, tuners, controls, image, mask) + data = self.format_parameters(model, model_source, prompt, + negative_prompt, parameters, mantras, + tuners, controls, image, mask) cfg = self.model_file.get(model)['config'] cfg = self.source_mapping(cfg, model_source) self.init_infer(model, cfg) if model.startswith('ACE'): output = self.diff_infer(**data[0], **data[1]) - output_image = torch.stack([ TT.ToTensor()(img) for img in output]).permute(0, 2, 3, 1).unsqueeze(0) + output_image = torch.stack([TT.ToTensor()(img) for img in output + ]).permute(0, 2, 3, 1).unsqueeze(0) else: output = self.diff_infer(data[0], **data[1]) x = output['images'].permute(0, 2, 3, 1) @@ -94,13 +96,13 @@ class ModelNode: elif source == 'Local': cfg_new = copy.deepcopy(cfg) cfg_new.MODEL = cfg_new.MODEL_LOCAL - if hasattr(cfg_new, 'EFINER_MODEL_LOCAL'): + if hasattr(cfg_new, 'REFINER_MODEL_LOCAL'): cfg_new.REFINER_MODEL = cfg_new.REFINER_MODEL_LOCAL return cfg_new elif source == 'HuggingFace': cfg_new = copy.deepcopy(cfg) cfg_new.MODEL = cfg_new.MODEL_HF - if hasattr(cfg_new, 'EFINER_MODEL_HF'): + if hasattr(cfg_new, 'REFINER_MODEL_HF'): cfg_new.REFINER_MODEL = cfg_new.REFINER_MODEL_HF return cfg_new else: @@ -147,17 +149,8 @@ class ModelNode: self.pipeline[model_name] = diff_infer self.diff_infer = diff_infer - def format_parameters(self, - model, - model_source, - prompt, - negative_prompt, - parameters, - mantras, - tuners, - controls, - image, - mask): + def format_parameters(self, model, model_source, prompt, negative_prompt, + parameters, mantras, tuners, controls, image, mask): input_data = {'prompt': prompt, 'negative_prompt': negative_prompt} input_params = { 'diffusion_model': self.model_file.get(model)['diffusion_model'], @@ -167,10 +160,10 @@ class ModelNode: } if image is not None: - input_data.update({"image": image}) + input_data.update({'image': image}) if mask is not None: - input_data.update({"mask": mask}) + input_data.update({'mask': mask}) if parameters: seed = parameters.pop('seed', -1)