V 0.6.0 - Pic2Story to Recycler

Nodes/Inputs.py

@@ -394,7 +394,7 @@ class PrimereMetaHandler:
         return {
             "required": {
                 "data_source": ("BOOLEAN", {"default": False, "label_on": "Use image meta", "label_off": "Use workflow settings"}),
-                "prompt_surce": ("BOOLEAN", {"default": True, "label_on": "Meta prompt", "label_off": "Workflow prompt"}),
+                "prompt_surce": ("BOOLEAN", {"default": True, "label_on": "Meta or workflow", "label_off": "Pic2story model"}),
                 "prompt_state": ("BOOLEAN", {"default": False, "label_on": "Use decoded prompt", "label_off": "Use dynamic prompt"}),
                 "model": ("BOOLEAN", {"default": True, "label_on": "Meta model", "label_off": "Workflow model"}),
                 "model_hash_check": ("BOOLEAN", {"default": False, "label_on": "Check model hash", "label_off": "Use model name"}),
@@ -723,18 +723,37 @@ class PrimereMetaHandler:
             else:
                 return len(utility.WORKFLOW_SORT_LIST)
 
+        image_file = Path(image_path)
+        if 'image' in kwargs and image_file.is_file() == True:
+            img = nodes.LoadImage.load_image(self, kwargs['image'])[0]
+
+            if kwargs['prompt_surce'] == False and workflow_tuple is None:
+                workflow_tuple = {}
+                workflow_tuple['exif_status'] = 'FAILED'
+
+            if kwargs['prompt_surce'] != False and workflow_tuple is not None:
+                workflow_tuple['pic2story'] = 'OFF'
+
+            if kwargs['prompt_surce'] == False and workflow_tuple is not None:
+                repo_id = "abhijit2111/Pic2Story"
+                prompts = ['Image of', 'Image creation style is', 'Colours on the picture']
+
+                story_out = utility.Pic2Story(repo_id, img, prompts, True, True)
+                if type(story_out) == str:
+                    workflow_tuple['pic2story'] = 'SUCCEED'
+                    workflow_tuple['pic2story_positive'] = story_out
+                else:
+                    workflow_tuple['pic2story'] = 'FAILED'
+
+        else:
+            img = None
+
         if workflow_tuple is not None and len(workflow_tuple) >= 1:
             workflow_tuple = dict(sorted(workflow_tuple.items(), key=lambda pair: DictSort(pair[0])))
             workflow_tuple['setup_states'] = kwargs
             if 'workflow_tuple' in workflow_tuple['setup_states']:
                 del workflow_tuple['setup_states']['workflow_tuple']
 
-        image_file = Path(image_path)
-        if 'image' in kwargs and image_file.is_file() == True:
-            img = nodes.LoadImage.load_image(self, kwargs['image'])[0]
-        else:
-            img = None
-
         return (workflow_tuple, original_exif, img,)
 
 class PrimereMetaDistributor:
@@ -752,7 +771,7 @@ class PrimereMetaDistributor:
         }
 
     def expand_meta(self, workflow_tuple):
-        PROCESSED_KEYS = ['positive', 'negative', 'positive_l', 'negative_l', 'positive_r', 'negative_r',
+        PROCESSED_KEYS = ['pic2story', 'positive', 'negative', 'positive_l', 'negative_l', 'positive_r', 'negative_r',
                           'model', 'model_version', 'model_concept', 'concept_data', 'vae',
                           'sampler', 'scheduler', 'steps', 'cfg',
                           'seed', 'width', 'height']
@@ -761,10 +780,27 @@ class PrimereMetaDistributor:
         if workflow_tuple is not None and type(workflow_tuple).__name__ == 'dict':
             for outputkeys in PROCESSED_KEYS:
                 if outputkeys in workflow_tuple:
-                    output_value = workflow_tuple[outputkeys]
-                    if output_value == "":
-                        output_value = None
-                    OUTPUT_TUPLE.append(output_value)
+                    match outputkeys:
+                        case "pic2story":
+                            if workflow_tuple[outputkeys] == 'SUCCEED' and 'pic2story_positive' in workflow_tuple:
+                                workflow_tuple['positive'] = workflow_tuple['pic2story_positive']
+                                workflow_tuple['prompt_state'] = 'Dynamic'
+                                workflow_tuple['exif_status'] = 'OFF'
+                                if 'decoded_positive' in workflow_tuple:
+                                    del workflow_tuple['decoded_positive']
+                                if 'decoded_negative' in workflow_tuple:
+                                    del workflow_tuple['decoded_negative']
+                                if 'pic2story_positive' in workflow_tuple:
+                                    del workflow_tuple['pic2story_positive']
+                                if 'exif_data_count' in workflow_tuple:
+                                    del workflow_tuple['exif_data_count']
+                                if 'meta_source' in workflow_tuple:
+                                    del workflow_tuple['meta_source']
+                        case _:
+                            output_value = workflow_tuple[outputkeys]
+                            if output_value == "":
+                                output_value = None
+                            OUTPUT_TUPLE.append(output_value)
                 else:
                     MISSING_VALUES = None
                     match outputkeys:

README.md

@@ -20,7 +20,7 @@ Git link: https://github.com/CosmicLaca/ComfyUI_Primere_Nodes
 - Multiply original resolution by integer, but can be define the final resolution by target megapixels from any image sizes. Image resolution multiplier can solve low memory error problem if using Ultimate SD Upscaler 
 - Remove previously included networks from the content of prompts (Embedding, Lora, Lycoris and Hypernetwork), use network remover if the selected model incompatible with them or if you want to try your prompt without included networks or want to change to different, or using SDXL checkpoint and SD Loras have to be changed to SDXL compatible version 
 - Use more than one prompt or style input nodes for testing and developing prompts, select any by 1 click at the 'Prompt Switch' node
-- Special image META/EXIF/PNGINFO reader, which handle model name and samplers from A1111 and ComfyUI .png or .jpg. Never was easier to recycle your older A1111 and ComfyUI images and re-using them with same or different workflow settings. With switches you can change or keep the original meta seed/model/size/etc... to workflow settings. Test workflow: **civitai-image-recycler.json** 
+- Special image META/EXIF/PNGINFO/PIC2PROMPT reader, which handle model name and samplers from A1111 and ComfyUI .png or .jpg. Never was easier to recycle your older A1111 and ComfyUI images and re-using them with same or different workflow settings. With switches you can change or keep the original meta seed/model/size/etc... to workflow settings. Test workflow: **civitai-image-recycler.json** 
 - Workflow and nodes support Lycoris in dedicated node, no need to copy Lycoris files to Loras path
 - Adjustable detailers and refiners for face, eye, hands, mouth, fashion wear, etc..., separated prompt input for detailers can be mixed to original for better result, included test workflow: **civitai-all-refiner.json**
 - Visual (select element by preview image instead of long list) loaders available for Checkpoints, Loras, Lycoris, Embedding, Hypernetworks and .csv prompts. You only have to create preview images to right name and path, see readme details under "Visual", or use 1 click preview creator
@@ -55,6 +55,7 @@ Git link: https://github.com/CosmicLaca/ComfyUI_Primere_Nodes
 
 ## Last changes:
 #### Usually after node changes have to reload/re-wire nodes within existing workflow, or open the latest workflows from the nodepack's **Workflow** folder.
+- Image recycler node read images without meta, using Pic2Story model to generate prompt from picture only 
 - Some nodes moved to **deprecated** subtree. Nodes can be used but not developed in the future.
 - **Segmented refiners** will mesure the aesthetic score of results, and if the original segment is better, changes will be ignored. Only in **Primere_full_workflow.json** workflow. Feature can switch off.
 - Friendly response icons in **segment refiners** if the detailer off or not found segment in the source image.
@@ -293,6 +294,7 @@ Follow the file schema for your own prompts but don't forget to rename the attac
 
 ### Primere image recycler:
 - This node read prompt-exif (called meta) from loaded image. Compatible with A1111 .jpg and .png, and usually with ComfyUI, but not with results of all other custom workflows.
+- The **prompt_source** input can be switch to **Pic2Story** mode, what creating prompt from the loaded image without reading metdata.
 
 <a href="./Workflow/readme_images/pimgrecycler.jpg" target="_blank"><img src="./Workflow/readme_images/pimgrecycler.jpg" height="340px"></a>
 

components/utility.py

@@ -41,10 +41,10 @@ PREVIEW_PATH_BY_TYPE = {
     "Embedding": os.path.join(PREVIEW_ROOT, "embeddings"),
 }
 
-WORKFLOW_SORT_LIST = ['exif_status', 'exif_data_count', 'positive', 'positive_l', 'positive_r', 'negative', 'negative_l', 'negative_r', 'prompt_state', 'decoded_positive', 'decoded_negative', 'model',
-             'model_concept', 'concept_data', 'model_version', 'is_sdxl', 'model_hash', 'vae', 'vae_hash', 'vae_name_sd', 'vae_name_sdxl', 'sampler', 'scheduler', 'steps',
-             'cfg', 'seed', 'width', 'height', 'size_string', 'preferred', 'saved_image_width', 'saved_image_heigth', 'upscaler_ratio',
-             'vae_name_sd', 'vae_name_sdxl']
+WORKFLOW_SORT_LIST = ['exif_status', 'exif_data_count', 'meta_source', 'pic2story', 'positive', 'positive_l', 'positive_r', 'negative', 'negative_l', 'negative_r', 'prompt_state', 'decoded_positive', 'decoded_negative', 'pic2story_positive',
+                      'model', 'model_concept', 'concept_data', 'model_version', 'is_sdxl', 'model_hash', 'vae', 'vae_hash', 'vae_name_sd', 'vae_name_sdxl', 'sampler', 'scheduler', 'steps',
+                      'cfg', 'seed', 'width', 'height', 'size_string', 'preferred', 'saved_image_width', 'saved_image_heigth', 'upscaler_ratio',
+                      'vae_name_sd', 'vae_name_sdxl']
 
 def merge_str_to_tuple(item1, item2):
     if not isinstance(item1, tuple):
@@ -641,6 +641,12 @@ def downloader(from_url, to_path):
             print('ERROR: Cannot download ' + TargetFilename)
             return False
 
+def hf_downloader(repo_id, model_local_dir):
+    from huggingface_hub import snapshot_download
+    model_path = f"{model_local_dir}/{repo_id.split('/')[-1]}"
+    snapshot_download(repo_id=repo_id, local_dir=model_path, local_dir_use_symlinks=True, max_workers=1)
+    return model_path
+
 def ModelConceptNames(ckpt_name, model_concept, lightning_selector, lightning_model_step):
     lora_name = None
     unet_name = None
@@ -840,4 +846,88 @@ def ImageLoaderFromPath(ImgPath, new_width = None, new_height = None):
         #    print('[Primere Error]: error loading image from path: ' + ImgPath)
         #    output_image = None
 
-    return output_image
+    return output_image
+
+def tensor_to_image(tensor):
+    tensor = tensor.cpu()
+    image_np = tensor.squeeze().mul(255).clamp(0, 255).byte().numpy()
+    image = Image.fromarray(image_np, mode='RGB')
+    return image
+
+def Pic2Story(repo_id, img, prompts, special_tokens_skip = True, clean_same_result = True):
+    story_out = None
+    from transformers import BlipProcessor, BlipForConditionalGeneration
+    import torch
+
+    os.environ['TRANSFORMERS_OFFLINE'] = "1"
+    processor = BlipProcessor.from_pretrained(repo_id)
+    pil_image = tensor_to_image(img)
+
+    try:
+        model = BlipForConditionalGeneration.from_pretrained(repo_id, torch_dtype=torch.float16).to("cuda")
+        if type(prompts) == str:
+            inputs = processor(pil_image, prompts, return_tensors="pt").to("cuda", torch.float16)
+            out = model.generate(**inputs)
+            story_out = processor.decode(out[0], skip_special_tokens=special_tokens_skip)
+
+        elif type(prompts).__name__ == 'list':
+            for prompt in prompts:
+                inputs = processor(pil_image, prompt, return_tensors="pt").to("cuda", torch.float16)
+                out = model.generate(**inputs)
+                Processed = processor.decode(out[0], skip_special_tokens=special_tokens_skip) + ', '
+                if story_out is not None:
+                    story_out = story_out + Processed
+                else:
+                    story_out = Processed
+
+    except Exception:
+        print('Pic2Story Float 16 failed')
+
+    if type(story_out) != str:
+        try:
+            model = BlipForConditionalGeneration.from_pretrained(repo_id).to("cuda")
+            if type(prompts) == str:
+                inputs = processor(pil_image, prompts, return_tensors="pt").to("cuda")
+                out = model.generate(**inputs)
+                story_out = processor.decode(out[0], skip_special_tokens=special_tokens_skip)
+
+            elif type(prompts).__name__ == 'list':
+                for prompt in prompts:
+                    inputs = processor(pil_image, prompt, return_tensors="pt").to("cuda")
+                    out = model.generate(**inputs)
+                    Processed = processor.decode(out[0], skip_special_tokens=special_tokens_skip) + ', '
+                    if story_out is not None:
+                        story_out = story_out + Processed
+                    else:
+                        story_out = Processed
+
+        except Exception:
+            print('Pic2Story GPU failed')
+
+    if type(story_out) != str:
+        try:
+            model = BlipForConditionalGeneration.from_pretrained(repo_id)
+            if type(prompts) == str:
+                inputs = processor(pil_image, prompts, return_tensors="pt")
+                out = model.generate(**inputs)
+                story_out = processor.decode(out[0], skip_special_tokens=special_tokens_skip)
+
+            elif type(prompts).__name__ == 'list':
+                for prompt in prompts:
+                    inputs = processor(pil_image, prompt, return_tensors="pt")
+                    out = model.generate(**inputs)
+                    Processed = processor.decode(out[0], skip_special_tokens=special_tokens_skip) + ', '
+                    if story_out is not None:
+                        story_out = story_out + Processed
+                    else:
+                        story_out = Processed
+
+        except Exception:
+            print('Pic2Story CPU failed')
+
+    if type(story_out) == str:
+        if clean_same_result == True:
+            story_out = ' '.join(dict.fromkeys(story_out.split()))
+        return story_out.rstrip(', ').replace(' and ', ' ').replace(' an ', ' ').replace(' is ', ' ').replace(' are ', ' ')
+    else:
+        return story_out

requirements.txt

@@ -32,4 +32,5 @@ tqdm
 transformers
 ultralytics
 yapf
-openai-clip
+openai-clip
+huggingface_hub