working from the commandline

Various changes made to make it work from the command line - Most of the functionality has been commented out - Gradio-specific code - HTML UI - Video processing - Operation modes other than mask generation
2024-09-29 20:39:16 +00:00
parent 6cb0f99037
commit 014a3ad88d
13 changed files with 384 additions and 286 deletions
@@ -6,3 +6,6 @@ __pycache__/

 .*.sw?
 *~
+
+# something on my MacOS keeps saving these resource forks for every single file --rdancer
+.*
@@ -14,3 +14,7 @@ Check out the configuration reference at https://huggingface.co/docs/hub/spaces-

 Code in this directory and its subdirectories based on SkalskiP's Florence 2 +
 SAM 2 space at https://huggingface.co/spaces/SkalskiP/florence-sam
+
+## Testing
+
+Run `python test.py test/*_source.png "products"` 👉 the resulting images must pixel-match `test/*_output.png`.
@@ -0,0 +1,53 @@
+# from app import process_image
+
+__all__ = ["process_image"]
+
+
+if __name__ == "__main__":
+    # detect which parameters are filenames -- those are images
+    # the rest are prompts
+    # call process_image with the images and prompts
+    # save the output images
+    # return the output images' filenames
+    import sys
+    import os
+    import argparse
+    from PIL import Image
+
+    def my_process_image(image_path, prompt):
+        image = Image.open(image_path).convert("RGB")
+        output_image, _ = process_image(image, prompt)
+        output_path = f"output_{os.path.basename(image_path)}"
+        output_image.save(output_path)
+        return output_path
+
+    if len(sys.argv) < 2:
+        print(f"Usage: python {os.path.basename(sys.argv[0])} <image_path>[ ...] [<prompt>]")
+        sys.exit(1)
+
+    # test which exist as filenames
+    images = []
+    prompts = []
+
+    for arg in sys.argv[1:]:
+        if not os.path.exists(arg):
+            prompts.append(arg)
+        else:
+            images.append(arg)
+    
+    if len(prompts) > 1:
+        raise ValueError("At most one prompt is required")
+    if len(images) < 1:
+        raise ValueError("At least one image is required")
+    
+    prompt = prompts[0].strip() if prompts else None
+
+    print(f"Processing {len(images)} image{'' if len(images) == 1 else 's'} with prompt: {prompt}")
+
+    from app import process_image
+
+    for image in images:
+        output_path = my_process_image(image, prompt)
+    print(f"Output image saved to {output_path}")
+else:
+    from app import process_image
@@ -12,51 +12,51 @@ from tqdm import tqdm
 from utils.video import generate_unique_name, create_directory, delete_directory

 from utils.florence import load_florence_model, run_florence_inference, \
-    FLORENCE_DETAILED_CAPTION_TASK, \
-    FLORENCE_CAPTION_TO_PHRASE_GROUNDING_TASK, FLORENCE_OPEN_VOCABULARY_DETECTION_TASK
-from utils.modes import IMAGE_INFERENCE_MODES, IMAGE_OPEN_VOCABULARY_DETECTION_MODE, \
-    IMAGE_CAPTION_GROUNDING_MASKS_MODE, VIDEO_INFERENCE_MODES
-from utils.sam import load_sam_image_model, run_sam_inference, load_sam_video_model
+    FLORENCE_OPEN_VOCABULARY_DETECTION_TASK #,
+    # FLORENCE_DETAILED_CAPTION_TASK, FLORENCE_CAPTION_TO_PHRASE_GROUNDING_TASK
+from utils.modes import IMAGE_INFERENCE_MODES, IMAGE_OPEN_VOCABULARY_DETECTION_MODE #, \
+    # IMAGE_CAPTION_GROUNDING_MASKS_MODE, VIDEO_INFERENCE_MODES
+from utils.sam import load_sam_image_model, run_sam_inference #, load_sam_video_model

-MARKDOWN = """
-# Florence2 + SAM2 🔥
+# MARKDOWN = """
+# # Florence2 + SAM2 🔥

-<div>
-    <a href="https://github.com/facebookresearch/segment-anything-2">
-        <img src="https://badges.aleen42.com/src/github.svg" alt="GitHub" style="display:inline-block;">
-    </a>
-    <a href="https://colab.research.google.com/github/roboflow-ai/notebooks/blob/main/notebooks/how-to-segment-images-with-sam-2.ipynb">
-        <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Colab" style="display:inline-block;">
-    </a>
-    <a href="https://blog.roboflow.com/what-is-segment-anything-2/">
-        <img src="https://raw.githubusercontent.com/roboflow-ai/notebooks/main/assets/badges/roboflow-blogpost.svg" alt="Roboflow" style="display:inline-block;">
-    </a>
-    <a href="https://www.youtube.com/watch?v=Dv003fTyO-Y">
-        <img src="https://badges.aleen42.com/src/youtube.svg" alt="YouTube" style="display:inline-block;">
-    </a>
-</div>
+# <div>
+#     <a href="https://github.com/facebookresearch/segment-anything-2">
+#         <img src="https://badges.aleen42.com/src/github.svg" alt="GitHub" style="display:inline-block;">
+#     </a>
+#     <a href="https://colab.research.google.com/github/roboflow-ai/notebooks/blob/main/notebooks/how-to-segment-images-with-sam-2.ipynb">
+#         <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Colab" style="display:inline-block;">
+#     </a>
+#     <a href="https://blog.roboflow.com/what-is-segment-anything-2/">
+#         <img src="https://raw.githubusercontent.com/roboflow-ai/notebooks/main/assets/badges/roboflow-blogpost.svg" alt="Roboflow" style="display:inline-block;">
+#     </a>
+#     <a href="https://www.youtube.com/watch?v=Dv003fTyO-Y">
+#         <img src="https://badges.aleen42.com/src/youtube.svg" alt="YouTube" style="display:inline-block;">
+#     </a>
+# </div>

-This demo integrates Florence2 and SAM2 by creating a two-stage inference pipeline. In 
-the first stage, Florence2 performs tasks such as object detection, open-vocabulary 
-object detection, image captioning, or phrase grounding. In the second stage, SAM2 
-performs object segmentation on the image.
-"""
+# This demo integrates Florence2 and SAM2 by creating a two-stage inference pipeline. In 
+# the first stage, Florence2 performs tasks such as object detection, open-vocabulary 
+# object detection, image captioning, or phrase grounding. In the second stage, SAM2 
+# performs object segmentation on the image.
+# """

-IMAGE_PROCESSING_EXAMPLES = [
-    [IMAGE_OPEN_VOCABULARY_DETECTION_MODE, "https://media.roboflow.com/notebooks/examples/dog-2.jpeg", 'straw, white napkin, black napkin, hair'],
-    [IMAGE_OPEN_VOCABULARY_DETECTION_MODE, "https://media.roboflow.com/notebooks/examples/dog-3.jpeg", 'tail'],
-    [IMAGE_CAPTION_GROUNDING_MASKS_MODE, "https://media.roboflow.com/notebooks/examples/dog-2.jpeg", None],
-    [IMAGE_CAPTION_GROUNDING_MASKS_MODE, "https://media.roboflow.com/notebooks/examples/dog-3.jpeg", None],
-]
-VIDEO_PROCESSING_EXAMPLES = [
-    ["videos/clip-07-camera-1.mp4", "player in white outfit, player in black outfit, ball, rim"],
-    ["videos/clip-07-camera-2.mp4", "player in white outfit, player in black outfit, ball, rim"],
-    ["videos/clip-07-camera-3.mp4", "player in white outfit, player in black outfit, ball, rim"]
-]
+# IMAGE_PROCESSING_EXAMPLES = [
+#     [IMAGE_OPEN_VOCABULARY_DETECTION_MODE, "https://media.roboflow.com/notebooks/examples/dog-2.jpeg", 'straw, white napkin, black napkin, hair'],
+#     [IMAGE_OPEN_VOCABULARY_DETECTION_MODE, "https://media.roboflow.com/notebooks/examples/dog-3.jpeg", 'tail'],
+#     [IMAGE_CAPTION_GROUNDING_MASKS_MODE, "https://media.roboflow.com/notebooks/examples/dog-2.jpeg", None],
+#     [IMAGE_CAPTION_GROUNDING_MASKS_MODE, "https://media.roboflow.com/notebooks/examples/dog-3.jpeg", None],
+# ]
+# VIDEO_PROCESSING_EXAMPLES = [
+#     ["videos/clip-07-camera-1.mp4", "player in white outfit, player in black outfit, ball, rim"],
+#     ["videos/clip-07-camera-2.mp4", "player in white outfit, player in black outfit, ball, rim"],
+#     ["videos/clip-07-camera-3.mp4", "player in white outfit, player in black outfit, ball, rim"]
+# ]

-VIDEO_SCALE_FACTOR = 0.5
-VIDEO_TARGET_DIRECTORY = "tmp"
-create_directory(directory_path=VIDEO_TARGET_DIRECTORY)
+# VIDEO_SCALE_FACTOR = 0.5
+# VIDEO_TARGET_DIRECTORY = "tmp"
+# create_directory(directory_path=VIDEO_TARGET_DIRECTORY)

 DEVICE = torch.device("cuda")
 # DEVICE = torch.device("cpu")
@@ -69,7 +69,7 @@ if torch.cuda.get_device_properties(0).major >= 8:

 FLORENCE_MODEL, FLORENCE_PROCESSOR = load_florence_model(device=DEVICE)
 SAM_IMAGE_MODEL = load_sam_image_model(device=DEVICE)
-SAM_VIDEO_MODEL = load_sam_video_model(device=DEVICE)
+# SAM_VIDEO_MODEL = load_sam_video_model(device=DEVICE)
 COLORS = ['#FF1493', '#00BFFF', '#FF6347', '#FFD700', '#32CD32', '#8A2BE2']
 COLOR_PALETTE = sv.ColorPalette.from_hex(COLORS)
 BOX_ANNOTATOR = sv.BoxAnnotator(color=COLOR_PALETTE, color_lookup=sv.ColorLookup.INDEX)
@@ -94,18 +94,20 @@ def annotate_image(image, detections):
    return output_image


-def on_mode_dropdown_change(text):
-    return [
-        gr.Textbox(visible=text == IMAGE_OPEN_VOCABULARY_DETECTION_MODE),
-        gr.Textbox(visible=text == IMAGE_CAPTION_GROUNDING_MASKS_MODE),
-    ]
+# def on_mode_dropdown_change(text):
+#     return [
+#         gr.Textbox(visible=text == IMAGE_OPEN_VOCABULARY_DETECTION_MODE),
+#         gr.Textbox(visible=text == IMAGE_CAPTION_GROUNDING_MASKS_MODE),
+#     ]


-@spaces.GPU
+def process_image(image: Image.Image, promt: str) -> Tuple[Optional[Image.Image], Optional[str]]:
+    return _process_image(IMAGE_OPEN_VOCABULARY_DETECTION_MODE, image, promt)
+
@torch.inference_mode()
@torch.autocast(device_type="cuda", dtype=torch.bfloat16)
-def process_image(
-    mode_dropdown, image_input, text_input
+def _process_image(
+    mode_dropdown=IMAGE_OPEN_VOCABULARY_DETECTION_MODE, image_input=None, text_input=None
 ) -> Tuple[Optional[Image.Image], Optional[str]]:
    if not image_input:
        gr.Info("Please upload an image.")
@@ -139,259 +141,259 @@ def process_image(
        detections = run_sam_inference(SAM_IMAGE_MODEL, image_input, detections)
        return annotate_image(image_input, detections), None

-    if mode_dropdown == IMAGE_CAPTION_GROUNDING_MASKS_MODE:
-        _, result = run_florence_inference(
-            model=FLORENCE_MODEL,
-            processor=FLORENCE_PROCESSOR,
-            device=DEVICE,
-            image=image_input,
-            task=FLORENCE_DETAILED_CAPTION_TASK
-        )
-        caption = result[FLORENCE_DETAILED_CAPTION_TASK]
-        _, result = run_florence_inference(
-            model=FLORENCE_MODEL,
-            processor=FLORENCE_PROCESSOR,
-            device=DEVICE,
-            image=image_input,
-            task=FLORENCE_CAPTION_TO_PHRASE_GROUNDING_TASK,
-            text=caption
-        )
-        detections = sv.Detections.from_lmm(
-            lmm=sv.LMM.FLORENCE_2,
-            result=result,
-            resolution_wh=image_input.size
-        )
-        detections = run_sam_inference(SAM_IMAGE_MODEL, image_input, detections)
-        return annotate_image(image_input, detections), caption
+#     if mode_dropdown == IMAGE_CAPTION_GROUNDING_MASKS_MODE:
+#         _, result = run_florence_inference(
+#             model=FLORENCE_MODEL,
+#             processor=FLORENCE_PROCESSOR,
+#             device=DEVICE,
+#             image=image_input,
+#             task=FLORENCE_DETAILED_CAPTION_TASK
+#         )
+#         caption = result[FLORENCE_DETAILED_CAPTION_TASK]
+#         _, result = run_florence_inference(
+#             model=FLORENCE_MODEL,
+#             processor=FLORENCE_PROCESSOR,
+#             device=DEVICE,
+#             image=image_input,
+#             task=FLORENCE_CAPTION_TO_PHRASE_GROUNDING_TASK,
+#             text=caption
+#         )
+#         detections = sv.Detections.from_lmm(
+#             lmm=sv.LMM.FLORENCE_2,
+#             result=result,
+#             resolution_wh=image_input.size
+#         )
+#         detections = run_sam_inference(SAM_IMAGE_MODEL, image_input, detections)
+#         return annotate_image(image_input, detections), caption


-@spaces.GPU(duration=300)
-@torch.inference_mode()
-@torch.autocast(device_type="cuda", dtype=torch.bfloat16)
-def process_video(
-    video_input, text_input, progress=gr.Progress(track_tqdm=True)
-) -> Optional[str]:
-    if not video_input:
-        gr.Info("Please upload a video.")
-        return None
+# @spaces.GPU(duration=300)
+# @torch.inference_mode()
+# @torch.autocast(device_type="cuda", dtype=torch.bfloat16)
+# def process_video(
+#     video_input, text_input, progress=gr.Progress(track_tqdm=True)
+# ) -> Optional[str]:
+#     if not video_input:
+#         gr.Info("Please upload a video.")
+#         return None

-    if not text_input:
-        gr.Info("Please enter a text prompt.")
-        return None
+#     if not text_input:
+#         gr.Info("Please enter a text prompt.")
+#         return None

-    frame_generator = sv.get_video_frames_generator(video_input)
-    frame = next(frame_generator)
-    frame = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
+#     frame_generator = sv.get_video_frames_generator(video_input)
+#     frame = next(frame_generator)
+#     frame = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))

-    texts = [prompt.strip() for prompt in text_input.split(",")]
-    detections_list = []
-    for text in texts:
-        _, result = run_florence_inference(
-            model=FLORENCE_MODEL,
-            processor=FLORENCE_PROCESSOR,
-            device=DEVICE,
-            image=frame,
-            task=FLORENCE_OPEN_VOCABULARY_DETECTION_TASK,
-            text=text
-        )
-        detections = sv.Detections.from_lmm(
-            lmm=sv.LMM.FLORENCE_2,
-            result=result,
-            resolution_wh=frame.size
-        )
-        detections = run_sam_inference(SAM_IMAGE_MODEL, frame, detections)
-        detections_list.append(detections)
+#     texts = [prompt.strip() for prompt in text_input.split(",")]
+#     detections_list = []
+#     for text in texts:
+#         _, result = run_florence_inference(
+#             model=FLORENCE_MODEL,
+#             processor=FLORENCE_PROCESSOR,
+#             device=DEVICE,
+#             image=frame,
+#             task=FLORENCE_OPEN_VOCABULARY_DETECTION_TASK,
+#             text=text
+#         )
+#         detections = sv.Detections.from_lmm(
+#             lmm=sv.LMM.FLORENCE_2,
+#             result=result,
+#             resolution_wh=frame.size
+#         )
+#         detections = run_sam_inference(SAM_IMAGE_MODEL, frame, detections)
+#         detections_list.append(detections)

-    detections = sv.Detections.merge(detections_list)
-    detections = run_sam_inference(SAM_IMAGE_MODEL, frame, detections)
+#     detections = sv.Detections.merge(detections_list)
+#     detections = run_sam_inference(SAM_IMAGE_MODEL, frame, detections)

-    if len(detections.mask) == 0:
-        gr.Info(
-            "No objects of class {text_input} found in the first frame of the video. "
-            "Trim the video to make the object appear in the first frame or try a "
-            "different text prompt."
-        )
-        return None
+#     if len(detections.mask) == 0:
+#         gr.Info(
+#             "No objects of class {text_input} found in the first frame of the video. "
+#             "Trim the video to make the object appear in the first frame or try a "
+#             "different text prompt."
+#         )
+#         return None

-    name = generate_unique_name()
-    frame_directory_path = os.path.join(VIDEO_TARGET_DIRECTORY, name)
-    frames_sink = sv.ImageSink(
-        target_dir_path=frame_directory_path,
-        image_name_pattern="{:05d}.jpeg"
-    )
+#     name = generate_unique_name()
+#     frame_directory_path = os.path.join(VIDEO_TARGET_DIRECTORY, name)
+#     frames_sink = sv.ImageSink(
+#         target_dir_path=frame_directory_path,
+#         image_name_pattern="{:05d}.jpeg"
+#     )

-    video_info = sv.VideoInfo.from_video_path(video_input)
-    video_info.width = int(video_info.width * VIDEO_SCALE_FACTOR)
-    video_info.height = int(video_info.height * VIDEO_SCALE_FACTOR)
+#     video_info = sv.VideoInfo.from_video_path(video_input)
+#     video_info.width = int(video_info.width * VIDEO_SCALE_FACTOR)
+#     video_info.height = int(video_info.height * VIDEO_SCALE_FACTOR)

-    frames_generator = sv.get_video_frames_generator(video_input)
-    with frames_sink:
-        for frame in tqdm(
-                frames_generator,
-                total=video_info.total_frames,
-                desc="splitting video into frames"
-        ):
-            frame = sv.scale_image(frame, VIDEO_SCALE_FACTOR)
-            frames_sink.save_image(frame)
+#     frames_generator = sv.get_video_frames_generator(video_input)
+#     with frames_sink:
+#         for frame in tqdm(
+#                 frames_generator,
+#                 total=video_info.total_frames,
+#                 desc="splitting video into frames"
+#         ):
+#             frame = sv.scale_image(frame, VIDEO_SCALE_FACTOR)
+#             frames_sink.save_image(frame)

-    inference_state = SAM_VIDEO_MODEL.init_state(
-        video_path=frame_directory_path,
-        device=DEVICE
-    )
+#     inference_state = SAM_VIDEO_MODEL.init_state(
+#         video_path=frame_directory_path,
+#         device=DEVICE
+#     )

-    for mask_index, mask in enumerate(detections.mask):
-        _, object_ids, mask_logits = SAM_VIDEO_MODEL.add_new_mask(
-            inference_state=inference_state,
-            frame_idx=0,
-            obj_id=mask_index,
-            mask=mask
-        )
+#     for mask_index, mask in enumerate(detections.mask):
+#         _, object_ids, mask_logits = SAM_VIDEO_MODEL.add_new_mask(
+#             inference_state=inference_state,
+#             frame_idx=0,
+#             obj_id=mask_index,
+#             mask=mask
+#         )

-    video_path = os.path.join(VIDEO_TARGET_DIRECTORY, f"{name}.mp4")
-    frames_generator = sv.get_video_frames_generator(video_input)
-    masks_generator = SAM_VIDEO_MODEL.propagate_in_video(inference_state)
-    with sv.VideoSink(video_path, video_info=video_info) as sink:
-        for frame, (_, tracker_ids, mask_logits) in zip(frames_generator, masks_generator):
-            frame = sv.scale_image(frame, VIDEO_SCALE_FACTOR)
-            masks = (mask_logits > 0.0).cpu().numpy().astype(bool)
-            if len(masks.shape) == 4:
-                masks = np.squeeze(masks, axis=1)
+#     video_path = os.path.join(VIDEO_TARGET_DIRECTORY, f"{name}.mp4")
+#     frames_generator = sv.get_video_frames_generator(video_input)
+#     masks_generator = SAM_VIDEO_MODEL.propagate_in_video(inference_state)
+#     with sv.VideoSink(video_path, video_info=video_info) as sink:
+#         for frame, (_, tracker_ids, mask_logits) in zip(frames_generator, masks_generator):
+#             frame = sv.scale_image(frame, VIDEO_SCALE_FACTOR)
+#             masks = (mask_logits > 0.0).cpu().numpy().astype(bool)
+#             if len(masks.shape) == 4:
+#                 masks = np.squeeze(masks, axis=1)

-            detections = sv.Detections(
-                xyxy=sv.mask_to_xyxy(masks=masks),
-                mask=masks,
-                class_id=np.array(tracker_ids)
-            )
-            annotated_frame = frame.copy()
-            annotated_frame = MASK_ANNOTATOR.annotate(
-                scene=annotated_frame, detections=detections)
-            annotated_frame = BOX_ANNOTATOR.annotate(
-                scene=annotated_frame, detections=detections)
-            sink.write_frame(annotated_frame)
+#             detections = sv.Detections(
+#                 xyxy=sv.mask_to_xyxy(masks=masks),
+#                 mask=masks,
+#                 class_id=np.array(tracker_ids)
+#             )
+#             annotated_frame = frame.copy()
+#             annotated_frame = MASK_ANNOTATOR.annotate(
+#                 scene=annotated_frame, detections=detections)
+#             annotated_frame = BOX_ANNOTATOR.annotate(
+#                 scene=annotated_frame, detections=detections)
+#             sink.write_frame(annotated_frame)

-    delete_directory(frame_directory_path)
-    return video_path
+#     delete_directory(frame_directory_path)
+#     return video_path


-with gr.Blocks() as demo:
-    gr.Markdown(MARKDOWN)
-    with gr.Tab("Image"):
-        image_processing_mode_dropdown_component = gr.Dropdown(
-            choices=IMAGE_INFERENCE_MODES,
-            value=IMAGE_INFERENCE_MODES[0],
-            label="Mode",
-            info="Select a mode to use.",
-            interactive=True
-        )
-        with gr.Row():
-            with gr.Column():
-                image_processing_image_input_component = gr.Image(
-                    type='pil', label='Upload image')
-                image_processing_text_input_component = gr.Textbox(
-                    label='Text prompt',
-                    placeholder='Enter comma separated text prompts')
-                image_processing_submit_button_component = gr.Button(
-                    value='Submit', variant='primary')
-            with gr.Column():
-                image_processing_image_output_component = gr.Image(
-                    type='pil', label='Image output')
-                image_processing_text_output_component = gr.Textbox(
-                    label='Caption output', visible=False)
+# with gr.Blocks() as demo:
+#     gr.Markdown(MARKDOWN)
+#     with gr.Tab("Image"):
+#         image_processing_mode_dropdown_component = gr.Dropdown(
+#             choices=IMAGE_INFERENCE_MODES,
+#             value=IMAGE_INFERENCE_MODES[0],
+#             label="Mode",
+#             info="Select a mode to use.",
+#             interactive=True
+#         )
+#         with gr.Row():
+#             with gr.Column():
+#                 image_processing_image_input_component = gr.Image(
+#                     type='pil', label='Upload image')
+#                 image_processing_text_input_component = gr.Textbox(
+#                     label='Text prompt',
+#                     placeholder='Enter comma separated text prompts')
+#                 image_processing_submit_button_component = gr.Button(
+#                     value='Submit', variant='primary')
+#             with gr.Column():
+#                 image_processing_image_output_component = gr.Image(
+#                     type='pil', label='Image output')
+#                 image_processing_text_output_component = gr.Textbox(
+#                     label='Caption output', visible=False)

-        with gr.Row():
-            gr.Examples(
-                fn=process_image,
-                examples=IMAGE_PROCESSING_EXAMPLES,
-                inputs=[
-                    image_processing_mode_dropdown_component,
-                    image_processing_image_input_component,
-                    image_processing_text_input_component
-                ],
-                outputs=[
-                    image_processing_image_output_component,
-                    image_processing_text_output_component
-                ],
-                run_on_click=True
-            )
-    with gr.Tab("Video"):
-        video_processing_mode_dropdown_component = gr.Dropdown(
-            choices=VIDEO_INFERENCE_MODES,
-            value=VIDEO_INFERENCE_MODES[0],
-            label="Mode",
-            info="Select a mode to use.",
-            interactive=True
-        )
-        with gr.Row():
-            with gr.Column():
-                video_processing_video_input_component = gr.Video(
-                    label='Upload video')
-                video_processing_text_input_component = gr.Textbox(
-                    label='Text prompt',
-                    placeholder='Enter comma separated text prompts')
-                video_processing_submit_button_component = gr.Button(
-                    value='Submit', variant='primary')
-            with gr.Column():
-                video_processing_video_output_component = gr.Video(
-                    label='Video output')
-        with gr.Row():
-            gr.Examples(
-                fn=process_video,
-                examples=VIDEO_PROCESSING_EXAMPLES,
-                inputs=[
-                    video_processing_video_input_component,
-                    video_processing_text_input_component
-                ],
-                outputs=video_processing_video_output_component,
-                run_on_click=True
-            )
+#         with gr.Row():
+#             gr.Examples(
+#                 fn=process_image,
+#                 examples=IMAGE_PROCESSING_EXAMPLES,
+#                 inputs=[
+#                     image_processing_mode_dropdown_component,
+#                     image_processing_image_input_component,
+#                     image_processing_text_input_component
+#                 ],
+#                 outputs=[
+#                     image_processing_image_output_component,
+#                     image_processing_text_output_component
+#                 ],
+#                 run_on_click=True
+#             )
+#     with gr.Tab("Video"):
+#         video_processing_mode_dropdown_component = gr.Dropdown(
+#             choices=VIDEO_INFERENCE_MODES,
+#             value=VIDEO_INFERENCE_MODES[0],
+#             label="Mode",
+#             info="Select a mode to use.",
+#             interactive=True
+#         )
+#         with gr.Row():
+#             with gr.Column():
+#                 video_processing_video_input_component = gr.Video(
+#                     label='Upload video')
+#                 video_processing_text_input_component = gr.Textbox(
+#                     label='Text prompt',
+#                     placeholder='Enter comma separated text prompts')
+#                 video_processing_submit_button_component = gr.Button(
+#                     value='Submit', variant='primary')
+#             with gr.Column():
+#                 video_processing_video_output_component = gr.Video(
+#                     label='Video output')
+#         with gr.Row():
+#             gr.Examples(
+#                 fn=process_video,
+#                 examples=VIDEO_PROCESSING_EXAMPLES,
+#                 inputs=[
+#                     video_processing_video_input_component,
+#                     video_processing_text_input_component
+#                 ],
+#                 outputs=video_processing_video_output_component,
+#                 run_on_click=True
+#             )

-    image_processing_submit_button_component.click(
-        fn=process_image,
-        inputs=[
-            image_processing_mode_dropdown_component,
-            image_processing_image_input_component,
-            image_processing_text_input_component
-        ],
-        outputs=[
-            image_processing_image_output_component,
-            image_processing_text_output_component
-        ]
-    )
-    image_processing_text_input_component.submit(
-        fn=process_image,
-        inputs=[
-            image_processing_mode_dropdown_component,
-            image_processing_image_input_component,
-            image_processing_text_input_component
-        ],
-        outputs=[
-            image_processing_image_output_component,
-            image_processing_text_output_component
-        ]
-    )
-    image_processing_mode_dropdown_component.change(
-        on_mode_dropdown_change,
-        inputs=[image_processing_mode_dropdown_component],
-        outputs=[
-            image_processing_text_input_component,
-            image_processing_text_output_component
-        ]
-    )
-    video_processing_submit_button_component.click(
-        fn=process_video,
-        inputs=[
-            video_processing_video_input_component,
-            video_processing_text_input_component
-        ],
-        outputs=video_processing_video_output_component
-    )
-    video_processing_text_input_component.submit(
-        fn=process_video,
-        inputs=[
-            video_processing_video_input_component,
-            video_processing_text_input_component
-        ],
-        outputs=video_processing_video_output_component
-    )
+#     image_processing_submit_button_component.click(
+#         fn=process_image,
+#         inputs=[
+#             image_processing_mode_dropdown_component,
+#             image_processing_image_input_component,
+#             image_processing_text_input_component
+#         ],
+#         outputs=[
+#             image_processing_image_output_component,
+#             image_processing_text_output_component
+#         ]
+#     )
+#     image_processing_text_input_component.submit(
+#         fn=process_image,
+#         inputs=[
+#             image_processing_mode_dropdown_component,
+#             image_processing_image_input_component,
+#             image_processing_text_input_component
+#         ],
+#         outputs=[
+#             image_processing_image_output_component,
+#             image_processing_text_output_component
+#         ]
+#     )
+#     image_processing_mode_dropdown_component.change(
+#         on_mode_dropdown_change,
+#         inputs=[image_processing_mode_dropdown_component],
+#         outputs=[
+#             image_processing_text_input_component,
+#             image_processing_text_output_component
+#         ]
+#     )
+#     video_processing_submit_button_component.click(
+#         fn=process_video,
+#         inputs=[
+#             video_processing_video_input_component,
+#             video_processing_text_input_component
+#         ],
+#         outputs=video_processing_video_output_component
+#     )
+#     video_processing_text_input_component.submit(
+#         fn=process_video,
+#         inputs=[
+#             video_processing_video_input_component,
+#             video_processing_text_input_component
+#         ],
+#         outputs=video_processing_video_output_component
+#     )

-demo.launch(debug=False, show_error=True)
+# demo.launch(debug=False, show_error=True)
@@ -0,0 +1 @@
+__init__.py
@@ -0,0 +1,32 @@
+
+- 1 image goes in and has a prompt "products" and then out comes a mask with all the segmented products in the image
+- But it could output a batch or list (I will probably just end up combining them all in the next node)
+
+Note: Instead of bounding boxes with labels we **just need the masks** that's it
+
+
+<table>
+<tr><th>source image</th><th>screenshot (left: source; right: result)</th></tr>
+<tr><td>
+
+![01](01_source.png)
+
+</td><td>
+
+![01](01_bounding_boxes.jpeg)
+
+</td></tr>
+
+<tr><td>
+
+![02](02_source.png)
+
+</td><td>
+
+![02](02_bounding_boxes.jpeg)
+
+</td></tr>
+
+</table>
+
+The test images have been confirmed to work with this version of the code, and the screenshots above were taken from the [HuggingFace space](https://huggingface.co/spaces/SkalskiP/florence-sam).
@@ -20,7 +20,10 @@ def fixed_get_imports(filename: Union[str, os.PathLike]) -> list[str]:
    if not str(filename).endswith("/modeling_florence2.py"):
        return get_imports(filename)
    imports = get_imports(filename)
-    imports.remove("flash_attn")
+    try:
+        imports.remove("flash_attn")
+    except ValueError:
+        pass
    return imports