diff --git a/.gitignore b/.gitignore index 3cd62da..d4d6813 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ __pycache__/ .*.sw? *~ + +# something on my MacOS keeps saving these resource forks for every single file --rdancer +.* diff --git a/README.md b/README.md index ae8e923..d2fee20 100644 --- a/README.md +++ b/README.md @@ -14,3 +14,7 @@ Check out the configuration reference at https://huggingface.co/docs/hub/spaces- Code in this directory and its subdirectories based on SkalskiP's Florence 2 + SAM 2 space at https://huggingface.co/spaces/SkalskiP/florence-sam + +## Testing + +Run `python test.py test/*_source.png "products"` 👉 the resulting images must pixel-match `test/*_output.png`. diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..a5415bf --- /dev/null +++ b/__init__.py @@ -0,0 +1,53 @@ +# from app import process_image + +__all__ = ["process_image"] + + +if __name__ == "__main__": + # detect which parameters are filenames -- those are images + # the rest are prompts + # call process_image with the images and prompts + # save the output images + # return the output images' filenames + import sys + import os + import argparse + from PIL import Image + + def my_process_image(image_path, prompt): + image = Image.open(image_path).convert("RGB") + output_image, _ = process_image(image, prompt) + output_path = f"output_{os.path.basename(image_path)}" + output_image.save(output_path) + return output_path + + if len(sys.argv) < 2: + print(f"Usage: python {os.path.basename(sys.argv[0])} [ ...] []") + sys.exit(1) + + # test which exist as filenames + images = [] + prompts = [] + + for arg in sys.argv[1:]: + if not os.path.exists(arg): + prompts.append(arg) + else: + images.append(arg) + + if len(prompts) > 1: + raise ValueError("At most one prompt is required") + if len(images) < 1: + raise ValueError("At least one image is required") + + prompt = prompts[0].strip() if prompts else None + + print(f"Processing {len(images)} image{'' if len(images) == 1 else 's'} with prompt: {prompt}") + + from app import process_image + + for image in images: + output_path = my_process_image(image, prompt) + print(f"Output image saved to {output_path}") +else: + from app import process_image diff --git a/app.py b/app.py index 85905ea..06dc5b5 100644 --- a/app.py +++ b/app.py @@ -12,51 +12,51 @@ from tqdm import tqdm from utils.video import generate_unique_name, create_directory, delete_directory from utils.florence import load_florence_model, run_florence_inference, \ - FLORENCE_DETAILED_CAPTION_TASK, \ - FLORENCE_CAPTION_TO_PHRASE_GROUNDING_TASK, FLORENCE_OPEN_VOCABULARY_DETECTION_TASK -from utils.modes import IMAGE_INFERENCE_MODES, IMAGE_OPEN_VOCABULARY_DETECTION_MODE, \ - IMAGE_CAPTION_GROUNDING_MASKS_MODE, VIDEO_INFERENCE_MODES -from utils.sam import load_sam_image_model, run_sam_inference, load_sam_video_model + FLORENCE_OPEN_VOCABULARY_DETECTION_TASK #, + # FLORENCE_DETAILED_CAPTION_TASK, FLORENCE_CAPTION_TO_PHRASE_GROUNDING_TASK +from utils.modes import IMAGE_INFERENCE_MODES, IMAGE_OPEN_VOCABULARY_DETECTION_MODE #, \ + # IMAGE_CAPTION_GROUNDING_MASKS_MODE, VIDEO_INFERENCE_MODES +from utils.sam import load_sam_image_model, run_sam_inference #, load_sam_video_model -MARKDOWN = """ -# Florence2 + SAM2 🔥 +# MARKDOWN = """ +# # Florence2 + SAM2 🔥 -
- - GitHub - - - Colab - - - Roboflow - - - YouTube - -
+#
+# +# GitHub +# +# +# Colab +# +# +# Roboflow +# +# +# YouTube +# +#
-This demo integrates Florence2 and SAM2 by creating a two-stage inference pipeline. In -the first stage, Florence2 performs tasks such as object detection, open-vocabulary -object detection, image captioning, or phrase grounding. In the second stage, SAM2 -performs object segmentation on the image. -""" +# This demo integrates Florence2 and SAM2 by creating a two-stage inference pipeline. In +# the first stage, Florence2 performs tasks such as object detection, open-vocabulary +# object detection, image captioning, or phrase grounding. In the second stage, SAM2 +# performs object segmentation on the image. +# """ -IMAGE_PROCESSING_EXAMPLES = [ - [IMAGE_OPEN_VOCABULARY_DETECTION_MODE, "https://media.roboflow.com/notebooks/examples/dog-2.jpeg", 'straw, white napkin, black napkin, hair'], - [IMAGE_OPEN_VOCABULARY_DETECTION_MODE, "https://media.roboflow.com/notebooks/examples/dog-3.jpeg", 'tail'], - [IMAGE_CAPTION_GROUNDING_MASKS_MODE, "https://media.roboflow.com/notebooks/examples/dog-2.jpeg", None], - [IMAGE_CAPTION_GROUNDING_MASKS_MODE, "https://media.roboflow.com/notebooks/examples/dog-3.jpeg", None], -] -VIDEO_PROCESSING_EXAMPLES = [ - ["videos/clip-07-camera-1.mp4", "player in white outfit, player in black outfit, ball, rim"], - ["videos/clip-07-camera-2.mp4", "player in white outfit, player in black outfit, ball, rim"], - ["videos/clip-07-camera-3.mp4", "player in white outfit, player in black outfit, ball, rim"] -] +# IMAGE_PROCESSING_EXAMPLES = [ +# [IMAGE_OPEN_VOCABULARY_DETECTION_MODE, "https://media.roboflow.com/notebooks/examples/dog-2.jpeg", 'straw, white napkin, black napkin, hair'], +# [IMAGE_OPEN_VOCABULARY_DETECTION_MODE, "https://media.roboflow.com/notebooks/examples/dog-3.jpeg", 'tail'], +# [IMAGE_CAPTION_GROUNDING_MASKS_MODE, "https://media.roboflow.com/notebooks/examples/dog-2.jpeg", None], +# [IMAGE_CAPTION_GROUNDING_MASKS_MODE, "https://media.roboflow.com/notebooks/examples/dog-3.jpeg", None], +# ] +# VIDEO_PROCESSING_EXAMPLES = [ +# ["videos/clip-07-camera-1.mp4", "player in white outfit, player in black outfit, ball, rim"], +# ["videos/clip-07-camera-2.mp4", "player in white outfit, player in black outfit, ball, rim"], +# ["videos/clip-07-camera-3.mp4", "player in white outfit, player in black outfit, ball, rim"] +# ] -VIDEO_SCALE_FACTOR = 0.5 -VIDEO_TARGET_DIRECTORY = "tmp" -create_directory(directory_path=VIDEO_TARGET_DIRECTORY) +# VIDEO_SCALE_FACTOR = 0.5 +# VIDEO_TARGET_DIRECTORY = "tmp" +# create_directory(directory_path=VIDEO_TARGET_DIRECTORY) DEVICE = torch.device("cuda") # DEVICE = torch.device("cpu") @@ -69,7 +69,7 @@ if torch.cuda.get_device_properties(0).major >= 8: FLORENCE_MODEL, FLORENCE_PROCESSOR = load_florence_model(device=DEVICE) SAM_IMAGE_MODEL = load_sam_image_model(device=DEVICE) -SAM_VIDEO_MODEL = load_sam_video_model(device=DEVICE) +# SAM_VIDEO_MODEL = load_sam_video_model(device=DEVICE) COLORS = ['#FF1493', '#00BFFF', '#FF6347', '#FFD700', '#32CD32', '#8A2BE2'] COLOR_PALETTE = sv.ColorPalette.from_hex(COLORS) BOX_ANNOTATOR = sv.BoxAnnotator(color=COLOR_PALETTE, color_lookup=sv.ColorLookup.INDEX) @@ -94,18 +94,20 @@ def annotate_image(image, detections): return output_image -def on_mode_dropdown_change(text): - return [ - gr.Textbox(visible=text == IMAGE_OPEN_VOCABULARY_DETECTION_MODE), - gr.Textbox(visible=text == IMAGE_CAPTION_GROUNDING_MASKS_MODE), - ] +# def on_mode_dropdown_change(text): +# return [ +# gr.Textbox(visible=text == IMAGE_OPEN_VOCABULARY_DETECTION_MODE), +# gr.Textbox(visible=text == IMAGE_CAPTION_GROUNDING_MASKS_MODE), +# ] -@spaces.GPU +def process_image(image: Image.Image, promt: str) -> Tuple[Optional[Image.Image], Optional[str]]: + return _process_image(IMAGE_OPEN_VOCABULARY_DETECTION_MODE, image, promt) + @torch.inference_mode() @torch.autocast(device_type="cuda", dtype=torch.bfloat16) -def process_image( - mode_dropdown, image_input, text_input +def _process_image( + mode_dropdown=IMAGE_OPEN_VOCABULARY_DETECTION_MODE, image_input=None, text_input=None ) -> Tuple[Optional[Image.Image], Optional[str]]: if not image_input: gr.Info("Please upload an image.") @@ -139,259 +141,259 @@ def process_image( detections = run_sam_inference(SAM_IMAGE_MODEL, image_input, detections) return annotate_image(image_input, detections), None - if mode_dropdown == IMAGE_CAPTION_GROUNDING_MASKS_MODE: - _, result = run_florence_inference( - model=FLORENCE_MODEL, - processor=FLORENCE_PROCESSOR, - device=DEVICE, - image=image_input, - task=FLORENCE_DETAILED_CAPTION_TASK - ) - caption = result[FLORENCE_DETAILED_CAPTION_TASK] - _, result = run_florence_inference( - model=FLORENCE_MODEL, - processor=FLORENCE_PROCESSOR, - device=DEVICE, - image=image_input, - task=FLORENCE_CAPTION_TO_PHRASE_GROUNDING_TASK, - text=caption - ) - detections = sv.Detections.from_lmm( - lmm=sv.LMM.FLORENCE_2, - result=result, - resolution_wh=image_input.size - ) - detections = run_sam_inference(SAM_IMAGE_MODEL, image_input, detections) - return annotate_image(image_input, detections), caption +# if mode_dropdown == IMAGE_CAPTION_GROUNDING_MASKS_MODE: +# _, result = run_florence_inference( +# model=FLORENCE_MODEL, +# processor=FLORENCE_PROCESSOR, +# device=DEVICE, +# image=image_input, +# task=FLORENCE_DETAILED_CAPTION_TASK +# ) +# caption = result[FLORENCE_DETAILED_CAPTION_TASK] +# _, result = run_florence_inference( +# model=FLORENCE_MODEL, +# processor=FLORENCE_PROCESSOR, +# device=DEVICE, +# image=image_input, +# task=FLORENCE_CAPTION_TO_PHRASE_GROUNDING_TASK, +# text=caption +# ) +# detections = sv.Detections.from_lmm( +# lmm=sv.LMM.FLORENCE_2, +# result=result, +# resolution_wh=image_input.size +# ) +# detections = run_sam_inference(SAM_IMAGE_MODEL, image_input, detections) +# return annotate_image(image_input, detections), caption -@spaces.GPU(duration=300) -@torch.inference_mode() -@torch.autocast(device_type="cuda", dtype=torch.bfloat16) -def process_video( - video_input, text_input, progress=gr.Progress(track_tqdm=True) -) -> Optional[str]: - if not video_input: - gr.Info("Please upload a video.") - return None +# @spaces.GPU(duration=300) +# @torch.inference_mode() +# @torch.autocast(device_type="cuda", dtype=torch.bfloat16) +# def process_video( +# video_input, text_input, progress=gr.Progress(track_tqdm=True) +# ) -> Optional[str]: +# if not video_input: +# gr.Info("Please upload a video.") +# return None - if not text_input: - gr.Info("Please enter a text prompt.") - return None +# if not text_input: +# gr.Info("Please enter a text prompt.") +# return None - frame_generator = sv.get_video_frames_generator(video_input) - frame = next(frame_generator) - frame = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) +# frame_generator = sv.get_video_frames_generator(video_input) +# frame = next(frame_generator) +# frame = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) - texts = [prompt.strip() for prompt in text_input.split(",")] - detections_list = [] - for text in texts: - _, result = run_florence_inference( - model=FLORENCE_MODEL, - processor=FLORENCE_PROCESSOR, - device=DEVICE, - image=frame, - task=FLORENCE_OPEN_VOCABULARY_DETECTION_TASK, - text=text - ) - detections = sv.Detections.from_lmm( - lmm=sv.LMM.FLORENCE_2, - result=result, - resolution_wh=frame.size - ) - detections = run_sam_inference(SAM_IMAGE_MODEL, frame, detections) - detections_list.append(detections) +# texts = [prompt.strip() for prompt in text_input.split(",")] +# detections_list = [] +# for text in texts: +# _, result = run_florence_inference( +# model=FLORENCE_MODEL, +# processor=FLORENCE_PROCESSOR, +# device=DEVICE, +# image=frame, +# task=FLORENCE_OPEN_VOCABULARY_DETECTION_TASK, +# text=text +# ) +# detections = sv.Detections.from_lmm( +# lmm=sv.LMM.FLORENCE_2, +# result=result, +# resolution_wh=frame.size +# ) +# detections = run_sam_inference(SAM_IMAGE_MODEL, frame, detections) +# detections_list.append(detections) - detections = sv.Detections.merge(detections_list) - detections = run_sam_inference(SAM_IMAGE_MODEL, frame, detections) +# detections = sv.Detections.merge(detections_list) +# detections = run_sam_inference(SAM_IMAGE_MODEL, frame, detections) - if len(detections.mask) == 0: - gr.Info( - "No objects of class {text_input} found in the first frame of the video. " - "Trim the video to make the object appear in the first frame or try a " - "different text prompt." - ) - return None +# if len(detections.mask) == 0: +# gr.Info( +# "No objects of class {text_input} found in the first frame of the video. " +# "Trim the video to make the object appear in the first frame or try a " +# "different text prompt." +# ) +# return None - name = generate_unique_name() - frame_directory_path = os.path.join(VIDEO_TARGET_DIRECTORY, name) - frames_sink = sv.ImageSink( - target_dir_path=frame_directory_path, - image_name_pattern="{:05d}.jpeg" - ) +# name = generate_unique_name() +# frame_directory_path = os.path.join(VIDEO_TARGET_DIRECTORY, name) +# frames_sink = sv.ImageSink( +# target_dir_path=frame_directory_path, +# image_name_pattern="{:05d}.jpeg" +# ) - video_info = sv.VideoInfo.from_video_path(video_input) - video_info.width = int(video_info.width * VIDEO_SCALE_FACTOR) - video_info.height = int(video_info.height * VIDEO_SCALE_FACTOR) +# video_info = sv.VideoInfo.from_video_path(video_input) +# video_info.width = int(video_info.width * VIDEO_SCALE_FACTOR) +# video_info.height = int(video_info.height * VIDEO_SCALE_FACTOR) - frames_generator = sv.get_video_frames_generator(video_input) - with frames_sink: - for frame in tqdm( - frames_generator, - total=video_info.total_frames, - desc="splitting video into frames" - ): - frame = sv.scale_image(frame, VIDEO_SCALE_FACTOR) - frames_sink.save_image(frame) +# frames_generator = sv.get_video_frames_generator(video_input) +# with frames_sink: +# for frame in tqdm( +# frames_generator, +# total=video_info.total_frames, +# desc="splitting video into frames" +# ): +# frame = sv.scale_image(frame, VIDEO_SCALE_FACTOR) +# frames_sink.save_image(frame) - inference_state = SAM_VIDEO_MODEL.init_state( - video_path=frame_directory_path, - device=DEVICE - ) +# inference_state = SAM_VIDEO_MODEL.init_state( +# video_path=frame_directory_path, +# device=DEVICE +# ) - for mask_index, mask in enumerate(detections.mask): - _, object_ids, mask_logits = SAM_VIDEO_MODEL.add_new_mask( - inference_state=inference_state, - frame_idx=0, - obj_id=mask_index, - mask=mask - ) +# for mask_index, mask in enumerate(detections.mask): +# _, object_ids, mask_logits = SAM_VIDEO_MODEL.add_new_mask( +# inference_state=inference_state, +# frame_idx=0, +# obj_id=mask_index, +# mask=mask +# ) - video_path = os.path.join(VIDEO_TARGET_DIRECTORY, f"{name}.mp4") - frames_generator = sv.get_video_frames_generator(video_input) - masks_generator = SAM_VIDEO_MODEL.propagate_in_video(inference_state) - with sv.VideoSink(video_path, video_info=video_info) as sink: - for frame, (_, tracker_ids, mask_logits) in zip(frames_generator, masks_generator): - frame = sv.scale_image(frame, VIDEO_SCALE_FACTOR) - masks = (mask_logits > 0.0).cpu().numpy().astype(bool) - if len(masks.shape) == 4: - masks = np.squeeze(masks, axis=1) +# video_path = os.path.join(VIDEO_TARGET_DIRECTORY, f"{name}.mp4") +# frames_generator = sv.get_video_frames_generator(video_input) +# masks_generator = SAM_VIDEO_MODEL.propagate_in_video(inference_state) +# with sv.VideoSink(video_path, video_info=video_info) as sink: +# for frame, (_, tracker_ids, mask_logits) in zip(frames_generator, masks_generator): +# frame = sv.scale_image(frame, VIDEO_SCALE_FACTOR) +# masks = (mask_logits > 0.0).cpu().numpy().astype(bool) +# if len(masks.shape) == 4: +# masks = np.squeeze(masks, axis=1) - detections = sv.Detections( - xyxy=sv.mask_to_xyxy(masks=masks), - mask=masks, - class_id=np.array(tracker_ids) - ) - annotated_frame = frame.copy() - annotated_frame = MASK_ANNOTATOR.annotate( - scene=annotated_frame, detections=detections) - annotated_frame = BOX_ANNOTATOR.annotate( - scene=annotated_frame, detections=detections) - sink.write_frame(annotated_frame) +# detections = sv.Detections( +# xyxy=sv.mask_to_xyxy(masks=masks), +# mask=masks, +# class_id=np.array(tracker_ids) +# ) +# annotated_frame = frame.copy() +# annotated_frame = MASK_ANNOTATOR.annotate( +# scene=annotated_frame, detections=detections) +# annotated_frame = BOX_ANNOTATOR.annotate( +# scene=annotated_frame, detections=detections) +# sink.write_frame(annotated_frame) - delete_directory(frame_directory_path) - return video_path +# delete_directory(frame_directory_path) +# return video_path -with gr.Blocks() as demo: - gr.Markdown(MARKDOWN) - with gr.Tab("Image"): - image_processing_mode_dropdown_component = gr.Dropdown( - choices=IMAGE_INFERENCE_MODES, - value=IMAGE_INFERENCE_MODES[0], - label="Mode", - info="Select a mode to use.", - interactive=True - ) - with gr.Row(): - with gr.Column(): - image_processing_image_input_component = gr.Image( - type='pil', label='Upload image') - image_processing_text_input_component = gr.Textbox( - label='Text prompt', - placeholder='Enter comma separated text prompts') - image_processing_submit_button_component = gr.Button( - value='Submit', variant='primary') - with gr.Column(): - image_processing_image_output_component = gr.Image( - type='pil', label='Image output') - image_processing_text_output_component = gr.Textbox( - label='Caption output', visible=False) +# with gr.Blocks() as demo: +# gr.Markdown(MARKDOWN) +# with gr.Tab("Image"): +# image_processing_mode_dropdown_component = gr.Dropdown( +# choices=IMAGE_INFERENCE_MODES, +# value=IMAGE_INFERENCE_MODES[0], +# label="Mode", +# info="Select a mode to use.", +# interactive=True +# ) +# with gr.Row(): +# with gr.Column(): +# image_processing_image_input_component = gr.Image( +# type='pil', label='Upload image') +# image_processing_text_input_component = gr.Textbox( +# label='Text prompt', +# placeholder='Enter comma separated text prompts') +# image_processing_submit_button_component = gr.Button( +# value='Submit', variant='primary') +# with gr.Column(): +# image_processing_image_output_component = gr.Image( +# type='pil', label='Image output') +# image_processing_text_output_component = gr.Textbox( +# label='Caption output', visible=False) - with gr.Row(): - gr.Examples( - fn=process_image, - examples=IMAGE_PROCESSING_EXAMPLES, - inputs=[ - image_processing_mode_dropdown_component, - image_processing_image_input_component, - image_processing_text_input_component - ], - outputs=[ - image_processing_image_output_component, - image_processing_text_output_component - ], - run_on_click=True - ) - with gr.Tab("Video"): - video_processing_mode_dropdown_component = gr.Dropdown( - choices=VIDEO_INFERENCE_MODES, - value=VIDEO_INFERENCE_MODES[0], - label="Mode", - info="Select a mode to use.", - interactive=True - ) - with gr.Row(): - with gr.Column(): - video_processing_video_input_component = gr.Video( - label='Upload video') - video_processing_text_input_component = gr.Textbox( - label='Text prompt', - placeholder='Enter comma separated text prompts') - video_processing_submit_button_component = gr.Button( - value='Submit', variant='primary') - with gr.Column(): - video_processing_video_output_component = gr.Video( - label='Video output') - with gr.Row(): - gr.Examples( - fn=process_video, - examples=VIDEO_PROCESSING_EXAMPLES, - inputs=[ - video_processing_video_input_component, - video_processing_text_input_component - ], - outputs=video_processing_video_output_component, - run_on_click=True - ) +# with gr.Row(): +# gr.Examples( +# fn=process_image, +# examples=IMAGE_PROCESSING_EXAMPLES, +# inputs=[ +# image_processing_mode_dropdown_component, +# image_processing_image_input_component, +# image_processing_text_input_component +# ], +# outputs=[ +# image_processing_image_output_component, +# image_processing_text_output_component +# ], +# run_on_click=True +# ) +# with gr.Tab("Video"): +# video_processing_mode_dropdown_component = gr.Dropdown( +# choices=VIDEO_INFERENCE_MODES, +# value=VIDEO_INFERENCE_MODES[0], +# label="Mode", +# info="Select a mode to use.", +# interactive=True +# ) +# with gr.Row(): +# with gr.Column(): +# video_processing_video_input_component = gr.Video( +# label='Upload video') +# video_processing_text_input_component = gr.Textbox( +# label='Text prompt', +# placeholder='Enter comma separated text prompts') +# video_processing_submit_button_component = gr.Button( +# value='Submit', variant='primary') +# with gr.Column(): +# video_processing_video_output_component = gr.Video( +# label='Video output') +# with gr.Row(): +# gr.Examples( +# fn=process_video, +# examples=VIDEO_PROCESSING_EXAMPLES, +# inputs=[ +# video_processing_video_input_component, +# video_processing_text_input_component +# ], +# outputs=video_processing_video_output_component, +# run_on_click=True +# ) - image_processing_submit_button_component.click( - fn=process_image, - inputs=[ - image_processing_mode_dropdown_component, - image_processing_image_input_component, - image_processing_text_input_component - ], - outputs=[ - image_processing_image_output_component, - image_processing_text_output_component - ] - ) - image_processing_text_input_component.submit( - fn=process_image, - inputs=[ - image_processing_mode_dropdown_component, - image_processing_image_input_component, - image_processing_text_input_component - ], - outputs=[ - image_processing_image_output_component, - image_processing_text_output_component - ] - ) - image_processing_mode_dropdown_component.change( - on_mode_dropdown_change, - inputs=[image_processing_mode_dropdown_component], - outputs=[ - image_processing_text_input_component, - image_processing_text_output_component - ] - ) - video_processing_submit_button_component.click( - fn=process_video, - inputs=[ - video_processing_video_input_component, - video_processing_text_input_component - ], - outputs=video_processing_video_output_component - ) - video_processing_text_input_component.submit( - fn=process_video, - inputs=[ - video_processing_video_input_component, - video_processing_text_input_component - ], - outputs=video_processing_video_output_component - ) +# image_processing_submit_button_component.click( +# fn=process_image, +# inputs=[ +# image_processing_mode_dropdown_component, +# image_processing_image_input_component, +# image_processing_text_input_component +# ], +# outputs=[ +# image_processing_image_output_component, +# image_processing_text_output_component +# ] +# ) +# image_processing_text_input_component.submit( +# fn=process_image, +# inputs=[ +# image_processing_mode_dropdown_component, +# image_processing_image_input_component, +# image_processing_text_input_component +# ], +# outputs=[ +# image_processing_image_output_component, +# image_processing_text_output_component +# ] +# ) +# image_processing_mode_dropdown_component.change( +# on_mode_dropdown_change, +# inputs=[image_processing_mode_dropdown_component], +# outputs=[ +# image_processing_text_input_component, +# image_processing_text_output_component +# ] +# ) +# video_processing_submit_button_component.click( +# fn=process_video, +# inputs=[ +# video_processing_video_input_component, +# video_processing_text_input_component +# ], +# outputs=video_processing_video_output_component +# ) +# video_processing_text_input_component.submit( +# fn=process_video, +# inputs=[ +# video_processing_video_input_component, +# video_processing_text_input_component +# ], +# outputs=video_processing_video_output_component +# ) -demo.launch(debug=False, show_error=True) +# demo.launch(debug=False, show_error=True) diff --git a/test.py b/test.py new file mode 120000 index 0000000..93f5256 --- /dev/null +++ b/test.py @@ -0,0 +1 @@ +__init__.py \ No newline at end of file diff --git a/test/01_bounding_boxes.jpeg b/test/01_bounding_boxes.jpeg new file mode 100644 index 0000000..6bcc037 Binary files /dev/null and b/test/01_bounding_boxes.jpeg differ diff --git a/test/01_output.png b/test/01_output.png new file mode 100644 index 0000000..53a321f Binary files /dev/null and b/test/01_output.png differ diff --git a/test/01_source.png b/test/01_source.png new file mode 100644 index 0000000..91a6419 Binary files /dev/null and b/test/01_source.png differ diff --git a/test/02_bounding_boxes.jpeg b/test/02_bounding_boxes.jpeg new file mode 100644 index 0000000..50172fe Binary files /dev/null and b/test/02_bounding_boxes.jpeg differ diff --git a/test/02_output.png b/test/02_output.png new file mode 100644 index 0000000..f56aee2 Binary files /dev/null and b/test/02_output.png differ diff --git a/test/02_source.png b/test/02_source.png new file mode 100644 index 0000000..f38fdbe Binary files /dev/null and b/test/02_source.png differ diff --git a/test/README.md b/test/README.md new file mode 100644 index 0000000..4765c85 --- /dev/null +++ b/test/README.md @@ -0,0 +1,32 @@ + +- 1 image goes in and has a prompt "products" and then out comes a mask with all the segmented products in the image +- But it could output a batch or list (I will probably just end up combining them all in the next node) + +Note: Instead of bounding boxes with labels we **just need the masks** that's it + + + + + + + + +
source imagescreenshot (left: source; right: result)
+ +![01](01_source.png) + + + +![01](01_bounding_boxes.jpeg) + +
+ +![02](02_source.png) + + + +![02](02_bounding_boxes.jpeg) + +
+ +The test images have been confirmed to work with this version of the code, and the screenshots above were taken from the [HuggingFace space](https://huggingface.co/spaces/SkalskiP/florence-sam). \ No newline at end of file diff --git a/utils/florence.py b/utils/florence.py index 9887f44..05d529a 100644 --- a/utils/florence.py +++ b/utils/florence.py @@ -20,7 +20,10 @@ def fixed_get_imports(filename: Union[str, os.PathLike]) -> list[str]: if not str(filename).endswith("/modeling_florence2.py"): return get_imports(filename) imports = get_imports(filename) - imports.remove("flash_attn") + try: + imports.remove("flash_attn") + except ValueError: + pass return imports