gesture recog

2025-05-16 15:22:09 -04:00
parent e1a3f40744
commit aecbd4456d
1 changed files with 98 additions and 102 deletions
@@ -1,4 +1,4 @@
-from typing import List
+from typing import Any, List

 import mediapipe as mp
 import numpy as np
@@ -8,12 +8,11 @@ from mediapipe.tasks.python import vision
 from mediapipe.tasks.python.core.base_options import BaseOptions

 from ...utils.timing import TimestampProvider  # Import TimestampProvider
-
-# Import new types
+from ..common import BaseDetector
 from ..types import GestureCategory, GestureRecognitionResult


-class GestureRecognizer:
+class GestureRecognizer(BaseDetector[GestureRecognitionResult]):
    """Recognizes hand gestures using MediaPipe GestureRecognizer."""

    def __init__(self, model_path: str):
@@ -24,33 +23,96 @@ class GestureRecognizer:
        self._current_options = None
        self._timestamp_provider = None  # Added

-    def _initialize_detector(
-        self,
-        num_hands: int,
-        min_detection_confidence: float,
-        min_tracking_confidence: float,
-        min_presence_confidence: float,
-        running_mode: str,
-        delegate: str,
-    ):  # Added running_mode
-        mode_enum = vision.RunningMode.IMAGE if running_mode == "image" else vision.RunningMode.VIDEO  # Added
-        delegate_enum = BaseOptions.Delegate.CPU if delegate.lower() == "cpu" else BaseOptions.Delegate.GPU
-        base_options = python.BaseOptions(model_asset_path=self.model_path, delegate=delegate_enum)
-        # Note: GestureRecognizerOptions has min_tracking_confidence, useful for VIDEO mode
-        options = vision.GestureRecognizerOptions(
+    def _create_detector_options(self, base_options: python.BaseOptions, 
+                               mode_enum: vision.RunningMode, **kwargs) -> vision.GestureRecognizerOptions:
+        """Create GestureRecognizer-specific options with parameters:
+            - num_hands: Maximum number of hands to detect
+            - min_detection_confidence: Minimum confidence for hand detection
+            - min_tracking_confidence: Minimum confidence for hand tracking
+            - min_presence_confidence: Minimum confidence for hand presence
+        """
+        return vision.GestureRecognizerOptions(
            base_options=base_options,
-            running_mode=mode_enum,  # Use enum mode
-            num_hands=num_hands,
-            min_hand_detection_confidence=min_detection_confidence,
-            min_tracking_confidence=min_tracking_confidence,
-            min_hand_presence_confidence=min_presence_confidence,
-            # Add other relevant VIDEO mode parameters if needed (e.g., canned_gestures_classifier_options)
+            running_mode=mode_enum,
+            num_hands=kwargs.get('num_hands', 2),
+            min_hand_detection_confidence=kwargs.get('min_detection_confidence', 0.5),
+            min_tracking_confidence=kwargs.get('min_tracking_confidence', 0.5),
+            min_hand_presence_confidence=kwargs.get('min_presence_confidence', 0.5),
        )
-        self._current_options = options
-        # Only create timestamp provider for VIDEO mode
-        self._timestamp_provider = TimestampProvider() if mode_enum == vision.RunningMode.VIDEO else None
+
+    def _create_detector_instance(self, options: vision.GestureRecognizerOptions) -> vision.GestureRecognizer:
+        """Create GestureRecognizer instance.
+        
+        Args:
+            options: Configured GestureRecognizerOptions.
+            
+        Returns:
+            MediaPipe GestureRecognizer instance.
+        """
        return vision.GestureRecognizer.create_from_options(options)

+    def _get_options_tuple(self, running_mode: str = None, delegate: str = None, **kwargs) -> tuple:
+        """Get tuple of options for comparison.
+        
+        Args:
+            running_mode: String representation of running mode ("image" or "video").
+            delegate: String representation of delegate ("cpu" or "gpu").
+            **kwargs: GestureRecognizer-specific parameters.
+            
+        Returns:
+            Tuple containing all configuration options for comparison.
+        """
+        if self._current_options:
+            # Getting values from existing options object
+            return (
+                self._current_options.num_hands,
+                self._current_options.min_hand_detection_confidence,
+                self._current_options.min_tracking_confidence,
+                self._current_options.min_hand_presence_confidence,
+                running_mode,
+                delegate,
+            )
+        else:
+            # Getting values from kwargs for new options
+            return (
+                kwargs.get('num_hands', 2),
+                kwargs.get('min_detection_confidence', 0.5),
+                kwargs.get('min_tracking_confidence', 0.5),
+                kwargs.get('min_presence_confidence', 0.5),
+                running_mode,
+                delegate,
+            )
+
+    def _process_detection_result(self, recognition_result: Any) -> List[GestureRecognitionResult]:
+        """Process GestureRecognizer detection results.
+        
+        Args:
+            recognition_result: Raw GestureRecognizer result from MediaPipe.
+            
+        Returns:
+            List of GestureRecognitionResult objects.
+        """
+        current_image_gestures = []
+        if recognition_result and recognition_result.gestures:
+            for hand_idx, hand_gestures in enumerate(recognition_result.gestures):
+                gesture_categories = [
+                    GestureCategory(
+                        index=cat.index,
+                        score=cat.score,
+                        display_name=cat.display_name,
+                        category_name=cat.category_name,
+                    )
+                    for cat in hand_gestures
+                ]
+                handedness = None
+                if recognition_result.handedness and hand_idx < len(recognition_result.handedness):
+                    handedness_cats = recognition_result.handedness[hand_idx]
+                    if handedness_cats:
+                        handedness = handedness_cats[0].display_name
+                current_image_gestures.append(GestureRecognitionResult(gestures=gesture_categories, handedness=handedness))
+        return current_image_gestures
+
+    # Public API method that keeps the same signature as before
    def recognize(
        self,
        image: torch.Tensor,
@@ -60,80 +122,14 @@ class GestureRecognizer:
        min_presence_confidence: float = 0.5,
        running_mode: str = "video",
        delegate: str = "cpu",
-    ) -> List[List[GestureRecognitionResult]]:  # Added running_mode
+    ) -> List[List[GestureRecognitionResult]]:
        """Recognizes hand gestures in the input image tensor."""
-        if image.dim() != 4:
-            raise ValueError("Input tensor must be in BHWC format.")
-
-        # Add running_mode to tuple check
-        new_options_tuple = (
-            num_hands,
-            min_detection_confidence,
-            min_tracking_confidence,
-            min_presence_confidence,
-            running_mode,
-            delegate.lower(),
+        return self.detect(
+            image,
+            running_mode=running_mode,
+            delegate=delegate,
+            num_hands=num_hands,
+            min_detection_confidence=min_detection_confidence,
+            min_tracking_confidence=min_tracking_confidence,
+            min_presence_confidence=min_presence_confidence,
        )
-        current_options_tuple = None
-        if self._current_options:
-            current_options_tuple = (
-                self._current_options.num_hands,
-                self._current_options.min_hand_detection_confidence,
-                self._current_options.min_tracking_confidence,
-                self._current_options.min_hand_presence_confidence,
-                "image" if self._current_options.running_mode == vision.RunningMode.IMAGE else "video",  # Map enum back to string
-                "cpu" if self._current_options.base_options.delegate == BaseOptions.Delegate.CPU else "gpu",
-            )
-
-        if self._detector_instance is None or new_options_tuple != current_options_tuple:
-            if self._detector_instance and hasattr(self._detector_instance, "close"):
-                self._detector_instance.close()
-            # Pass new params to initializer
-            self._detector_instance = self._initialize_detector(
-                num_hands,
-                min_detection_confidence,
-                min_tracking_confidence,
-                min_presence_confidence,
-                running_mode,
-                delegate,
-            )  # Pass running_mode
-
-        batch_size = image.shape[0]
-        batch_results = []
-        # Determine the correct recognition function based on mode
-        is_video_mode = self._current_options.running_mode == vision.RunningMode.VIDEO
-        recognize_func = self._detector_instance.recognize_for_video if is_video_mode else self._detector_instance.recognize
-
-        for i in range(batch_size):
-            img_tensor = image[i]
-            np_image = (img_tensor.cpu().numpy() * 255).astype(np.uint8)
-            mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=np_image)
-
-            if is_video_mode:
-                timestamp_ms = self._timestamp_provider.next()
-                recognition_result = recognize_func(mp_image, timestamp_ms)
-            else:
-                recognition_result = recognize_func(mp_image)
-
-            current_image_gestures = []
-            if recognition_result and recognition_result.gestures:  # Added check for recognition_result
-                for hand_idx, hand_gestures in enumerate(recognition_result.gestures):
-                    gesture_categories = [
-                        GestureCategory(
-                            index=cat.index,
-                            score=cat.score,
-                            display_name=cat.display_name,
-                            category_name=cat.category_name,
-                        )
-                        for cat in hand_gestures
-                    ]
-                    handedness = None
-                    if recognition_result.handedness and hand_idx < len(recognition_result.handedness):
-                        handedness_cats = recognition_result.handedness[hand_idx]
-                        if handedness_cats:
-                            handedness = handedness_cats[0].display_name
-                    current_image_gestures.append(GestureRecognitionResult(gestures=gesture_categories, handedness=handedness))
-            batch_results.append(current_image_gestures)
-
-        # self._detector_instance.close() # Close handled on re-init
-        return batch_results