import itertools import cv2 import numpy as np from shapely.geometry import Polygon from shapely.validation import make_valid class MangaPanelExtractor: def __init__(self, threshold_value: int = 240, min_rel_panel_area: float = 0.01, expansion_pixels: int = 50, max_vertices: int = 6): """ Initialize the manga panel extractor. Args: threshold_value: Threshold for converting to binary (higher = more selective for white) min_rel_panel_area: Minimum panel area as percentage of total page area (0.01 = 1%) expansion_pixels: Pixels to expand image borders for edge panel detection max_vertices: Maximum vertices allowed in simplified polygon (default 6) """ self.threshold_value = threshold_value self.min_rel_panel_area = min_rel_panel_area self.expansion_pixels = expansion_pixels self.max_vertices = max_vertices self.min_panel_area = None # Will be computed from image size def find_empty_corner(self, binary_img: np.ndarray, corner_size: int = 50) -> tuple[int, int]: """ Find an empty corner (white area) to start flood fill. Checks all four corners and returns coordinates of the emptiest one. """ h, w = binary_img.shape corners = [ (0, 0), # Top-left (0, w - 1), # Top-right (h - 1, 0), # Bottom-left (h - 1, w - 1) # Bottom-right ] best_corner = corners[0] max_white_pixels = 0 for corner in corners: y, x = corner # Define sample region around corner y_start = max(0, y - corner_size // 2) y_end = min(h, y + corner_size // 2) x_start = max(0, x - corner_size // 2) x_end = min(w, x + corner_size // 2) sample_region = binary_img[y_start:y_end, x_start:x_end] white_pixels = np.sum(sample_region == 255) if white_pixels > max_white_pixels: max_white_pixels = white_pixels best_corner = corner return best_corner def expand_image(self, img: np.ndarray, pixels: int) -> tuple[np.ndarray, tuple[int, int]]: """ Expand image by adding white border around it. Returns: Expanded image and offset coordinates (for coordinate correction later) """ if len(img.shape) == 3: # Color image expanded = cv2.copyMakeBorder( img, pixels, pixels, pixels, pixels, cv2.BORDER_CONSTANT, value=[255, 255, 255] ) else: # Grayscale expanded = cv2.copyMakeBorder( img, pixels, pixels, pixels, pixels, cv2.BORDER_CONSTANT, value=255 ) return expanded, (pixels, pixels) def simplify_contour(self, contour: np.ndarray, max_vertices: int = None) -> np.ndarray: """ Simplify contour to reduce vertices, preferring rectangular shapes. """ if max_vertices is None: max_vertices = self.max_vertices # Calculate contour perimeter for epsilon calculation perimeter = cv2.arcLength(contour, True) # Start with a small epsilon and gradually increase until we get desired vertex count epsilon_factor = 0.01 simplified = contour while len(simplified) > max_vertices and epsilon_factor < 0.1: epsilon = epsilon_factor * perimeter simplified = cv2.approxPolyDP(contour, epsilon, True) epsilon_factor += 0.005 # If still too many vertices, try more aggressive simplification if len(simplified) > max_vertices: epsilon = 0.1 * perimeter simplified = cv2.approxPolyDP(contour, epsilon, True) return simplified def contour_to_shapely_polygon(self, contour: np.ndarray, offset: tuple[int, int] = (0, 0)) -> Polygon: """ Convert OpenCV contour to Shapely polygon, adjusting for image expansion offset. """ # Adjust coordinates back to original image space offset_x, offset_y = offset points = [] for point in contour: x, y = point[0] # Subtract offset to get original coordinates original_x = x - offset_x original_y = y - offset_y points.append((original_x, original_y)) # Create polygon and ensure it's valid try: polygon = Polygon(points) if not polygon.is_valid: polygon = make_valid(polygon) return polygon except Exception: # Fallback: create a simple polygon from bounding box x_coords = [p[0] for p in points] y_coords = [p[1] for p in points] min_x, max_x = min(x_coords), max(x_coords) min_y, max_y = min(y_coords), max(y_coords) return Polygon([(min_x, min_y), (max_x, min_y), (max_x, max_y), (min_x, max_y)]) def calculate_internal_angle(self, p1: np.ndarray, p2: np.ndarray, p3: np.ndarray) -> float: """ Calculate the internal angle at point p2 formed by p1-p2-p3. Returns angle in degrees. """ # Create vectors v1 = p1 - p2 v2 = p3 - p2 # Calculate angle using dot product cos_angle = np.dot(v1, v2) / (np.linalg.norm(v1) * np.linalg.norm(v2)) cos_angle = np.clip(cos_angle, -1, 1) # Handle floating point errors angle_rad = np.arccos(cos_angle) angle_deg = np.degrees(angle_rad) return angle_deg def score_vertex_for_removal(self, contour: np.ndarray, vertex_idx: int) -> tuple[float, float]: """ Score a vertex for removal based on internal angle and area gain. Returns: (angle_score, area_score) where higher scores = better candidates for removal """ n_points = len(contour) if n_points <= 3: # Can't remove from triangle or smaller return 0.0, 0.0 # Get the three points for angle calculation prev_idx = (vertex_idx - 1) % n_points next_idx = (vertex_idx + 1) % n_points p1 = contour[prev_idx][0] p2 = contour[vertex_idx][0] p3 = contour[next_idx][0] # Calculate internal angle internal_angle = self.calculate_internal_angle(p1, p2, p3) # Angle score: higher for angles close to 180° (straight lines/jagged cuts) # Peak score at 180°, lower scores for acute angles angle_score = 1.0 - abs(internal_angle - 180.0) / 180.0 # Calculate area gain by removing this vertex original_area = cv2.contourArea(contour) # Create new contour without this vertex new_contour = np.delete(contour, vertex_idx, axis=0) if len(new_contour) >= 3: new_area = cv2.contourArea(new_contour) area_gain = new_area - original_area # Normalize area score (higher is better for removal) area_score = max(0, area_gain / original_area) if original_area > 0 else 0 else: area_score = 0.0 return angle_score, area_score def convert_nonconvex_to_convex(self, contour: np.ndarray, max_iterations: int = 10) -> np.ndarray: """ Attempt to convert a non-convex shape to convex by removing problematic vertices. Args: contour: OpenCV contour max_iterations: Maximum number of vertices to remove Returns: Modified contour (convex if successful) """ current_contour = contour.copy() for iteration in range(max_iterations): # Check if already convex if self.is_convex(current_contour): break # Need at least 4 points to continue removing if len(current_contour) <= 3: break # Score all vertices for removal vertex_scores = [] for i in range(len(current_contour)): angle_score, area_score = self.score_vertex_for_removal(current_contour, i) # Combined score: prioritize high internal angles, then area gain combined_score = angle_score * 2.0 + area_score * 1.0 vertex_scores.append((i, combined_score, angle_score, area_score)) # Sort by combined score (highest first) vertex_scores.sort(key=lambda x: x[1], reverse=True) # Remove the vertex with highest score if vertex_scores and vertex_scores[0][1] > 0: best_vertex_idx = vertex_scores[0][0] current_contour = np.delete(current_contour, best_vertex_idx, axis=0) else: # No good candidates for removal break return current_contour def analyze_nonconvex_recovery(self, results: dict) -> dict: """ Attempt to recover panels from non-convex shapes using vertex removal. Returns: Updated results with recovered panels """ recovered_panels = [] remaining_nonconvex = [] for shape in results['non_convex_shapes']: original_contour = shape['contour'] # Attempt to make it convex recovered_contour = self.convert_nonconvex_to_convex(original_contour) # Check if we successfully made it convex and it's still a reasonable size if (self.is_convex(recovered_contour) and cv2.contourArea(recovered_contour) > self.min_panel_area * 0.5): # Allow smaller after recovery # Convert to polygon and analyze polygon = self.contour_to_shapely_polygon(recovered_contour, results['offset']) shape_analysis = self.analyze_panel_shape(polygon) recovered_panels.append({ 'contour': recovered_contour, 'original_contour': original_contour, 'polygon': polygon, 'area': cv2.contourArea(recovered_contour), 'bbox': cv2.boundingRect(recovered_contour), 'shape_analysis': shape_analysis, 'type': 'recovered_panel', 'recovery_info': { 'original_vertices': len(original_contour), 'final_vertices': len(recovered_contour), 'vertices_removed': len(original_contour) - len(recovered_contour) } }) else: # Keep as non-convex remaining_nonconvex.append(shape) # Update results results['recovered_panels'] = recovered_panels results['non_convex_shapes'] = remaining_nonconvex results['panels'].extend(recovered_panels) # Add to main panels list return results def analyze_panel_shape(self, polygon: Polygon) -> dict: """ Analyze the shape characteristics of a panel polygon. """ coords = list(polygon.exterior.coords[:-1]) # Remove duplicate last point num_vertices = len(coords) # Calculate aspect ratio from bounding box bounds = polygon.bounds # (minx, miny, maxx, maxy) width = bounds[2] - bounds[0] height = bounds[3] - bounds[1] aspect_ratio = width / height if height > 0 else 1.0 # Determine likely panel type if num_vertices == 4: panel_type = "rectangular" elif num_vertices == 3: panel_type = "triangular" elif num_vertices <= 6: panel_type = f"{num_vertices}-sided" else: panel_type = "complex" return { 'vertices': num_vertices, 'aspect_ratio': aspect_ratio, 'panel_type': panel_type, 'area': polygon.area, 'bounds': bounds } def is_convex(self, contour: np.ndarray) -> bool: """ Check if a contour represents a convex shape. """ hull = cv2.convexHull(contour) hull_area = cv2.contourArea(hull) contour_area = cv2.contourArea(contour) if hull_area == 0: return False # If the ratio is close to 1, the shape is convex convexity_ratio = contour_area / hull_area return convexity_ratio > 0.85 # Allow some tolerance def extract_panels(self, img: np.ndarray) -> dict: """ Extract panels from manga page using refined flood fill approach. :param img: openCV ready, in BGR format, image. :return: Dictionary containing panels, non-convex shapes, and debug images """ # Load and preprocess image # img = cv2.imread(image_path) #if img is None: # raise ValueError(f"Could not load image: {image_path}") # (image[0].cpu().numpy()[..., ::-1] * 255).astype(np.uint8) # Store original dimensions for reference original_shape = img.shape page_area = original_shape[0] * original_shape[1] # height * width self.min_panel_area = page_area * self.min_rel_panel_area # Convert to grayscale gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) # Expand image to detect edge panels expanded_img, offset = self.expand_image(img, self.expansion_pixels) expanded_gray, _ = self.expand_image(gray, self.expansion_pixels) # Create binary image (white background, black content) _, binary = cv2.threshold(expanded_gray, self.threshold_value, 255, cv2.THRESH_BINARY) # Find empty corner for flood fill (now in expanded image) start_point = self.find_empty_corner(binary) start_point = list(start_point)[::-1] # swap x and y for floodfill =/ # Create flood fill mask h, w = binary.shape mask = np.zeros((h + 2, w + 2), np.uint8) # Perform flood fill from empty corner flood_filled = binary.copy() cv2.floodFill(flood_filled, mask, start_point, 128) # Fill with gray (128) # Create mask of non-filled areas (panels and content) panel_mask = (flood_filled != 128).astype(np.uint8) * 255 # Find contours in the panel mask contours, _ = cv2.findContours(panel_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) # Classify and process contours panels = [] non_convex_shapes = [] for contour in contours: area = cv2.contourArea(contour) # Filter out very small areas (noise) if area < self.min_panel_area: continue # Simplify contour simplified_contour = self.simplify_contour(contour) # Convert to Shapely polygon polygon = self.contour_to_shapely_polygon(simplified_contour, offset) # Analyze shape shape_analysis = self.analyze_panel_shape(polygon) # Check if convex and classify if self.is_convex(simplified_contour): panels.append({ 'contour': simplified_contour, 'original_contour': contour, 'polygon': polygon, 'area': area, 'bbox': cv2.boundingRect(simplified_contour), 'shape_analysis': shape_analysis, 'type': 'panel' }) else: non_convex_shapes.append({ 'contour': simplified_contour, 'original_contour': contour, 'polygon': polygon, 'area': area, 'bbox': cv2.boundingRect(simplified_contour), 'shape_analysis': shape_analysis, 'type': 'non_convex' }) return { 'original': img, 'expanded': expanded_img, 'binary': binary, 'flood_filled': flood_filled, 'panel_mask': panel_mask, 'panels': panels, 'non_convex_shapes': non_convex_shapes, 'recovered_panels': [], # Will be populated if recovery is run 'start_point': start_point, 'offset': offset, 'original_shape': original_shape } # optional user "enforced" panel shape related funcs def simplify_panels(self, results: dict, method: str) -> dict: for i, r in enumerate(results["panels"]): results["panels"][i]["polygon"] = \ MangaPanelExtractor.simplify_polygon_to_rectangle(results["panels"][i]["polygon"], method) return results @staticmethod def simplify_polygon_to_rectangle(polygon: Polygon, method: str = 'max_area_combination') -> Polygon: """ Simplify a polygon with >4 vertices to exactly 4 vertices (rectangle) by maximizing the area of the resulting shape. Args: polygon: Shapely polygon with >4 vertices method: 'max_area_combination' or 'bounding_box' Returns: Simplified 4-vertex polygon with maximum possible area """ coords = list(polygon.exterior.coords[:-1]) # Remove duplicate last point if method == 'bounding_box': return MangaPanelExtractor._simplify_to_bounding_box(polygon) if len(coords) <= 4: return polygon return MangaPanelExtractor.simplify_maximize_area(polygon) @staticmethod def simplify_maximize_area(polygon: Polygon) -> Polygon: """ Simplify polygon to 4 vertices by finding the combination that maximizes area. """ coords, original_area = polygon.exterior.coords, polygon.area n_vertices = len(coords) if n_vertices <= 4: return Polygon(coords) # For very large polygons, use a heuristic approach to avoid combinatorial explosion if n_vertices > 12: return MangaPanelExtractor._simplify_large_polygon_heuristic(coords) # Try all combinations of 4 vertices from the original polygon max_area = 0 best_combination = None for combination in itertools.combinations(range(n_vertices), 4): # Extract the 4 vertices selected_coords = [coords[i] for i in combination] # Ensure proper ordering (clockwise or counter-clockwise) ordered_coords = MangaPanelExtractor._order_vertices_properly(selected_coords) try: candidate_polygon = Polygon(ordered_coords) if not candidate_polygon.is_valid: candidate_polygon = make_valid(candidate_polygon) # We want to maximize area candidate_area = candidate_polygon.area if candidate_area > max_area: max_area = candidate_area best_combination = ordered_coords except Exception: continue # Skip invalid combinations if best_combination is None: # Fallback to bounding box (guaranteed to be large) return MangaPanelExtractor._simplify_to_bounding_box(Polygon(coords)) return Polygon(best_combination) @staticmethod def _order_vertices_properly(vertices: list[tuple[float, float]]) -> list[tuple[float, float]]: """ Order vertices in proper sequence (clockwise or counter-clockwise) for a valid polygon. """ # Calculate centroid centroid_x = sum(v[0] for v in vertices) / len(vertices) centroid_y = sum(v[1] for v in vertices) / len(vertices) # Calculate angle from centroid to each vertex def angle_from_centroid(vertex): return np.arctan2(vertex[1] - centroid_y, vertex[0] - centroid_x) # Sort vertices by angle sorted_vertices = sorted(vertices, key=angle_from_centroid) return sorted_vertices @staticmethod def _simplify_to_bounding_box(polygon: Polygon) -> Polygon: """Convert polygon to its bounding box rectangle.""" bounds = polygon.bounds # (minx, miny, maxx, maxy) minx, miny, maxx, maxy = bounds rectangle_coords = [ (minx, miny), (maxx, miny), (maxx, maxy), (minx, maxy) ] return Polygon(rectangle_coords) @staticmethod def _simplify_large_polygon_heuristic(coords: list[tuple[float, float]]) -> Polygon: """ Heuristic approach for polygons with high vertex count (>12). Selects vertices that are most important for maintaining shape. """ n = len(coords) # Calculate importance score for each vertex vertex_scores = [] for i in range(n): prev_idx = (i - 1) % n next_idx = (i + 1) % n # Calculate the area of triangle formed by this vertex and its neighbors p1, p2, p3 = coords[prev_idx], coords[i], coords[next_idx] triangle_area = abs((p2[0] - p1[0]) * (p3[1] - p1[1]) - (p3[0] - p1[0]) * (p2[1] - p1[1])) / 2 # Calculate distance from centroid (corner vertices are typically further) centroid = (sum(c[0] for c in coords) / n, sum(c[1] for c in coords) / n) distance_from_centroid = ((coords[i][0] - centroid[0]) ** 2 + (coords[i][1] - centroid[1]) ** 2) ** 0.5 # Combined importance score importance = triangle_area * 0.7 + distance_from_centroid * 0.3 vertex_scores.append((i, importance)) # Sort by importance and take top 4 vertex_scores.sort(key=lambda x: x[1], reverse=True) top_4_indices = [idx for idx, _ in vertex_scores[:4]] top_4_indices.sort() # Maintain original ordering selected_coords = [coords[i] for i in top_4_indices] ordered_coords = MangaPanelExtractor._order_vertices_properly(selected_coords) return Polygon(ordered_coords)