diff --git a/README.md b/README.md index d7c8d2d..d306d48 100644 --- a/README.md +++ b/README.md @@ -119,22 +119,48 @@ This is the main node that fetches the image. --- -## 4. (Experimental) Panoramic Loader Node +## 4. Panoramic Loader Node -For users who need to create wide, cinematic landscapes, the project includes an experimental **Street View Pano Loader** node. +For users who need to create wide, cinematic landscapes, the project includes the **Street View Pano Loader** node. -![Street View Pano Loader Node](media/preview_3.png) +This node overcomes the API's FOV limitations by using a sophisticated stitching algorithm. It fetches multiple overlapping image "tiles" and then uses the OpenCV library to analyze, warp, and seamlessly blend them into a single, perspective-corrected panoramic image. -### What It Does -This node overcomes the API's FOV limitations by fetching multiple image "tiles" and stitching them side-by-side. For example, requesting **3 images** will result in three `640x640` images being stitched into a single `1920x640` image. +![Street View Pano Loader Node in ComfyUI](media/preview_3-new.png) -### **⚠️ Important Experimental Notes:** +![Street View Pano Loader Node result](media/StreetView_Pano_00010_.png) + +### How to Use & Parameter Suggestions + +1. Add the **"Street View Pano Loader"** node to your canvas. +2. Provide a `location` and `center_heading` (the direction you want the middle of your panorama to face). +3. Fine-tune the parameters for a successful stitch: + - **`overlap_percentage`**: This is the most critical setting. For OpenCV to work, it needs to see the same features in adjacent images. An overlap of **30-50%** is a great starting point. **If a stitch fails, increase this value first.** + - **`fov_per_image`**: A narrower Field of View (like 70-80) can reduce distortion at the edges of each tile, making it easier for the algorithm to find matching points. However, you may need to increase the `num_images` to capture the same total width. + - **`num_images`**: Controls the final width of your panorama. Start with 3 and increase if needed. + +### Understanding the Output: Warping & Black Borders + +A successful, high-quality stitch will **not** be a perfect rectangle. To correctly align the perspectives, the stitcher "warps" the flat photos onto a virtual cylinder. + +**The curved edges and black borders are not an error; they are proof that the perspective correction worked!** This warped image is now a seamless, geometrically correct panorama, ready for refinement. + +### Refining Your Panorama: Optional Next Steps + +Once you have your stitched result, you have two great options to create a final, rectangular image: + +**1. Cropping (The Simple Method)** +- **Goal:** To get a clean, cinematic widescreen image. +- **How:** Connect the `IMAGE` output from the Pano Loader to a `Crop` node in ComfyUI. Adjust the crop box to frame the best part of the scene and remove the black areas. + +**2. AI Outpainting (The Advanced Method)** +- **Goal:** To use AI to intelligently fill in the missing areas, creating a larger, natural-looking scene. +- **How:** Feed the panoramic image into your main workflow (`VAE Encode`, `KSampler`, etc.) with a descriptive prompt of the scene and a **low denoise** (e.g., 0.3-0.5). The AI will use the existing pixels as a guide to generate new details in the black corners. + +### Important Notes - **API Usage:** This node makes multiple API calls. A panorama with **3 images** will count as **3 requests** against your free monthly Google Cloud credit. -- **Simple Stitching:** This feature uses a basic side-by-side stitch and does not perform advanced perspective correction. It works best for distant landscapes where distortion is minimal. -- **Resolution:** The output image will be very wide but only 640px tall. It is highly recommended to chain the output of this node into an **Upscale Image** node. - ---- +- **Stitching Process:** If the OpenCV algorithm cannot find enough matching features, it will automatically **fall back to a simple side-by-side stitch** to ensure you always get an output. If this happens, the best solution is to increase the `overlap_percentage`. +- **Resolution:** The output image will be very wide but only 640px tall. It is **highly recommended** to chain the output of this node into an **Upscale Image** node to increase the final resolution for your projects. ## 5. Troubleshooting diff --git a/__init__.py b/__init__.py index eaad462..464745e 100644 --- a/__init__.py +++ b/__init__.py @@ -16,8 +16,6 @@ NODE_DISPLAY_NAME_MAPPINGS = { "StreetViewPanoLoader": "Street View Pano Loader", } -__all__ = ['NODE_CLASS_MAPPINGS', 'NODE_DISPLAY_NAME_MAPPINGS'] - print("------------------------------------------") print("ComfyUI Street View Loader Node: Loaded.") print("------------------------------------------") \ No newline at end of file diff --git a/media/StreetView_Pano_00010_.png b/media/StreetView_Pano_00010_.png new file mode 100644 index 0000000..114c4dd Binary files /dev/null and b/media/StreetView_Pano_00010_.png differ diff --git a/media/preview_3-new.png b/media/preview_3-new.png new file mode 100644 index 0000000..97c4055 Binary files /dev/null and b/media/preview_3-new.png differ diff --git a/media/preview_3.png b/media/preview_3.png deleted file mode 100644 index 5d18a80..0000000 Binary files a/media/preview_3.png and /dev/null differ diff --git a/nodes/streetview_loader.py b/nodes/streetview_loader.py index d321748..49c8c37 100644 --- a/nodes/streetview_loader.py +++ b/nodes/streetview_loader.py @@ -7,7 +7,6 @@ from dotenv import load_dotenv from PIL import Image # Import the refactored API call function from our utility file -# No changes are needed in connect_api_utils.py from ..utils.connect_api_utils import fetch_streetview_image # --- Load API Key from .env file --- @@ -44,7 +43,6 @@ class StreetViewLoader: "heading": ("FLOAT", {"default": 151.78, "min": 0, "max": 360, "step": 0.1, "display": "slider"}), "pitch": ("FLOAT", {"default": -0.76, "min": -90, "max": 90, "step": 0.1, "display": "slider"}), "fov": ("INT", {"default": 90, "min": 10, "max": 120, "step": 1, "display": "slider"}), - # NEW: Aspect ratio dropdown replaces width/height inputs "aspect_ratio": ([ "1:1 Square (640x640)", "16:9 Widescreen (640x360)", @@ -61,11 +59,11 @@ class StreetViewLoader: CATEGORY = "Ru4ls/StreetView" def load_image(self, location, heading, pitch, fov, aspect_ratio): - # The API key is now taken directly from the globally loaded variable. + if not API_KEY_FROM_ENV: raise ValueError("Google Street View API key not found in .env file. Please ensure GOOGLE_STREET_VIEW_API_KEY is set in ComfyUI_StreetView-Loader/.env") - # NEW: Logic to determine width and height based on the selected aspect ratio + # Logic to determine width and height based on the selected aspect ratio if aspect_ratio == "1:1 Square (640x640)": width, height = 640, 640 elif aspect_ratio == "16:9 Widescreen (640x360)": diff --git a/nodes/streetview_pano_loader.py b/nodes/streetview_pano_loader.py index efd6024..24a21ca 100644 --- a/nodes/streetview_pano_loader.py +++ b/nodes/streetview_pano_loader.py @@ -5,11 +5,11 @@ import numpy as np import os from dotenv import load_dotenv from PIL import Image +import cv2 -# We can reuse the exact same utility function! This is the power of good refactoring. from ..utils.connect_api_utils import fetch_streetview_image -# --- Load API Key from .env file --- +# --- Load API Key --- current_dir = os.path.dirname(os.path.abspath(__file__)) parent_dir = os.path.dirname(current_dir) dotenv_path = os.path.join(parent_dir, '.env') @@ -19,19 +19,20 @@ API_KEY_FROM_ENV = os.getenv("GOOGLE_STREET_VIEW_API_KEY") class StreetViewPanoLoader: """ - A ComfyUI node to load a panoramic image by fetching multiple Google Street - View images with different headings and stitching them side-by-side. + A ComfyUI node that creates a panorama by fetching multiple Street View + images and stitching them using OpenCV for a seamless result. """ @classmethod def INPUT_TYPES(s): return { "required": { - "location": ("STRING", {"multiline": False, "default": "46.6237597,8.0305018"}), # Switzerland example + "location": ("STRING", {"multiline": False, "default": "46.6237597,8.0305018"}), "center_heading": ("FLOAT", {"default": 133.44, "min": 0, "max": 360, "step": 0.1, "display": "slider"}), "pitch": ("FLOAT", {"default": -5.0, "min": -90, "max": 90, "step": 0.1, "display": "slider"}), "fov_per_image": ("INT", {"default": 90, "min": 30, "max": 120, "step": 1, "display": "slider"}), "num_images": ("INT", {"default": 3, "min": 2, "max": 5, "step": 1, "display": "slider"}), + "overlap_percentage": ("INT", {"default": 30, "min": 10, "max": 70, "step": 1, "display": "slider"}), } } @@ -40,63 +41,57 @@ class StreetViewPanoLoader: FUNCTION = "load_panorama" CATEGORY = "Ru4ls/StreetView" - def load_panorama(self, location, center_heading, pitch, fov_per_image, num_images): - if not API_KEY_FROM_ENV: - raise ValueError("Google Street View API key not found in .env file.") - - images = [] - # We will fetch 640x640 images as they give the most vertical data for stitching. - width, height = 640, 640 - - # --- Calculate Headings for Each Image --- - # This determines how far apart each camera shot is. - # For a simple side-by-side stitch, the step angle is equal to the field of view. - step_angle = fov_per_image - - # Calculate the heading for the very first (leftmost) image - start_heading = center_heading - (step_angle * (num_images - 1) / 2.0) - - print(f"StreetView Pano: Fetching {num_images} images with {fov_per_image}° FOV each.") - - for i in range(num_images): - # Calculate the heading for the current shot in the sequence - current_heading = (start_heading + i * step_angle) % 360 - - print(f" - Fetching image {i+1}/{num_images} at heading {current_heading:.2f}°...") - - # Fetch a single image using our existing utility function - image_pil, _ = fetch_streetview_image( - api_key=API_KEY_FROM_ENV, - location=location, - heading=current_heading, - pitch=pitch, - fov=fov_per_image, - width=width, - height=height - ) - if image_pil: - images.append(image_pil) - - if not images: - print("StreetView Pano: Failed to fetch any images.") - blank_image = torch.zeros((1, height, width, 3), dtype=torch.float32) - return (blank_image, "Failed to fetch any images.") - - # --- Stitch the Images Together --- + def simple_stitch(self, images, width, height): + """ Fallback function for a simple side-by-side stitch if OpenCV fails. """ total_width = width * len(images) stitched_image = Image.new('RGB', (total_width, height)) - for i, img in enumerate(images): stitched_image.paste(img, (i * width, 0)) - - print(f"StreetView Pano: Stitching complete. Final size: {total_width}x{height}") - - # Convert the final stitched PIL image to the tensor format ComfyUI expects - final_tensor = self.pil_to_tensor(stitched_image) - metadata = f"Stitched {len(images)} images. Center Heading: {center_heading}, Total Width: {total_width}px" - - return (final_tensor, metadata) + return stitched_image def pil_to_tensor(self, image: Image.Image): image_np = np.array(image).astype(np.float32) / 255.0 - return torch.from_numpy(image_np)[None,] \ No newline at end of file + return torch.from_numpy(image_np)[None,] + + def load_panorama(self, location, center_heading, pitch, fov_per_image, num_images, overlap_percentage): + if not API_KEY_FROM_ENV: + raise ValueError("Google Street View API key not found in .env file.") + + images_pil = [] + width, height = 640, 640 # Fetch square images for max data + + # Calculate Headings Based on Overlap + step_angle = fov_per_image * (1 - (overlap_percentage / 100.0)) + start_heading = center_heading - (step_angle * (num_images - 1) / 2.0) + + print(f"StreetView Pano: Fetching {num_images} images with {fov_per_image}° FOV and {overlap_percentage}% overlap.") + + for i in range(num_images): + current_heading = (start_heading + i * step_angle) % 360 + print(f" - Fetching image {i+1}/{num_images} at heading {current_heading:.2f}°...") + image_pil, _ = fetch_streetview_image(API_KEY_FROM_ENV, location, current_heading, pitch, fov_per_image, width, height) + if image_pil: + images_pil.append(image_pil) + + if not images_pil: + return (torch.zeros((1, height, width, 3), dtype=torch.float32), "Failed to fetch any images.") + + # --- OpenCV Stitching --- + images_cv = [cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR) for img in images_pil] + + stitcher = cv2.Stitcher_create() + (status, stitched_image_bgr) = stitcher.stitch(images_cv) + + if status == cv2.Stitcher_OK: + print("StreetView Pano: OpenCV stitching successful!") + stitched_image_rgb = cv2.cvtColor(stitched_image_bgr, cv2.COLOR_BGR2RGB) + final_image = Image.fromarray(stitched_image_rgb) + metadata = f"OpenCV Stitched {len(images_pil)} images. Final size: {final_image.width}x{final_image.height}" + else: + print(f"StreetView Pano: OpenCV stitching failed (Status code: {status}). Reason: Not enough matching features.") + print(" - FALLING BACK to simple side-by-side stitching. Try increasing overlap or changing FOV.") + final_image = self.simple_stitch(images_pil, width, height) + metadata = f"STITCHING FAILED. Fallback to simple stitch. Size: {final_image.width}x{final_image.height}" + + final_tensor = self.pil_to_tensor(final_image) + return (final_tensor, metadata) \ No newline at end of file diff --git a/nodes/streetview_url_parser.py b/nodes/streetview_url_parser.py index a8f0bb8..d5275ad 100644 --- a/nodes/streetview_url_parser.py +++ b/nodes/streetview_url_parser.py @@ -20,7 +20,7 @@ class StreetViewURLParser: RETURN_TYPES = ("STRING", "FLOAT", "FLOAT", "INT") RETURN_NAMES = ("location", "heading", "pitch", "fov") FUNCTION = "parse_url" - CATEGORY = "Ru4ls/StreetView/Utils" # A sub-category for utilities + CATEGORY = "Ru4ls/StreetView/Utils" def parse_url(self, url): # Default values in case parsing fails diff --git a/requirements.txt b/requirements.txt index d44fe44..d2fa0f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ requests +opencv-python python-dotenv \ No newline at end of file