From 033f73d66c6ebdabd3added076e06750b9ba4eff Mon Sep 17 00:00:00 2001 From: Eric Rollei <97568200+EricRollei@users.noreply.github.com> Date: Sat, 17 Jan 2026 19:08:01 -0800 Subject: [PATCH] Add ModelMayhem, Wix, Portfolio handlers; fix Reddit video extraction; update dependencies New Features: - Add ModelMayhem handler with username-to-ID resolution and worksafe mode handling - Add Wix handler for Wix-hosted websites - Add Portfolio handler for generic portfolio sites - Add persistent settings utility for saving node preferences Reddit Improvements: - Fix API credential loading in prefers_api() method - Add external video host resolution (RedGifs, Gfycat, Imgur) using yt-dlp - Add v.redd.it video URL resolution Gallery-dl Config: - Add RedGifs, Gfycat, Imgur format preferences - Enable Reddit video downloads with comments=0, videos=true Dependencies: - Add asyncpraw for Reddit API - Add atproto for Bluesky - Add instaloader for Instagram - Add aiohttp for async HTTP - Add Pillow explicitly Other: - Update .gitignore to exclude node_settings.json - Update example configs with new settings --- .gitignore | 1 + README.md | 38 +- configs/auth_config.json.example | 10 + configs/gallery-dl.conf.example | 16 +- nodes/gallery_dl_downloader.py | 221 +++++-- nodes/web_image_scraper_v082.py | 89 ++- requirements.txt | 56 +- site_handlers/modelmayhem_handler.py | 879 +++++++++++++++++++++++++++ site_handlers/portfolio_handler.py | 402 ++++++++++++ site_handlers/reddit_handler.py | 124 +++- site_handlers/wix_handler.py | 695 +++++++++++++++++++++ utils/__init__.py | 19 + utils/persistent_settings.py | 167 +++++ 13 files changed, 2637 insertions(+), 80 deletions(-) create mode 100644 site_handlers/modelmayhem_handler.py create mode 100644 site_handlers/portfolio_handler.py create mode 100644 site_handlers/wix_handler.py create mode 100644 utils/__init__.py create mode 100644 utils/persistent_settings.py diff --git a/.gitignore b/.gitignore index 5cecbc6..ecfb128 100644 --- a/.gitignore +++ b/.gitignore @@ -118,6 +118,7 @@ configs/auth_config.json configs/gallery-dl.conf configs/gallery-dl-*.conf configs/web_scraper/auth_config.json +configs/node_settings.json *.secret *.key diff --git a/README.md b/README.md index cc57305..d3751bf 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,12 @@ ComfyUI custom nodes for downloading media from 1000+ websites including Instagr - **Gallery-dl Node** - Download images and videos from 100+ websites - Instagram, Reddit, Twitter/X, DeviantArt, Pixiv, and more - - Supports authentication via browser cookies + - **Resolution filtering** - Skip small images (default: 768px minimum) + - Supports authentication via config file or browser cookies - Automatic file organization - Download archive to avoid duplicates + - **Persistent config paths** - Paths are remembered between sessions + - **Configurable timeout** - Up to 1 hour for large downloads - **Yt-dlp Node** - Download videos and audio from 1000+ platforms - YouTube, TikTok, Vimeo, Twitch, and more @@ -55,9 +58,11 @@ ComfyUI custom nodes for downloading media from 1000+ websites including Instagr 1. Add "Gallery-dl Downloader" node to your workflow 2. Enter a URL (e.g., Instagram profile, Reddit post) 3. Configure options: - - Enable `use_browser_cookies` for private content + - Set `config_path` to `./configs/gallery-dl.conf` (auto-saved for next time) + - Enable `filter_by_resolution` to skip small images (768px default) - Enable `organize_files` to sort by type - Enable `use_download_archive` to avoid duplicates + - Increase `download_timeout` for large galleries (default: 600s) 4. Execute! **Supported Sites:** Instagram, Reddit, Twitter, DeviantArt, Pixiv, Tumblr, Pinterest, Flickr, and 90+ more. See [gallery-dl supported sites](https://github.com/mikf/gallery-dl/blob/master/docs/supportedsites.md). @@ -82,9 +87,15 @@ ComfyUI custom nodes for downloading media from 1000+ websites including Instagr | Setting | Value | Notes | |---------|-------|-------| -| `config_path` | `./configs/gallery-dl.conf` | Contains your Instagram cookies | +| `config_path` | `./configs/gallery-dl.conf` | Contains all site credentials (auto-saved) | | `cookie_file` | *(leave empty)* | Cookies are in config file | -| `use_browser_cookies` | āŒ False | Browser cookies require admin access | +| `use_browser_cookies` | āŒ False | Config file is more reliable | +| `filter_by_resolution` | āœ… True | Skip thumbnails and small images | +| `min_image_width/height` | 768 | Minimum resolution in pixels | + +### One Config For All Sites + +The `gallery-dl.conf` file contains credentials for **all supported sites** (Instagram, Reddit, 500px, DeviantArt, Pinterest, Flickr, Bluesky). Gallery-dl automatically uses the right credentials based on the URL you're downloading from. ### Why Not Browser Cookies? @@ -138,13 +149,15 @@ Many sites require authentication for private content: Config files are stored in `download-tools/configs/`: -- `gallery-dl.conf` - Gallery-dl settings -- `gallery-dl-browser-cookies.conf` - Browser cookie configuration +- `gallery-dl.conf` - **Main config** with credentials for all sites (Instagram, Reddit, 500px, DeviantArt, Pinterest, Flickr, Bluesky) +- `gallery-dl.conf.example` - Template for creating your own config - `yt-dlp.conf` - Yt-dlp default settings - `yt-dlp-audio.conf` - Audio extraction preset - `yt-dlp-hq.conf` - High quality preset -You can create custom config files and reference them in the nodes. +### Persistent Paths + +Config file paths are **automatically saved** between ComfyUI sessions. Once you set `config_path` or `cookie_file`, they'll be remembered for next time. ## šŸ“š Documentation @@ -173,6 +186,17 @@ Authentication required: - Or enable `use_browser_cookies` with Firefox - 401 Unauthorized = expired cookies, export fresh ones +### "Downloads timing out" +For large galleries: +- Increase `download_timeout` (default: 600s, max: 3600s = 1 hour) +- Large Instagram profiles may need the full hour + +### "Too many small images" +Enable resolution filtering: +- Set `filter_by_resolution` to True +- Adjust `min_image_width` and `min_image_height` (default: 768px) +- Videos are never filtered, only images + ### "CUDA out of memory" / "FFmpeg not found" For audio extraction: - Install FFmpeg separately diff --git a/configs/auth_config.json.example b/configs/auth_config.json.example index 036ca48..bb07faf 100644 --- a/configs/auth_config.json.example +++ b/configs/auth_config.json.example @@ -38,6 +38,16 @@ "domain": "example.com", "username": "YOUR_USERNAME", "password": "YOUR_PASSWORD" + }, + "modelmayhem.com": { + "auth_type": "form", + "domain": "modelmayhem.com", + "username": "YOUR_EMAIL", + "password": "YOUR_PASSWORD", + "timeout": 30000, + "scroll_delay_ms": 1500, + "max_scroll_count": 15, + "note": "Login credentials for ModelMayhem portfolio access" } } } \ No newline at end of file diff --git a/configs/gallery-dl.conf.example b/configs/gallery-dl.conf.example index 8e8f018..2eca8d1 100644 --- a/configs/gallery-dl.conf.example +++ b/configs/gallery-dl.conf.example @@ -9,7 +9,21 @@ "client-secret": "YOUR_REDDIT_CLIENT_SECRET", "user-agent": "MediaTool (by /u/YOUR_USERNAME)", "username": "YOUR_REDDIT_USERNAME", - "password": "YOUR_REDDIT_PASSWORD" + "password": "YOUR_REDDIT_PASSWORD", + "comments": 0, + "videos": true + }, + + "redgifs": { + "format": ["hd", "sd", "gif"] + }, + + "gfycat": { + "format": ["mp4", "webm", "gif"] + }, + + "imgur": { + "mp4": true }, "500px": { diff --git a/nodes/gallery_dl_downloader.py b/nodes/gallery_dl_downloader.py index da96c97..db375bb 100644 --- a/nodes/gallery_dl_downloader.py +++ b/nodes/gallery_dl_downloader.py @@ -53,6 +53,26 @@ import folder_paths from pathlib import Path from typing import Any, Dict, List, Tuple +# Import persistent settings manager +try: + from ..utils.persistent_settings import get_persistent_setting, set_persistent_setting + PERSISTENT_SETTINGS_AVAILABLE = True +except ImportError: + try: + import sys + utils_dir = os.path.dirname(os.path.dirname(__file__)) + if utils_dir not in sys.path: + sys.path.insert(0, utils_dir) + from utils.persistent_settings import get_persistent_setting, set_persistent_setting + PERSISTENT_SETTINGS_AVAILABLE = True + except ImportError: + PERSISTENT_SETTINGS_AVAILABLE = False + def get_persistent_setting(node_type, key, default=""): + return default + def set_persistent_setting(node_type, key, value): + return False + print("Warning: persistent_settings not available. Config paths will not persist.") + class GalleryDLDownloader: def __init__( @@ -72,6 +92,12 @@ class GalleryDLDownloader: # New advanced options instagram_include: str = "posts", extra_options: str = "", + # Minimum resolution filtering + min_image_width: int = 768, + min_image_height: int = 768, + filter_by_resolution: bool = True, + # Timeout settings + download_timeout: int = 600, ): self.url_list = url_list or [] self.url_file = url_file @@ -87,6 +113,10 @@ class GalleryDLDownloader: self.organize_files = organize_files self.instagram_include = instagram_include self.extra_options = extra_options + self.min_image_width = min_image_width + self.min_image_height = min_image_height + self.filter_by_resolution = filter_by_resolution + self.download_timeout = download_timeout # Initialize debug info self.debug_info = [] # Store debug information for status reporting @@ -106,11 +136,6 @@ class GalleryDLDownloader: command += ["-d", self.output_dir] - # Use config file if provided - if self.config_path: - command += ["--config", self.config_path] - self.debug_info.append(f"šŸ“„ Using config file: {self.config_path}") - # Detect target sites for better authentication handling target_sites = set() for url in urls: @@ -120,7 +145,20 @@ class GalleryDLDownloader: target_sites.add('reddit') elif 'twitter.com' in url or 'x.com' in url: target_sites.add('twitter') - # Add more site detection as needed + elif 'flickr.com' in url: + target_sites.add('flickr') + elif 'deviantart.com' in url: + target_sites.add('deviantart') + elif '500px.com' in url: + target_sites.add('500px') + elif 'pinterest' in url: + target_sites.add('pinterest') + elif 'bsky.app' in url or 'bluesky' in url: + target_sites.add('bluesky') + elif 'tumblr.com' in url: + target_sites.add('tumblr') + elif 'artstation.com' in url: + target_sites.add('artstation') if target_sites: self.debug_info.append(f"šŸŽÆ Detected target sites: {', '.join(target_sites)}") @@ -130,23 +168,30 @@ class GalleryDLDownloader: self.debug_info.append("āš ļø WARNING: Reddit may hang or require updated API credentials") self.debug_info.append(" Consider testing with Instagram or other sites first") - # Handle cookie file (takes precedence over browser cookies) - if self.cookie_file: - # Warn if using cookies for non-matching sites (but allow Instagram + config combo) - if target_sites and 'instagram' not in target_sites: - self.debug_info.append(f"āš ļø Warning: Using Instagram cookies for {', '.join(target_sites)} - this may cause authentication conflicts") - elif target_sites and 'instagram' in target_sites: - self.debug_info.append("šŸ”— Using Instagram cookies - optimal setup!") - + # Use config file if provided (contains site-specific credentials) + if self.config_path: + command += ["--config", self.config_path] + self.debug_info.append(f"šŸ“„ Using config file: {self.config_path}") + + # Handle authentication: Browser cookies toggle vs cookie file + # When use_browser_cookies is TRUE: use browser cookies (even if cookie file exists) + # When use_browser_cookies is FALSE: use cookie file if provided + if self.use_browser_cookies: + # User explicitly wants browser cookies + command += ["--cookies-from-browser", self.browser_name] + self.debug_info.append(f"šŸŖ Using cookies from {self.browser_name} browser (toggle enabled)") + if self.cookie_file: + self.debug_info.append(f" ā„¹ļø Note: Cookie file '{self.cookie_file}' ignored - browser cookies selected") + # Test if browser cookies can be accessed + self._test_browser_cookie_access() + elif self.cookie_file: + # Use cookie file when browser cookies toggle is off converted_cookie_file = self._convert_cookie_file() if converted_cookie_file: command += ["--cookies", converted_cookie_file] self.debug_info.append(f"šŸŖ Using cookie file: {converted_cookie_file}") - elif self.use_browser_cookies: - command += ["--cookies-from-browser", self.browser_name] - self.debug_info.append(f"šŸŖ Extracting cookies from {self.browser_name} browser") - # Test if browser cookies can be accessed - self._test_browser_cookie_access() + if 'instagram' in target_sites: + self.debug_info.append("šŸ”— Instagram + cookie file - optimal setup!") if self.use_download_archive: command += ["--download-archive", self.archive_file] @@ -159,6 +204,39 @@ class GalleryDLDownloader: ] self.debug_info.append("šŸŽ¬ Video files will be skipped") + # Add minimum resolution filter for images if enabled + if self.filter_by_resolution and (self.min_image_width > 0 or self.min_image_height > 0): + # gallery-dl filter expression for minimum resolution + # IMPORTANT: Not all extractors provide width/height metadata (e.g., Bunkr, Cyberdrop) + # gallery-dl uses eval() with metadata as locals(), so we can use locals().get() + # If width/height don't exist, we default to 99999 so the file passes the filter + # (download files with unknown dimensions rather than skip them) + filter_expr = [] + if self.min_image_width > 0: + # If width is not available in metadata, default to 99999 (pass filter) + filter_expr.append(f"locals().get('width', 99999) >= {self.min_image_width}") + if self.min_image_height > 0: + # If height is not available in metadata, default to 99999 (pass filter) + filter_expr.append(f"locals().get('height', 99999) >= {self.min_image_height}") + + # Combine with video filter if needed, or apply image filter + # Note: gallery-dl only allows one --filter, so we need to combine + if self.skip_videos: + # Already added video filter above, need to modify it + # Remove the last filter we added and combine + command = [c for c in command if c != "--filter"] + command = [c for c in command if "extension not in" not in c] + combined_filter = f"(extension not in ('mp4', 'webm', 'mkv', 'avi', 'mov', 'wmv', 'flv', 'm4v')) and ({' and '.join(filter_expr)})" + command += ["--filter", combined_filter] + else: + # Only filter images by resolution (videos pass through) + # Apply filter only to images, let videos through + # Also pass through if width/height are not available (unknown dimensions) + filter_condition = f"(extension in ('mp4', 'webm', 'mkv', 'avi', 'mov', 'wmv', 'flv', 'm4v')) or ({' and '.join(filter_expr)})" + command += ["--filter", filter_condition] + + self.debug_info.append(f"šŸ“ Minimum resolution filter: {self.min_image_width}x{self.min_image_height}px (files with unknown dimensions will be downloaded)") + # Add rate limiting to be respectful to servers (use correct format) command += ["--sleep", "1.0"] @@ -440,15 +518,15 @@ class GalleryDLDownloader: universal_newlines=True, cwd=self.output_dir, # Set working directory ) - stdout, stderr = process.communicate(timeout=300) # 5 minute timeout + stdout, stderr = process.communicate(timeout=self.download_timeout) # Use configurable timeout process_returncode = process.returncode except subprocess.TimeoutExpired: process.kill() process_success = False process_returncode = -1 - stderr = "Download process timed out after 5 minutes" + stderr = f"Download process timed out after {self.download_timeout // 60} minutes" stdout = "" - self.debug_info.append("ā° Download timed out, but continuing with file organization...") + self.debug_info.append(f"ā° Download timed out after {self.download_timeout}s, but continuing with file organization...") except Exception as e: process_success = False process_returncode = -1 @@ -671,65 +749,108 @@ class GalleryDLNode: @classmethod def INPUT_TYPES(cls) -> Dict[str, Any]: + # Load saved config paths from persistent settings + saved_config_path = get_persistent_setting("gallery_dl", "config_path", "") + saved_cookie_file = get_persistent_setting("gallery_dl", "cookie_file", "") + return { "required": { "url_list": ("STRING", { "multiline": True, - "default": "# Enter URLs here, one per line\n# Example:\n# https://imgur.com/gallery/example\n# https://example.com/image.jpg" + "default": "# Enter URLs here, one per line\n# Supports 100+ sites: Instagram, Reddit, Twitter/X, DeviantArt, Pixiv, 500px, Flickr, Pinterest, Bluesky, and more\n# Examples:\n# https://www.instagram.com/username/\n# https://www.reddit.com/r/subreddit/\n# https://twitter.com/username", + "tooltip": "Enter URLs to download from, one per line. Supports Instagram, Reddit, Twitter/X, DeviantArt, Pixiv, 500px, Flickr, Pinterest, Bluesky, and 90+ other sites. Lines starting with # are ignored." }), "output_dir": ("STRING", { "default": "./gallery-dl-output", - "tooltip": "Directory where downloaded files will be saved" + "tooltip": "Directory where downloaded files will be saved. Relative paths are relative to ComfyUI's output folder. Files are organized into subfolders by site and user." }) }, "optional": { "url_file": ("STRING", { "default": "", - "tooltip": "Path to a text file containing URLs (one per line)" + "tooltip": "Optional: Path to a text file containing URLs (one per line). Useful for batch downloading from a prepared list. Leave empty to use url_list above." }), "config_path": ("STRING", { - "default": "", - "tooltip": "Path to gallery-dl config file. For Instagram: use './configs/gallery-dl.conf' (has cookies). For Reddit: use './configs/gallery-dl-no-reddit.conf' or add API credentials to gallery-dl.conf." + "default": saved_config_path, + "tooltip": "Path to gallery-dl.conf config file. This single file contains credentials for ALL sites (Instagram, Reddit, 500px, etc.) - gallery-dl auto-selects the right ones based on URL. Path is auto-saved for next time. Recommended: ./configs/gallery-dl.conf" }), "cookie_file": ("STRING", { - "default": "", - "tooltip": "Path to exported cookie JSON file. For Instagram: use './configs/instagram_cookies.json'. For Reddit: LEAVE EMPTY (use browser cookies instead)." + "default": saved_cookie_file, + "tooltip": "Path to exported cookie file (Netscape/JSON format). Only used when 'Use Browser Cookies' is OFF and you need site-specific cookies not in config. Usually leave empty - config file credentials work better. Path is auto-saved." }), "use_browser_cookies": ("BOOLEAN", { - "default": True, - "tooltip": "Use browser cookies for authentication (ignored if cookie file is provided)" + "default": False, + "label_on": "Browser Cookies", + "label_off": "Config/Cookie File", + "tooltip": "OFF (recommended): Use credentials from config file - most reliable. ON: Extract cookies directly from browser (Firefox works best; Chrome/Edge require admin rights and browser closed)." }), "browser_name": (["firefox", "chrome", "chromium", "edge", "safari", "opera"], { "default": "firefox", - "tooltip": "Browser to extract cookies from. Firefox works without admin. Chrome/Edge require admin AND browser closed. Recommended: use config file with cookies instead." + "tooltip": "Browser to extract cookies from when 'Use Browser Cookies' is ON. Firefox recommended - works without admin rights. Chrome/Edge need admin + browser closed." }), "use_download_archive": ("BOOLEAN", { "default": True, - "tooltip": "Use archive file to skip already downloaded content" + "label_on": "Skip Downloaded", + "label_off": "Re-download All", + "tooltip": "ON (recommended): Track downloaded files in SQLite database to skip duplicates on future runs. OFF: Re-download everything, may create duplicates." }), "archive_file": ("STRING", { "default": "./gallery-dl-archive.sqlite3", - "tooltip": "Path to the download archive database" + "tooltip": "Path to download archive database (SQLite). Tracks what's been downloaded to avoid duplicates. Delete this file to force re-downloading everything." }), "skip_videos": ("BOOLEAN", { "default": False, - "tooltip": "Skip video files, download only images" + "label_on": "Images Only", + "label_off": "Images + Videos", + "tooltip": "ON: Download only images, skip all video files. OFF: Download both images and videos. Useful when you only want still images from a mixed gallery." + }), + "filter_by_resolution": ("BOOLEAN", { + "default": True, + "label_on": "Filter Small Images", + "label_off": "Keep All Sizes", + "tooltip": "ON: Skip images smaller than min_image_width/height (removes thumbnails, icons, low-res images). Videos are never filtered. Sites without dimension metadata (Bunkr, Cyberdrop) will download all images. OFF: Download all images regardless of size." + }), + "min_image_width": ("INT", { + "default": 768, + "min": 0, + "max": 8192, + "step": 64, + "tooltip": "Minimum image width in pixels when filter_by_resolution is ON. Images narrower than this are skipped. Set to 0 to disable width filtering. Note: Sites that don't provide dimension metadata will download all images." + }), + "min_image_height": ("INT", { + "default": 768, + "min": 0, + "max": 8192, + "step": 64, + "tooltip": "Minimum image height in pixels when filter_by_resolution is ON. Images shorter than this are skipped. Set to 0 to disable height filtering. Note: Sites that don't provide dimension metadata will download all images." }), "extract_metadata": ("BOOLEAN", { "default": True, - "tooltip": "Save download metadata to JSON file" + "label_on": "Save Metadata", + "label_off": "No Metadata", + "tooltip": "ON: Save download metadata to JSON file (URLs, timestamps, file counts, errors). Useful for tracking what was downloaded. OFF: No metadata file created." }), "organize_files": ("BOOLEAN", { "default": True, - "tooltip": "Sort downloaded files into subfolders (images/, videos/, audio/, other/)" + "label_on": "Sort by Type", + "label_off": "All in One Folder", + "tooltip": "ON: Sort downloaded files into subfolders by type (images/, videos/, audio/, other/). OFF: Put all files in the main output folder without sorting." }), "instagram_include": (["posts", "stories", "highlights", "reels", "tagged", "info", "avatar", "all", "posts,stories", "posts,reels", "stories,highlights"], { "default": "posts", - "tooltip": "For Instagram profiles: What to include (posts, stories, highlights, reels, tagged, info, avatar, or all)" + "tooltip": "Instagram only: What content to download. 'posts' = feed posts, 'stories' = current stories (24hr), 'highlights' = saved story highlights, 'reels' = reels videos, 'all' = everything. Combine with comma: 'posts,reels'." + }), + "download_timeout": ("INT", { + "default": 600, + "min": 60, + "max": 3600, + "step": 60, + "tooltip": "Maximum time in seconds to wait for downloads to complete. Default 600s (10 min). Increase for large galleries - Instagram profiles with 1000+ posts may need 1800-3600s (30-60 min)." }), "extra_options": ("STRING", { "default": "", - "tooltip": "Additional options for gallery-dl (e.g., '--no-skip-download')" + "multiline": True, + "tooltip": "Advanced gallery-dl options (one per line or space-separated):\nā€¢ --limit N: Max N files to download\nā€¢ --range 1-50: Download items 1-50 only\nā€¢ --no-download: Simulate without downloading\nā€¢ --write-metadata: Save per-file JSON metadata\nā€¢ --ugoira-conv: Convert Pixiv ugoira to video\nā€¢ -v: Verbose output for debugging\nā€¢ --sleep N: Wait N seconds between requests\nā€¢ --retries N: Retry failed downloads N times\nSee gallery-dl docs: https://github.com/mikf/gallery-dl/blob/master/docs/options.md" }) } } @@ -739,7 +860,7 @@ class GalleryDLNode: FUNCTION = "execute" CATEGORY = "Downloaders" - DESCRIPTION = "Download images and media from various websites using gallery-dl" + DESCRIPTION = "Download images and media from 100+ websites using gallery-dl. Supports Instagram, Reddit, Twitter/X, DeviantArt, Pixiv, 500px, Flickr, Pinterest, Bluesky, and more. Features: resolution filtering, duplicate detection, automatic file organization, and persistent config paths." def execute( self, @@ -753,10 +874,14 @@ class GalleryDLNode: use_download_archive: bool = True, archive_file: str = "./gallery-dl-archive.sqlite3", skip_videos: bool = False, + filter_by_resolution: bool = True, + min_image_width: int = 768, + min_image_height: int = 768, extract_metadata: bool = True, organize_files: bool = True, # New advanced options instagram_include: str = "posts", + download_timeout: int = 600, extra_options: str = "", ) -> Tuple[str, str, int, bool]: """ @@ -766,6 +891,10 @@ class GalleryDLNode: tuple: (output_dir, summary, download_count, success) """ try: + # Save original config values for persistence before cleaning + original_config_path = config_path.strip() if config_path else "" + original_cookie_file = cookie_file.strip() if cookie_file else "" + # Clean up input parameters if url_file and url_file.strip() == "": url_file = None @@ -775,6 +904,14 @@ class GalleryDLNode: cookie_file = None if archive_file and archive_file.strip() == "": archive_file = "./gallery-dl-archive.sqlite3" + + # Save config_path and cookie_file for future use if they were provided + if original_config_path: + set_persistent_setting("gallery_dl", "config_path", original_config_path) + print(f"Saved config_path for future use: {original_config_path}") + if original_cookie_file: + set_persistent_setting("gallery_dl", "cookie_file", original_cookie_file) + print(f"Saved cookie_file for future use: {original_cookie_file}") # Sanitize output directory base_output_dir = os.path.abspath(folder_paths.get_output_directory()) @@ -829,9 +966,13 @@ class GalleryDLNode: use_download_archive=use_download_archive, archive_file=archive_file, skip_videos=skip_videos, + filter_by_resolution=filter_by_resolution, + min_image_width=min_image_width, + min_image_height=min_image_height, extract_metadata=extract_metadata, organize_files=organize_files, instagram_include=instagram_include, + download_timeout=download_timeout, extra_options=extra_options, ) diff --git a/nodes/web_image_scraper_v082.py b/nodes/web_image_scraper_v082.py index de39f94..1cdfefe 100644 --- a/nodes/web_image_scraper_v082.py +++ b/nodes/web_image_scraper_v082.py @@ -97,6 +97,27 @@ except ImportError: import pathlib +# Import persistent settings manager +try: + from ..utils.persistent_settings import get_persistent_setting, set_persistent_setting + PERSISTENT_SETTINGS_AVAILABLE = True +except ImportError: + try: + import sys + utils_dir = os.path.dirname(os.path.dirname(__file__)) + if utils_dir not in sys.path: + sys.path.insert(0, utils_dir) + from utils.persistent_settings import get_persistent_setting, set_persistent_setting + PERSISTENT_SETTINGS_AVAILABLE = True + except ImportError: + PERSISTENT_SETTINGS_AVAILABLE = False + def get_persistent_setting(node_type, key, default=""): + return default + def set_persistent_setting(node_type, key, value): + return False + print("Warning: persistent_settings not available. Config paths will not persist.") + + # Direct Playwright support try: import playwright @@ -292,8 +313,15 @@ def load_site_handlers(): elif handler_class.__name__ == "YouTubeHandler": return 5 # Give site-specific handlers higher priority - elif handler_class.__name__ in ["RedditHandler", "BskyHandler", "FlickrHandler", "ArtsyHandler"]: + elif handler_class.__name__ in ["RedditHandler", "BskyHandler", "FlickrHandler", "ArtsyHandler", "ModelMayhemHandler"]: return 10 + # Give WixHandler high priority (Wix-powered sites need special handling) + elif handler_class.__name__ == "WixHandler": + return 15 + # Give PortfolioHandler medium-high priority (handles generic portfolio sites) + # Note: PortfolioHandler must be lower priority than site-specific handlers like ModelMayhemHandler + elif handler_class.__name__ == "PortfolioHandler": + return 50 # Default priority return 100 @@ -326,6 +354,9 @@ class EricWebFileScraper: def INPUT_TYPES(cls): # Define common hash algorithms based on availability hash_algos = ["average_hash", "phash", "dhash", "whash"] if IMAGEHASH_AVAILABLE else ["none"] + + # Load saved auth_config_path from persistent settings + saved_auth_config = get_persistent_setting("web_scraper", "auth_config_path", "") return { "required": { @@ -359,7 +390,7 @@ class EricWebFileScraper: "crawl_subfolders": ("BOOLEAN", {"default": True, "label_on": "Subfolder Per Page", "label_off": "Single Folder"}), "skip_first_page_download": ("BOOLEAN", {"default": False, "label_on": "Skip Listing Page", "label_off": "Download All Pages"}), "link_include_pattern": ("STRING", {"default": "", "multiline": False, "placeholder": "Regex to match links (e.g. /[A-Z][a-z]+_)"}), - "link_exclude_pattern": ("STRING", {"default": "/production|/men|/women|/direct|/newface|/contacts|/about|/policy|/store|international", "multiline": False, "placeholder": "Regex to exclude links"}), + "link_exclude_pattern": ("STRING", {"default": "", "multiline": False, "placeholder": "Regex: /contact|/news|/about"}), # --- Media Types --- "download_audio": ("BOOLEAN", {"default": False, "label_on": "Extract Audio", "label_off": "Skip Audio"}), @@ -382,7 +413,7 @@ class EricWebFileScraper: "scroll_delay_ms": ("INT", {"default": 1000, "min": 50, "max": 5000, "step": 50, "display": "Scroll Delay (ms)"}), # --- Interactions & Auth --- "interaction_sequence": ("STRING", {"multiline": True, "placeholder": '[{"type": "click", "selector": "#button"}, ...]', "display": "Interaction Sequence (JSON)"}), - "auth_config_path": ("STRING", {"default": "", "placeholder": "Path to auth_config.json"}), + "auth_config_path": ("STRING", {"default": saved_auth_config, "placeholder": "Path to auth_config.json (saved for future use)"}), "save_cookies": ("BOOLEAN", {"default": False, "label_on": "Save Cookies", "label_off": "Don't Save Cookies"}), # --- Screenshots & Debug --- "take_screenshot": ("BOOLEAN", {"default": False, "label_on": "Take Screenshot", "label_off": "No Screenshot"}), @@ -987,6 +1018,11 @@ class EricWebFileScraper: return ("", "", 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, json.dumps(stats)) self.auth_config = self.load_auth_config(auth_config_path) + + # Save auth_config_path for future use if it was provided + if auth_config_path and auth_config_path.strip(): + set_persistent_setting("web_scraper", "auth_config_path", auth_config_path) + print(f"Saved auth_config_path for future use: {auth_config_path}") handler_instance = self._get_handler_for_url(url) if handler_instance: @@ -2315,7 +2351,8 @@ class EricWebFileScraper: cdn_domains = [ 'tildacdn.com', 'cloudfront.net', 'cloudflare.com', 'akamaihd.net', 'fastly.net', 'imgix.net', 'twimg.com', - 'cdninstagram.com', 'googleapis.com', 'gstatic.com' + 'cdninstagram.com', 'googleapis.com', 'gstatic.com', + 'photos.modelmayhem.com', 'assets.modelmayhem.com' # ModelMayhem CDN domains ] is_trusted_cdn = any(cdn in item_domain.lower() for cdn in cdn_domains) if is_trusted_cdn: @@ -2591,12 +2628,51 @@ class EricWebFileScraper: if is_video and not download_videos: reason = "Video download disabled" return None, None, None, None, None, f"Skipped ({reason}): {url}" + # Get item dict for additional metadata (page_url, qualities, etc.) + item = url_or_dict if isinstance(url_or_dict, dict) else {} + try: print(f" Downloading {file_type}: {url}") headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'} + + # Add Referer header for Wix videos (required to avoid 403) + page_url = item.get('page_url', '') + if 'wixstatic.com' in url: + if page_url: + headers['Referer'] = page_url + headers['Origin'] = '/'.join(page_url.split('/')[:3]) # Extract origin + else: + # Try to construct a reasonable referer from the URL + headers['Referer'] = 'https://www.wix.com/' + response = requests.get(url, stream=True, timeout=30, headers=headers) - if response.status_code != 200: + # Handle Wix video 403 errors with quality fallback + if response.status_code == 403 and 'video.wixstatic.com' in url: + qualities = item.get('qualities', {}) + original_url = item.get('original_url', '') + + # Try fallback qualities: 720p, 480p, then original + fallback_urls = [] + if qualities: + for q in ['720p', '480p', '360p']: + if q in qualities and qualities[q] != url: + fallback_urls.append((q, qualities[q])) + if original_url and original_url != url: + fallback_urls.append(('original', original_url)) + + for quality, fallback_url in fallback_urls: + print(f" Trying {quality} fallback: {fallback_url}") + response = requests.get(fallback_url, stream=True, timeout=30, headers=headers) + if response.status_code == 200: + url = fallback_url # Update URL for filename generation + print(f" Success with {quality} quality") + break + else: + # All fallbacks failed + print(f"HTTP error {response.status_code} for {url} (all quality fallbacks failed)") + return None, None, None, None, None, f"HTTP error {response.status_code}" + elif response.status_code != 200: print(f"HTTP error {response.status_code} for {url}") return None, None, None, None, None, f"HTTP error {response.status_code}" @@ -3857,7 +3933,8 @@ class EricWebFileScraper: # Fallback: check common CDN domains directly (handles items from DevTools cache) if not is_trusted_cdn: cdn_domains = ['tildacdn.com', 'cloudfront.net', 'cloudflare.com', 'akamaized.net', - 'fastly.net', 'cdn.', 'assets.', 'static.', 'media.', 'images.'] + 'fastly.net', 'cdn.', 'assets.', 'static.', 'media.', 'images.', + 'photos.modelmayhem.com', 'assets.modelmayhem.com'] # ModelMayhem CDN is_trusted_cdn = any(cdn in item_domain.lower() for cdn in cdn_domains) if is_trusted_cdn: with global_lock: diff --git a/requirements.txt b/requirements.txt index 1826928..c931503 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,9 @@ # Download Tools - Requirements # ComfyUI custom nodes for media downloading +# Author: Eric Hiss (GitHub: EricRollei) # ============================================================================ -# DOWNLOADER TOOLS (Required) +# DOWNLOADER TOOLS (Required for gallery-dl and yt-dlp nodes) # ============================================================================ # Gallery-dl for downloading from various websites @@ -11,25 +12,44 @@ gallery-dl>=1.26.0 # Yt-dlp for video/audio downloading yt-dlp>=2023.12.0 -# Browser cookie support (for authentication) +# Browser cookie support (for authentication with gallery-dl/yt-dlp) browser-cookie3>=0.19.0 # ============================================================================ -# WEB SCRAPER (Required for web_image_scraper node) +# WEB SCRAPER - Core Dependencies (Required for web_image_scraper node) # ============================================================================ -# Playwright for browser automation +# Playwright for browser automation (run 'playwright install' after pip install) playwright>=1.40.0 -# Scrapling for enhanced web scraping +# Scrapling for enhanced web scraping with anti-bot detection scrapling>=0.2.0 # Image hashing for deduplication imagehash>=4.3.0 +# PIL/Pillow for image processing +Pillow>=10.0.0 + # Async event loop nesting for ComfyUI compatibility nest-asyncio>=1.5.0 +# ============================================================================ +# WEB SCRAPER - Site-Specific API Libraries (Optional but recommended) +# ============================================================================ + +# AsyncPRAW for Reddit API access +asyncpraw>=7.7.0 + +# AT Protocol SDK for Bluesky +atproto>=0.0.40 + +# Instaloader for Instagram (optional, provides additional features) +instaloader>=4.10.0 + +# aiohttp for async HTTP requests (used by some handlers) +aiohttp>=3.9.0 + # ============================================================================ # CORE UTILITIES (Required) # ============================================================================ @@ -46,25 +66,33 @@ colorama>=0.4.6 jsonschema>=4.19.0 # ============================================================================ -# NOTES +# INSTALLATION NOTES # ============================================================================ -# 1. FFmpeg is required for yt-dlp audio extraction +# 1. FFmpeg is required for yt-dlp audio extraction and video processing # Download from: https://ffmpeg.org/download.html # Or install via package manager: # - Windows: choco install ffmpeg # - macOS: brew install ffmpeg # - Linux: apt-get install ffmpeg -# 2. For gallery-dl authentication with private content: -# - Use browser cookies (automatic) +# 2. After installing playwright, run: playwright install +# This downloads the browser binaries needed for web scraping. + +# 3. For gallery-dl authentication with private content: +# - Use browser cookies (automatic via browser-cookie3) # - Or export cookies to configs/cookies.json -# 3. For web_image_scraper node: -# - After installing playwright, run: playwright install -# - Configure authentication in configs/web_scraper/auth_config.json -# - See configs/web_scraper/README.md for auth setup guide +# 4. For web_image_scraper node: +# - Configure authentication in configs/auth_config.json +# - See CONFIG_SETUP.md for auth setup guide -# 4. These tools are installed system-wide and can be used from command line: +# 5. Site-specific API setup: +# - Reddit: Create app at https://www.reddit.com/prefs/apps +# Add client_id, client_secret to auth_config.json +# - Bluesky: Add username/password to auth_config.json +# - Instagram: Add cookies or use browser session + +# 6. These tools can also be used from command line: # - gallery-dl # - yt-dlp diff --git a/site_handlers/modelmayhem_handler.py b/site_handlers/modelmayhem_handler.py new file mode 100644 index 0000000..fcd6370 --- /dev/null +++ b/site_handlers/modelmayhem_handler.py @@ -0,0 +1,879 @@ +""" +ModelMayhem Handler + +Description: Handler for ModelMayhem.com portfolio/photo galleries +Author: Eric Hiss (GitHub: EricRollei) +Contact: eric@historic.camera, eric@rollei.us +License: Dual License (Non-Commercial and Commercial Use) +Copyright (c) 2025 Eric Hiss. All rights reserved. + +Dual License: +1. Non-Commercial Use: This software is licensed under the terms of the + Creative Commons Attribution-NonCommercial 4.0 International License. + To view a copy of this license, visit http://creativecommons.org/licenses/by-nc/4.0/ + +2. Commercial Use: For commercial use, a separate license is required. + Please contact Eric Hiss at eric@historic.camera or eric@rollei.us for licensing options. + +Dependencies: +This code depends on several third-party libraries, each with its own license. +See CREDITS.md for a comprehensive list of dependencies and their licenses. + +Third-party code: +- See CREDITS.md for complete list of dependencies +""" + +""" +ModelMayhem-specific handler for the Web Image Scraper +Extracts full-resolution images from ModelMayhem portfolios + +URL patterns: +- Profile: https://www.modelmayhem.com/4554791 +- Portfolio (all): https://www.modelmayhem.com/portfolio/4554791/viewall +- Single photo: https://www.modelmayhem.com/portfolio/pic/48745727 +- Photo CDN: https://photos.modelmayhem.com/photos/251111/17/6913e94c8f27f.jpg +""" + +from site_handlers.base_handler import BaseSiteHandler +from urllib.parse import urlparse, urljoin +from typing import List, Dict, Any, Optional +import re +import json +import time +import asyncio +import traceback + +# Playwright (async) ā€“ load only if available +try: + from playwright.async_api import Page as AsyncPage + from playwright.async_api import Browser, BrowserContext + PLAYWRIGHT_AVAILABLE = True +except ImportError: + AsyncPage = None + Browser = None + BrowserContext = None + PLAYWRIGHT_AVAILABLE = False + + +class ModelMayhemHandler(BaseSiteHandler): + """Handler for ModelMayhem.com portfolio sites""" + + # Class attributes for configuration + PRIORITY = 40 # Higher priority for this specific handler + + # Domains associated with ModelMayhem + DOMAINS = [ + "modelmayhem.com", + "www.modelmayhem.com", + "m.modelmayhem.com", + "secure.modelmayhem.com", + "m.secure.modelmayhem.com", + "photos.modelmayhem.com" + ] + + @classmethod + def can_handle(cls, url: str) -> bool: + """Check if this handler can process the URL""" + url_lower = url.lower() + for domain in cls.DOMAINS: + if domain in url_lower: + return True + return False + + def __init__(self, url, scraper=None): + super().__init__(url, scraper) + # Configuration defaults + self.max_scroll_count = 15 + self.scroll_delay_ms = 1500 + self.use_stealth_mode = True + self.retry_attempts = 2 + self.dynamic_content_wait_ms = 2000 + self.request_delay_ms = 1000 + self.last_request_time = 0 + + # Track state + self.is_logged_in = False + self.extracted_media_cache = {} + self.seen_urls = set() + self._username_to_resolve = None # For username-based URLs that need numeric ID resolution + + # Load credentials from auth config + self._load_api_credentials() + + print(f"ModelMayhemHandler initialized for URL: {url}") + + def get_trusted_domains(self) -> List[str]: + """Return list of trusted CDN domains for ModelMayhem""" + return [ + "photos.modelmayhem.com", # Main photo CDN + "assets.modelmayhem.com", # Assets CDN + "modelmayhem.com", # Main domain + "cloudfront.net" # Potential CDN + ] + + def _load_api_credentials(self): + """Load credentials from the auth_config if available""" + self.username = None + self.password = None + + if not hasattr(self, 'scraper') or not self.scraper: + return + + if not hasattr(self.scraper, 'auth_config') or not self.scraper.auth_config: + return + + auth_config = self.scraper.auth_config + + # Try multiple domain variations + domains_to_try = [ + "modelmayhem.com", + "www.modelmayhem.com", + "secure.modelmayhem.com" + ] + + domain_config = None + + # Look in 'sites' section first + if 'sites' in auth_config: + for domain in domains_to_try: + if domain in auth_config['sites']: + domain_config = auth_config['sites'][domain] + print(f"ModelMayhemHandler: Found auth config for {domain}") + break + + # Also try directly in the config (older format) + if not domain_config: + for domain in domains_to_try: + if domain in auth_config: + domain_config = auth_config[domain] + print(f"ModelMayhemHandler: Found auth config (legacy) for {domain}") + break + + if domain_config: + self.username = domain_config.get('username') + self.password = domain_config.get('password') + # Load additional config if available + if 'max_scroll_count' in domain_config: + self.max_scroll_count = domain_config.get('max_scroll_count') + if 'scroll_delay_ms' in domain_config: + self.scroll_delay_ms = domain_config.get('scroll_delay_ms') + if self.username: + print(f"ModelMayhemHandler: Loaded credentials for user {self.username}") + + def prefers_api(self) -> bool: + """This handler doesn't use APIs, it scrapes the web pages""" + return False + + def requires_api(self) -> bool: + """This handler doesn't require API access""" + return False + + async def _perform_login(self, page) -> bool: + """Perform login to ModelMayhem if credentials are available""" + if not self.username or not self.password: + print("ModelMayhemHandler: No credentials available, continuing without login") + return False + + if self.is_logged_in: + return True + + try: + print(f"ModelMayhemHandler: Attempting login as {self.username}") + + # Navigate to login page - use domcontentloaded for faster loading + await page.goto("https://www.modelmayhem.com/login", wait_until="domcontentloaded", timeout=60000) + await asyncio.sleep(3) + + # Check if already logged in by looking for user menu or login form + is_logged_in_already = await page.evaluate(''' + () => { + // Check for signs of being logged in + const userMenu = document.querySelector('.user-menu, .logged-in, [data-user], .account-menu'); + const loginForm = document.querySelector('form[action*="login"], #login-form, .login-form'); + return userMenu !== null || loginForm === null; + } + ''') + + if is_logged_in_already: + print("ModelMayhemHandler: Already logged in") + self.is_logged_in = True + return True + + # Fill in login form + # Try multiple selector patterns for username field + username_selectors = [ + 'input[name="username"]', + 'input[name="email"]', + 'input[type="email"]', + '#username', + '#email', + 'input[placeholder*="mail"]', + 'input[placeholder*="user"]' + ] + + for selector in username_selectors: + try: + username_field = await page.query_selector(selector) + if username_field: + await username_field.fill(self.username) + print(f"ModelMayhemHandler: Filled username field using selector: {selector}") + break + except: + continue + + # Try multiple selector patterns for password field + password_selectors = [ + 'input[name="password"]', + 'input[type="password"]', + '#password' + ] + + for selector in password_selectors: + try: + password_field = await page.query_selector(selector) + if password_field: + await password_field.fill(self.password) + print(f"ModelMayhemHandler: Filled password field using selector: {selector}") + break + except: + continue + + # Submit login form + submit_selectors = [ + 'button[type="submit"]', + 'input[type="submit"]', + 'button:has-text("Log In")', + 'button:has-text("Sign In")', + '.login-button', + '#login-submit' + ] + + for selector in submit_selectors: + try: + submit_button = await page.query_selector(selector) + if submit_button: + await submit_button.click() + print(f"ModelMayhemHandler: Clicked submit using selector: {selector}") + break + except: + continue + + # Wait for navigation/login to complete + await asyncio.sleep(3) + + # Verify login succeeded + current_url = page.url + if "login" not in current_url.lower(): + print("ModelMayhemHandler: Login successful") + self.is_logged_in = True + return True + else: + print("ModelMayhemHandler: Login may have failed, continuing anyway") + return False + + except Exception as e: + print(f"ModelMayhemHandler: Login error: {e}") + traceback.print_exc() + return False + + def _normalize_portfolio_url(self, url: str) -> str: + """Convert any ModelMayhem URL to the portfolio viewall URL""" + # Extract user ID or username from various URL patterns + patterns = [ + # Numeric ID patterns + (r'modelmayhem\.com/portfolio/(\d+)/viewall', r'\1', True), # Already viewall: /portfolio/4554791/viewall + (r'modelmayhem\.com/portfolio/(\d+)', r'\1', True), # Portfolio URL: /portfolio/4554791 + (r'modelmayhem\.com/(\d+)', r'\1', True), # Profile URL with ID: /4554791 + # Username patterns (alphanumeric, may include underscores) + (r'modelmayhem\.com/portfolio/([a-zA-Z][a-zA-Z0-9_]+)/viewall', r'\1', False), # Already viewall with username + (r'modelmayhem\.com/portfolio/([a-zA-Z][a-zA-Z0-9_]+)', r'\1', False), # Portfolio with username + (r'modelmayhem\.com/([a-zA-Z][a-zA-Z0-9_]+)(?:/|$)', r'\1', False), # Profile URL with username: /albertobevacqua + ] + + for pattern, group, is_numeric in patterns: + match = re.search(pattern, url) + if match: + user_id = match.group(1) + # Skip if it matches a reserved path + if user_id.lower() in ['portfolio', 'login', 'signup', 'search', 'browse', 'help', 'about', 'contact', 'terms', 'privacy', 'pic']: + continue + + # For username-based URLs, we need to visit the profile page first to get the numeric ID + # Store the username for later resolution + if not is_numeric: + self._username_to_resolve = user_id + # Return profile URL - we'll resolve to numeric ID in extract_with_direct_playwright + result_url = f"https://www.modelmayhem.com/{user_id}" + print(f"ModelMayhemHandler: Username URL detected, will resolve numeric ID from profile: {result_url}") + return result_url + + result_url = f"https://www.modelmayhem.com/portfolio/{user_id}/viewall" + print(f"ModelMayhemHandler: Normalized URL to {result_url}") + return result_url + + # If it's already a pic URL, just return it + if '/portfolio/pic/' in url: + return url + + # Last resort: return the URL with /viewall appended if it looks like a profile + print(f"ModelMayhemHandler: Could not normalize URL, using as-is: {url}") + return url + + def _upgrade_thumbnail_to_full_res(self, url: str) -> str: + """ + Convert a ModelMayhem thumbnail URL to full resolution. + + ModelMayhem URL patterns: + - Thumbnail: https://photos.modelmayhem.com/photos/251111/17/6913e94c8f27f_m.jpg + - Full res: https://photos.modelmayhem.com/photos/251111/17/6913e94c8f27f.jpg + + The _m suffix indicates medium thumbnail. Remove it for full resolution. + """ + if not url: + return url + + # Remove thumbnail suffixes (_m, _s, _t) before the extension + # Pattern: filename_X.ext -> filename.ext where X is m, s, or t + upgraded = re.sub(r'_[mst]\.([a-zA-Z]+)$', r'.\1', url) + + if upgraded != url: + print(f"ModelMayhemHandler: Upgraded URL: {url} -> {upgraded}") + + return upgraded + + async def _verify_and_get_best_url(self, page, thumbnail_url: str, verbose: bool = False) -> tuple[str, str]: + """ + Verify if the full-res URL exists, fall back to thumbnail if not. + + Returns: + tuple: (best_url, original_thumbnail_or_none) + - best_url: The URL that should be used for download + - original_thumbnail_or_none: The thumbnail URL if different from best_url, else None + """ + full_res_url = self._upgrade_thumbnail_to_full_res(thumbnail_url) + + # If no upgrade happened (URL stayed the same), just use it + if full_res_url == thumbnail_url: + return (thumbnail_url, None) + + # Verify the full-res URL exists + try: + response = await page.context.request.head(full_res_url, headers={ + 'Referer': 'https://www.modelmayhem.com/', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36', + }) + + if response.status < 400: + # Full-res URL exists, use it with thumbnail as fallback + if verbose: + print(f"ModelMayhemHandler: Full-res URL verified: {full_res_url}") + return (full_res_url, thumbnail_url) + else: + # Full-res URL doesn't exist, use thumbnail instead + if verbose: + print(f"ModelMayhemHandler: Full-res URL returned {response.status}, falling back to thumbnail: {thumbnail_url}") + return (thumbnail_url, None) + + except Exception as e: + # On error, use thumbnail as safe fallback + if verbose: + print(f"ModelMayhemHandler: Error verifying full-res URL ({e}), using thumbnail: {thumbnail_url}") + return (thumbnail_url, None) + + async def extract_with_direct_playwright(self, page, **kwargs) -> List[Dict[str, Any]]: + """Main extraction method using Playwright""" + print(f"ModelMayhemHandler: Starting extraction from {self.url}") + start_time = time.time() + media_items = [] + + try: + # First, normalize the URL and check if we need to resolve a username + target_url = self._normalize_portfolio_url(self.url) + + # Attempt login first if we have credentials + await self._perform_login(page) + + # If we have a username to resolve, go to profile page first to get numeric ID + if self._username_to_resolve: + print(f"ModelMayhemHandler: Navigating to profile page to resolve username: {target_url}") + try: + # Use domcontentloaded instead of networkidle for faster loading + await page.goto(target_url, wait_until="domcontentloaded", timeout=60000) + await asyncio.sleep(3) # Give extra time for dynamic content + except Exception as e: + print(f"ModelMayhemHandler: Navigation timeout, retrying with load event: {e}") + await page.goto(target_url, wait_until="load", timeout=60000) + await asyncio.sleep(3) + + numeric_id = await self._resolve_numeric_id_from_profile(page) + if numeric_id: + target_url = f"https://www.modelmayhem.com/portfolio/{numeric_id}/viewall" + print(f"ModelMayhemHandler: Resolved username '{self._username_to_resolve}' to numeric ID {numeric_id}") + else: + print(f"ModelMayhemHandler: Could not resolve numeric ID for username '{self._username_to_resolve}'") + return media_items + + # Navigate to the portfolio page + print(f"ModelMayhemHandler: Navigating to portfolio: {target_url}") + try: + await page.goto(target_url, wait_until="domcontentloaded", timeout=60000) + await asyncio.sleep(3) + except Exception as e: + print(f"ModelMayhemHandler: Navigation timeout, retrying with load event: {e}") + await page.goto(target_url, wait_until="load", timeout=60000) + await asyncio.sleep(3) + + # Disable worksafe mode if it's enabled (critical for seeing actual images) + await self._disable_worksafe_mode(page) + + # Scroll to load all images (lazy loading) + await self._scroll_to_load_all(page) + + # Check if this is a single pic page or portfolio page + if '/portfolio/pic/' in self.url: + # Single photo page - extract that one image + media_items = await self._extract_single_photo(page) + else: + # Portfolio page - extract all photo links then get full-res images + media_items = await self._extract_portfolio_images(page) + + print(f"ModelMayhemHandler: Extracted {len(media_items)} images in {time.time() - start_time:.2f}s") + + except Exception as e: + print(f"ModelMayhemHandler: Error during extraction: {e}") + traceback.print_exc() + + return media_items + + async def _resolve_numeric_id_from_profile(self, page) -> Optional[str]: + """ + Extract the numeric Model Mayhem ID from a profile page. + The ID is shown in "Model Mayhem #: XXXXXX" or in portfolio link URLs. + """ + try: + numeric_id = await page.evaluate(''' + () => { + // Method 1: Look for "Model Mayhem #:" text + const textContent = document.body.innerText; + const mmMatch = textContent.match(/Model Mayhem #:\\s*(\\d+)/); + if (mmMatch) return mmMatch[1]; + + // Method 2: Look for portfolio link with numeric ID + const portfolioLink = document.querySelector('a[href*="/portfolio/"][href*="/viewall"]'); + if (portfolioLink) { + const match = portfolioLink.href.match(/\\/portfolio\\/(\\d+)/); + if (match) return match[1]; + } + + // Method 3: Look for any link with /portfolio/NUMBER pattern + const allLinks = document.querySelectorAll('a[href*="/portfolio/"]'); + for (const link of allLinks) { + const match = link.href.match(/\\/portfolio\\/(\\d+)/); + if (match) return match[1]; + } + + // Method 4: Look for profile link in navigation + const profileLinks = document.querySelectorAll('a[href^="/"]'); + for (const link of profileLinks) { + const match = link.href.match(/modelmayhem\\.com\\/(\\d+)$/); + if (match) return match[1]; + } + + return null; + } + ''') + + if numeric_id: + print(f"ModelMayhemHandler: Found numeric ID: {numeric_id}") + return numeric_id + else: + print("ModelMayhemHandler: Could not find numeric ID on profile page") + return None + + except Exception as e: + print(f"ModelMayhemHandler: Error resolving numeric ID: {e}") + return None + + async def _disable_worksafe_mode(self, page): + """ + Disable worksafe mode to show actual images instead of placeholders. + ModelMayhem defaults to worksafe mode which shows 'nopic_worksafe-on.gif' placeholders. + """ + try: + # Check if worksafe mode is currently ON (link to turn it OFF is visible) + worksafe_off_link = await page.query_selector('a[href="/worksafe/0"], a[href*="worksafe/0"]') + + if worksafe_off_link: + print("ModelMayhemHandler: Worksafe mode is ON, disabling it...") + await worksafe_off_link.click() + await asyncio.sleep(2) # Wait for page to reload/update + print("ModelMayhemHandler: Worksafe mode disabled") + else: + # Check the toggle text to see current state + page_text = await page.evaluate('() => document.body.innerText') + if 'Toggle Worksafe Mode: Off' in page_text: + print("ModelMayhemHandler: Worksafe mode already OFF") + elif 'Toggle Worksafe Mode:' in page_text and 'On' in page_text: + # Try clicking via JavaScript + clicked = await page.evaluate(''' + () => { + const links = document.querySelectorAll('a'); + for (const link of links) { + if (link.textContent.trim() === 'Off' && + link.closest && + link.closest('[class*="worksafe"], [id*="worksafe"]') || + link.previousSibling?.textContent?.includes('Worksafe')) { + link.click(); + return true; + } + } + // Alternative: find by href pattern + const wsLink = document.querySelector('a[href*="worksafe"]'); + if (wsLink && wsLink.textContent.trim() === 'Off') { + wsLink.click(); + return true; + } + return false; + } + ''') + if clicked: + await asyncio.sleep(2) + print("ModelMayhemHandler: Worksafe mode disabled via JS click") + else: + print("ModelMayhemHandler: Could not determine worksafe mode state") + + except Exception as e: + print(f"ModelMayhemHandler: Error disabling worksafe mode: {e}") + + async def _scroll_to_load_all(self, page): + """Scroll down the page to trigger lazy loading of images""" + print("ModelMayhemHandler: Scrolling to load all images...") + + try: + previous_height = 0 + scroll_count = 0 + + while scroll_count < self.max_scroll_count: + # Get current scroll height + current_height = await page.evaluate("document.body.scrollHeight") + + if current_height == previous_height: + # No new content loaded, we're done + break + + previous_height = current_height + + # Scroll down + await page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + await asyncio.sleep(self.scroll_delay_ms / 1000) + + scroll_count += 1 + print(f"ModelMayhemHandler: Scroll {scroll_count}/{self.max_scroll_count}") + + # Scroll back to top + await page.evaluate("window.scrollTo(0, 0)") + await asyncio.sleep(0.5) + + except Exception as e: + print(f"ModelMayhemHandler: Scroll error: {e}") + + async def _extract_single_photo(self, page) -> List[Dict[str, Any]]: + """Extract the full-resolution image from a single photo page""" + media_items = [] + + try: + # Look for the main photo on the page + photo_data = await page.evaluate(''' + () => { + const results = []; + + // Look for images from photos.modelmayhem.com + const imgs = document.querySelectorAll('img'); + for (const img of imgs) { + const src = img.src || img.dataset.src || ''; + if (src.includes('photos.modelmayhem.com')) { + results.push({ + url: src, + width: img.naturalWidth || 0, + height: img.naturalHeight || 0, + alt: img.alt || '' + }); + } + } + + // Also look for og:image meta tag (often has full-res URL) + const ogImage = document.querySelector('meta[property="og:image"]'); + if (ogImage && ogImage.content) { + results.push({ + url: ogImage.content, + width: 0, + height: 0, + alt: 'og:image' + }); + } + + return results; + } + ''') + + for item in photo_data: + url = item.get('url', '') + if not url: + continue + + # Upgrade thumbnail to full resolution + full_res_url = self._upgrade_thumbnail_to_full_res(url) + + if full_res_url not in self.seen_urls: + self.seen_urls.add(full_res_url) + self.seen_urls.add(url) # Also mark original as seen + media_items.append({ + 'url': full_res_url, + 'type': 'image', + 'width': item.get('width', 0), + 'height': item.get('height', 0), + 'source_page': self.url, + 'trusted_cdn': True + }) + + except Exception as e: + print(f"ModelMayhemHandler: Error extracting single photo: {e}") + + return media_items + + async def _extract_portfolio_images(self, page) -> List[Dict[str, Any]]: + """ + Extract all images from a portfolio page. + + Strategy: Extract thumbnail URLs from the portfolio grid page and upgrade them + to full resolution URLs. This is much faster than visiting each individual + pic page, and works because ModelMayhem uses predictable URL patterns: + - Thumbnail: /photos/.../filename_m.jpg + - Full res: /photos/.../filename.jpg + """ + media_items = [] + + try: + # Debug: Log current URL and page title + current_url = page.url + page_title = await page.title() + print(f"ModelMayhemHandler: Current page URL: {current_url}") + print(f"ModelMayhemHandler: Page title: {page_title}") + + # Debug: Count all images on page + all_img_count = await page.evaluate('() => document.querySelectorAll("img").length') + print(f"ModelMayhemHandler: Total img elements on page: {all_img_count}") + + # Extract all images directly from the portfolio page + # These are thumbnails but we'll upgrade them to full resolution + page_images = await page.evaluate(''' + () => { + const images = []; + const debugInfo = { total: 0, modelmayhem: 0, worksafe: 0, other: [] }; + const imgs = document.querySelectorAll('img'); + debugInfo.total = imgs.length; + + for (const img of imgs) { + const src = img.src || img.dataset.src || img.getAttribute('data-src') || ''; + + // Skip worksafe placeholder images + if (src.includes('nopic_worksafe') || src.includes('worksafe')) { + debugInfo.worksafe++; + continue; + } + + // Only include photos.modelmayhem.com images (the actual photos) + if (src.includes('photos.modelmayhem.com/photos/') || + src.includes('photos.modelmayhem.com/covers/')) { + debugInfo.modelmayhem++; + images.push({ + url: src, + width: img.naturalWidth || 0, + height: img.naturalHeight || 0 + }); + } else if (src && src.length > 10 && debugInfo.other.length < 5) { + // Log first few non-matching URLs for debugging + debugInfo.other.push(src.substring(0, 100)); + } + } + + // Also check for background images in divs (some galleries use this) + const divs = document.querySelectorAll('[style*="background-image"]'); + for (const div of divs) { + const style = div.getAttribute('style') || ''; + const match = style.match(/url\\(['"]?(https?:\\/\\/photos\\.modelmayhem\\.com[^'")\s]+)['"]?\\)/); + if (match && match[1]) { + debugInfo.modelmayhem++; + images.push({ + url: match[1], + width: 0, + height: 0 + }); + } + } + + // Check data attributes for lazy-loaded images + const lazyImgs = document.querySelectorAll('[data-src*="photos.modelmayhem.com"], [data-lazy-src*="photos.modelmayhem.com"]'); + for (const img of lazyImgs) { + const src = img.dataset.src || img.dataset.lazySrc || ''; + if (src && src.includes('photos.modelmayhem.com')) { + debugInfo.modelmayhem++; + images.push({ + url: src, + width: 0, + height: 0 + }); + } + } + + // Return both images and debug info + return { images: images, debug: debugInfo }; + } + ''') + + # Extract debug info and images + debug_info = page_images.get('debug', {}) + actual_images = page_images.get('images', []) + + print(f"ModelMayhemHandler: Debug - Total imgs: {debug_info.get('total', 0)}, ModelMayhem imgs: {debug_info.get('modelmayhem', 0)}, Worksafe placeholders: {debug_info.get('worksafe', 0)}") + if debug_info.get('other'): + print(f"ModelMayhemHandler: Sample other image URLs: {debug_info.get('other', [])[:3]}") + + print(f"ModelMayhemHandler: Found {len(actual_images)} ModelMayhem images on portfolio page") + + # Upgrade each thumbnail URL to full resolution (with verification) and deduplicate + print(f"ModelMayhemHandler: Verifying {len(actual_images)} image URLs...") + verified_count = 0 + fallback_count = 0 + + for img in actual_images: + thumb_url = img.get('url', '') + if not thumb_url: + continue + + # Skip if we've already seen this thumbnail URL + if thumb_url in self.seen_urls: + continue + + # Verify and get the best URL (full-res if it exists, else thumbnail) + best_url, original_thumbnail = await self._verify_and_get_best_url(page, thumb_url) + + # Skip if we've already seen the best URL + if best_url in self.seen_urls: + continue + + self.seen_urls.add(best_url) + self.seen_urls.add(thumb_url) # Also mark thumbnail as seen + + # Track verification stats + if original_thumbnail: + verified_count += 1 + else: + fallback_count += 1 + + media_items.append({ + 'url': best_url, + 'type': 'image', + 'width': img.get('width', 0), + 'height': img.get('height', 0), + 'source_page': self.url, + 'trusted_cdn': True, + 'original_thumbnail': original_thumbnail + }) + + print(f"ModelMayhemHandler: Extracted {len(media_items)} unique image URLs ({verified_count} full-res verified, {fallback_count} using thumbnail)") + + # If we didn't find many images, fall back to visiting individual pic pages + # This handles edge cases where the portfolio page doesn't show all thumbnails + if len(media_items) < 10: + print("ModelMayhemHandler: Few images found, trying pic page extraction as fallback...") + fallback_items = await self._extract_via_pic_pages(page) + + # Add any new URLs from fallback + for item in fallback_items: + url = item.get('url', '') + if url and url not in self.seen_urls: + self.seen_urls.add(url) + media_items.append(item) + + print(f"ModelMayhemHandler: After fallback, total {len(media_items)} images") + + except Exception as e: + print(f"ModelMayhemHandler: Error extracting portfolio images: {e}") + traceback.print_exc() + + return media_items + + async def _extract_via_pic_pages(self, page) -> List[Dict[str, Any]]: + """ + Fallback method: Visit individual pic pages to extract full-res images. + Used when the portfolio page doesn't have all thumbnails visible. + """ + media_items = [] + + try: + # Collect all portfolio pic links + pic_links = await page.evaluate(''' + () => { + const links = []; + const anchors = document.querySelectorAll('a[href*="/portfolio/pic/"]'); + for (const a of anchors) { + if (a.href && !links.includes(a.href)) { + links.push(a.href); + } + } + return links; + } + ''') + + print(f"ModelMayhemHandler: Found {len(pic_links)} pic page links for fallback extraction") + + # Visit each pic page (no artificial limit - get all images) + for i, pic_link in enumerate(pic_links): + try: + if self.request_delay_ms > 0: + await asyncio.sleep(self.request_delay_ms / 1000) + + if (i + 1) % 10 == 0: + print(f"ModelMayhemHandler: Processing pic {i+1}/{len(pic_links)}") + + # Navigate to the pic page + await page.goto(pic_link, wait_until="domcontentloaded", timeout=15000) + await asyncio.sleep(0.5) + + # Extract the full-res image from this page + pic_images = await self._extract_single_photo(page) + + for item in pic_images: + url = item.get('url', '') + if url and url not in self.seen_urls: + self.seen_urls.add(url) + media_items.append(item) + + except Exception as e: + print(f"ModelMayhemHandler: Error processing pic {i+1}: {e}") + continue + + except Exception as e: + print(f"ModelMayhemHandler: Error in pic page extraction: {e}") + + return media_items + + async def extract_media_items_async(self, page_adaptor) -> List[Dict[str, Any]]: + """ + Async extraction method - wrapper that gets Playwright page from adaptor + """ + print(f"ModelMayhemHandler: extract_media_items_async called") + + # Try to get the underlying Playwright page + pw_page = None + if hasattr(page_adaptor, 'page'): + pw_page = page_adaptor.page + elif hasattr(self, 'get_playwright_page'): + pw_page = await self.get_playwright_page(page_adaptor) + + if pw_page: + return await self.extract_with_direct_playwright(pw_page) + else: + print("ModelMayhemHandler: No Playwright page available") + return [] diff --git a/site_handlers/portfolio_handler.py b/site_handlers/portfolio_handler.py new file mode 100644 index 0000000..b96d3d3 --- /dev/null +++ b/site_handlers/portfolio_handler.py @@ -0,0 +1,402 @@ +""" +Portfolio Handler + +Description: Handler for model portfolio sites like juliaromanova.com that use clickable image galleries +Author: Eric Hiss (GitHub: EricRollei) +Contact: eric@historic.camera, eric@rollei.us +License: Dual License (Non-Commercial and Commercial Use) +Copyright (c) 2025 Eric Hiss. All rights reserved. + +Dual License: +1. Non-Commercial Use: This software is licensed under the terms of the + Creative Commons Attribution-NonCommercial 4.0 International License. + To view a copy of this license, visit http://creativecommons.org/licenses/by-nc/4.0/ + +2. Commercial Use: For commercial use, a separate license is required. + Please contact Eric Hiss at eric@historic.camera or eric@rollei.us for licensing options. +""" + +""" +Portfolio handler for simple model portfolio sites with clickable image galleries. +Handles sites like juliaromanova.com, artfolio-powered sites, and similar simple galleries. +""" + +from site_handlers.base_handler import BaseSiteHandler +from urllib.parse import urljoin, urlparse +import re +import time +import traceback + +# Try importing Playwright types safely +try: + from playwright.async_api import Page as AsyncPage + PLAYWRIGHT_AVAILABLE = True +except ImportError: + AsyncPage = None + PLAYWRIGHT_AVAILABLE = False + + +class PortfolioHandler(BaseSiteHandler): + """ + Handler for simple portfolio sites with clickable image galleries. + + Targets sites that: + - Have thumbnail galleries where clicking opens larger images + - Use patterns like /galleries/, /portfolio/, /photos/ + - Often use numbered image paths or ID-based URLs + - Include artfolio.com-powered sites + """ + + # Sites and patterns this handler can process + KNOWN_PORTFOLIO_PATTERNS = [ + r'juliaromanova\.com', + r'artfolio\.com', + r'/galleries?/', + r'/portfolio/', + r'/photos?/', + r'/book/', + r'/albums?/', + r'/models?/', + r'/shoots?/', + ] + + # Image URL patterns that indicate resizable images (can be upgraded to larger versions) + RESIZABLE_PATTERNS = [ + (r'g_10_', 'g_30_'), # Artfolio pattern: 10% thumb to 30% large + (r'_thumb\.', '_full.'), + (r'_small\.', '_large.'), + (r'_t\.', '_l.'), + (r'/s/', '/l/'), # some sites use /s/ for small + (r'/small/', '/large/'), + (r'/thumb/', '/full/'), + (r'width=\d+', 'width=2000'), + (r'w=\d+', 'w=2000'), + (r'size=\w+', 'size=original'), + ] + + # Sites that have their own specific handlers - don't handle these + EXCLUDED_DOMAINS = [ + 'modelmayhem.com', + 'instagram.com', + 'flickr.com', + '500px.com', + 'deviantart.com', + 'artstation.com', + 'behance.net', + ] + + @classmethod + def can_handle(cls, url): + """Check if this handler can process the URL""" + url_lower = url.lower() + + # Don't handle sites that have their own specific handlers + for excluded_domain in cls.EXCLUDED_DOMAINS: + if excluded_domain in url_lower: + return False + + # Check for known portfolio patterns + for pattern in cls.KNOWN_PORTFOLIO_PATTERNS: + if re.search(pattern, url_lower, re.IGNORECASE): + print(f"PortfolioHandler can handle: {url} (matched pattern: {pattern})") + return True + + return False + + def __init__(self, url, scraper=None): + """Initialize the portfolio handler""" + super().__init__(url, scraper) + self.debug_mode = True + self.gallery_links = [] + self.seen_urls = set() + self.image_extensions = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp', '.tiff'] + print(f"PortfolioHandler initialized for URL: {url}") + + def prefers_api(self): + """This handler doesn't use APIs""" + return False + + def requires_api(self): + """This handler doesn't require API access""" + return False + + async def extract_media_items_async(self, page): + """ + Extract media items from portfolio gallery pages. + + This handler: + 1. Finds all gallery links (clickable thumbnails) + 2. Extracts direct image URLs from thumbnails + 3. Upgrades thumbnail URLs to full-size versions + """ + print(f"PortfolioHandler: Extracting media from {self.url}") + start_time = time.time() + media_items = [] + + # Get the Playwright page + pw_page = await self._get_playwright_page_async(page) + if not pw_page: + print("No Playwright page available, falling back to DOM extraction") + return await self._extract_from_html(page) + + try: + # Wait for the page to load + await pw_page.wait_for_load_state('networkidle', timeout=15000) + + # First, try to extract all images directly with full resolution + media_items = await self._extract_and_upgrade_images(pw_page) + + # If we found images, return them + if media_items: + print(f"PortfolioHandler: Found {len(media_items)} media items") + return media_items + + # Fallback: Extract from gallery links and navigate to each + gallery_items = await self._extract_gallery_links(pw_page) + if gallery_items: + media_items.extend(gallery_items) + + except Exception as e: + print(f"PortfolioHandler error: {e}") + traceback.print_exc() + + print(f"PortfolioHandler: Extracted {len(media_items)} media items in {time.time() - start_time:.2f}s") + return media_items + + async def _extract_and_upgrade_images(self, pw_page): + """Extract images from the page and upgrade to full resolution""" + media_items = [] + + try: + # Get all images from the page + images = await pw_page.evaluate(''' + () => { + const imgs = document.querySelectorAll('img'); + const result = []; + + imgs.forEach(img => { + const src = img.src || img.dataset.src || img.getAttribute('data-lazy-src') || ''; + if (src && !src.includes('data:image') && !src.includes('blank.gif')) { + // Get the largest version we can find + const srcset = img.srcset || ''; + let largestSrc = src; + let largestWidth = img.naturalWidth || 0; + + // Parse srcset for larger versions + if (srcset) { + const srcsetParts = srcset.split(','); + for (const part of srcsetParts) { + const match = part.trim().match(/^(\S+)\s+(\d+)w$/); + if (match) { + const [, url, width] = match; + if (parseInt(width) > largestWidth) { + largestWidth = parseInt(width); + largestSrc = url; + } + } + } + } + + // Get parent link if exists (for galleries) + let parentLink = null; + const link = img.closest('a'); + if (link && link.href) { + parentLink = link.href; + } + + result.push({ + src: largestSrc, + originalSrc: src, + alt: img.alt || '', + title: img.title || '', + width: img.naturalWidth || 0, + height: img.naturalHeight || 0, + parentLink: parentLink + }); + } + }); + + return result; + } + ''') + + print(f"PortfolioHandler: Found {len(images)} images on page") + + for img_data in images: + src = img_data.get('src', '') + if not src: + continue + + # Skip tracking pixels and tiny images + width = img_data.get('width', 0) + height = img_data.get('height', 0) + if width > 0 and width < 50 and height > 0 and height < 50: + continue + + # Try to upgrade to full resolution + full_res_url = self._upgrade_to_full_resolution(src) + + # Skip if we've already seen this URL + if full_res_url in self.seen_urls: + continue + self.seen_urls.add(full_res_url) + + media_item = { + 'url': full_res_url, + 'original_url': src, + 'type': 'image', + 'alt_text': img_data.get('alt', ''), + 'title': img_data.get('title', ''), + 'source': 'portfolio_handler', + 'parent_link': img_data.get('parentLink', '') + } + media_items.append(media_item) + + except Exception as e: + print(f"Error extracting images: {e}") + traceback.print_exc() + + return media_items + + def _upgrade_to_full_resolution(self, url): + """ + Upgrade a thumbnail URL to full resolution version. + Uses known patterns to transform URLs. + """ + upgraded_url = url + + for pattern, replacement in self.RESIZABLE_PATTERNS: + if re.search(pattern, url): + upgraded_url = re.sub(pattern, replacement, url) + if upgraded_url != url: + print(f"Upgraded URL: {url[:50]}... -> {upgraded_url[:50]}...") + break + + return upgraded_url + + async def _extract_gallery_links(self, pw_page): + """Extract links to gallery pages (for multi-page galleries)""" + media_items = [] + + try: + # Get all links that look like gallery item links + links = await pw_page.evaluate(''' + () => { + const links = document.querySelectorAll('a'); + const galleryLinks = []; + + for (const link of links) { + const href = link.href; + // Look for gallery/portfolio-style links with numbers or slugs + if (href && ( + href.match(/\\/galleries?\\/.*\\/\\d+/) || + href.match(/\\/portfolio\\/.*\\//) || + href.match(/\\/photos?\\/.*\\//) || + href.match(/\\/book\\/[\\w-]+/) || + href.match(/\\/albums?\\/.*\\//) + )) { + // Check if link contains an image (thumbnail) + const img = link.querySelector('img'); + if (img) { + galleryLinks.push({ + href: href, + title: link.title || img.alt || '', + thumbSrc: img.src || '' + }); + } + } + } + + return galleryLinks; + } + ''') + + print(f"PortfolioHandler: Found {len(links)} gallery links") + + # For each gallery link, try to extract the full-size image + for link_data in links: + href = link_data.get('href', '') + thumb_src = link_data.get('thumbSrc', '') + + if href in self.seen_urls: + continue + self.seen_urls.add(href) + + # Try to derive full-res URL from thumbnail + if thumb_src: + full_res_url = self._upgrade_to_full_resolution(thumb_src) + + if full_res_url not in self.seen_urls: + self.seen_urls.add(full_res_url) + media_item = { + 'url': full_res_url, + 'original_url': thumb_src, + 'type': 'image', + 'alt_text': link_data.get('title', ''), + 'source': 'portfolio_handler', + 'gallery_page': href + } + media_items.append(media_item) + + except Exception as e: + print(f"Error extracting gallery links: {e}") + traceback.print_exc() + + return media_items + + async def _get_playwright_page_async(self, page): + """Get the underlying Playwright page object""" + if not PLAYWRIGHT_AVAILABLE: + return None + + # Handle different page wrapper types + if hasattr(page, '_playwright_page'): + return page._playwright_page + elif hasattr(page, 'page'): + return page.page + elif isinstance(page, AsyncPage) if AsyncPage else False: + return page + elif hasattr(page, 'playwright_page'): + return page.playwright_page + + return page + + async def _extract_from_html(self, page): + """Fallback HTML extraction when Playwright isn't available""" + media_items = [] + + try: + # Try to get HTML content + html = None + if hasattr(page, 'html'): + html = page.html + elif hasattr(page, 'content'): + html = await page.content() if callable(page.content) else page.content + + if html: + # Simple regex to find image URLs + img_pattern = r']+src=["\']([^"\']+)["\']' + matches = re.findall(img_pattern, html, re.IGNORECASE) + + for src in matches: + if src and not src.startswith('data:'): + full_url = urljoin(self.url, src) + full_res = self._upgrade_to_full_resolution(full_url) + + if full_res not in self.seen_urls: + self.seen_urls.add(full_res) + media_items.append({ + 'url': full_res, + 'original_url': full_url, + 'type': 'image', + 'source': 'portfolio_handler_html' + }) + + except Exception as e: + print(f"Error in HTML extraction: {e}") + + return media_items + + async def extract_with_direct_playwright(self, page, **kwargs): + """Direct Playwright extraction method called by the scraper""" + return await self.extract_media_items_async(page) diff --git a/site_handlers/reddit_handler.py b/site_handlers/reddit_handler.py index c488bab..c61a972 100644 --- a/site_handlers/reddit_handler.py +++ b/site_handlers/reddit_handler.py @@ -123,14 +123,18 @@ class RedditHandler(BaseSiteHandler): self._extract_identifiers_from_url() # --- Load API credentials --- - # self._load_api_credentials() remove as main code has not loaded confgig yet + # Credentials will be loaded lazily in prefers_api() when the scraper's auth_config is available def prefers_api(self) -> bool: """Reddit handler prefers API if credentials were loaded.""" + # Ensure credentials are loaded from scraper's auth_config if not already done + if not self.api_available and self.scraper: + self._load_api_credentials() + # Check if the necessary attributes exist AND have truthy values has_creds = bool(self.client_id and self.client_secret) - print(f"RedditHandler prefers_api check. Result: {has_creds}") + print(f"RedditHandler prefers_api check. Result: {has_creds} (api_available={self.api_available})") return has_creds async def extract_api_data_async(self, **kwargs) -> list: @@ -316,7 +320,24 @@ class RedditHandler(BaseSiteHandler): # Check if it's a self post (text post) if not submission.is_self and hasattr(submission, 'url'): url = submission.url - if self._is_image_url(url): + + # Check if this is an external video host that needs resolution + if self._is_external_video_host(url): + resolved = await self._resolve_external_video_url(url) + if resolved: + media_items.append({ + 'url': resolved['url'], + 'alt': post_title, + 'title': resolved.get('title') or meta_title, + 'source_url': post_url, + 'credits': credit_line, + 'type': 'video', + 'width': resolved.get('width', 0), + 'height': resolved.get('height', 0), + 'duration': resolved.get('duration'), + '_headers': {'Referer': url} # Use original URL as referer + }) + elif self._is_image_url(url): media_items.append({ 'url': url, 'alt': post_title, @@ -327,15 +348,29 @@ class RedditHandler(BaseSiteHandler): '_headers': {'Referer': post_url} }) elif self._is_video_url(url): - media_items.append({ - 'url': url, - 'alt': post_title, - 'title': meta_title, - 'source_url': post_url, - 'credits': credit_line, - 'type': 'video', - '_headers': {'Referer': post_url} - }) + # For v.redd.it links, they usually need special handling too + if 'v.redd.it' in url: + resolved = await self._resolve_external_video_url(url) + if resolved: + media_items.append({ + 'url': resolved['url'], + 'alt': post_title, + 'title': meta_title, + 'source_url': post_url, + 'credits': credit_line, + 'type': 'video', + '_headers': {'Referer': post_url} + }) + else: + media_items.append({ + 'url': url, + 'alt': post_title, + 'title': meta_title, + 'source_url': post_url, + 'credits': credit_line, + 'type': 'video', + '_headers': {'Referer': post_url} + }) # Handle Reddit-hosted videos if submission.is_video and submission.media and 'reddit_video' in submission.media: @@ -502,6 +537,71 @@ class RedditHandler(BaseSiteHandler): return False + def _is_external_video_host(self, url: str) -> bool: + """Check if URL is from an external video host that needs resolution.""" + external_hosts = ['redgifs.com', 'gfycat.com', 'imgur.com/a/', 'imgur.com/gallery/'] + return any(host in url.lower() for host in external_hosts) + + async def _resolve_external_video_url(self, url: str) -> Optional[Dict[str, Any]]: + """ + Resolve external video host URLs (RedGifs, Gfycat, Imgur) to direct video URLs. + Uses yt-dlp as it has excellent support for these sites. + + Returns: + Dict with 'url' and 'type' keys, or None if resolution fails + """ + import subprocess + import tempfile + + try: + print(f"RedditHandler: Resolving external video URL: {url}") + + # Use yt-dlp to extract video info + result = subprocess.run( + ['yt-dlp', '--dump-json', '--no-download', url], + capture_output=True, + text=True, + timeout=30 + ) + + if result.returncode == 0 and result.stdout: + info = json.loads(result.stdout) + + # Get the best video URL + video_url = info.get('url') + if not video_url: + # Try to get from formats + formats = info.get('formats', []) + if formats: + # Prefer mp4, then webm + for fmt in reversed(formats): + if fmt.get('ext') == 'mp4' and fmt.get('url'): + video_url = fmt['url'] + break + if not video_url: + video_url = formats[-1].get('url') + + if video_url: + print(f"RedditHandler: Resolved to: {video_url[:80]}...") + return { + 'url': video_url, + 'type': 'video', + 'ext': info.get('ext', 'mp4'), + 'title': info.get('title', ''), + 'duration': info.get('duration'), + 'width': info.get('width'), + 'height': info.get('height') + } + else: + print(f"RedditHandler: yt-dlp failed for {url}: {result.stderr[:200] if result.stderr else 'no error'}") + + except subprocess.TimeoutExpired: + print(f"RedditHandler: Timeout resolving {url}") + except Exception as e: + print(f"RedditHandler: Error resolving external video: {e}") + + return None + async def _extract_media_from_dom_async(self, page: AsyncPage, **kwargs) -> list: """Extract media from DOM using async Playwright""" media_items = [] diff --git a/site_handlers/wix_handler.py b/site_handlers/wix_handler.py new file mode 100644 index 0000000..31525a8 --- /dev/null +++ b/site_handlers/wix_handler.py @@ -0,0 +1,695 @@ +""" +Wix Handler + +Description: Handler for Wix-powered websites with image galleries +Author: Eric Hiss (GitHub: EricRollei) +Contact: eric@historic.camera, eric@rollei.us +License: Dual License (Non-Commercial and Commercial Use) +Copyright (c) 2025 Eric Hiss. All rights reserved. + +Dual License: +1. Non-Commercial Use: This software is licensed under the terms of the + Creative Commons Attribution-NonCommercial 4.0 International License. + To view a copy of this license, visit http://creativecommons.org/licenses/by-nc/4.0/ + +2. Commercial Use: For commercial use, a separate license is required. + Please contact Eric Hiss at eric@historic.camera or eric@rollei.us for licensing options. +""" + +""" +Wix handler for websites built on the Wix platform. +Handles image extraction and resolution upgrades for Wix's image CDN. + +Wix image URL patterns: +- static.wixstatic.com/media/[id]/v1/fill/w_[w],h_[h],q_[q],... +- static.wixstatic.com/media/[id]~mv2.jpg (original) +- static.wixstatic.com/media/[id].jpg + +Transform parameters: +- w_[width] - target width +- h_[height] - target height +- q_[quality] - quality (1-100) +- al_c - alignment center +- usm_0.66_1.00_0.01 - unsharp mask +- enc_avif - encoding format (avif, webp, jpg, png) +""" + +from site_handlers.base_handler import BaseSiteHandler +from urllib.parse import urljoin, urlparse, parse_qs, urlencode +import re +import time +import traceback + +# Try importing Playwright types safely +try: + from playwright.async_api import Page as AsyncPage + PLAYWRIGHT_AVAILABLE = True +except ImportError: + AsyncPage = None + PLAYWRIGHT_AVAILABLE = False + + +class WixHandler(BaseSiteHandler): + """ + Handler for Wix-powered websites. + + Wix sites use static.wixstatic.com for image hosting with URL-based + image transformations. This handler extracts images and upgrades + them to maximum resolution. + """ + + # Patterns to identify Wix sites + WIX_PATTERNS = [ + r'wixstatic\.com', + r'wix\.com', + r'wixsite\.com', + r'_wix_', + r'wix-code', + ] + + # Known Wix-powered domains (model agencies, portfolios, etc.) + KNOWN_WIX_DOMAINS = [ + r'new1mgmt\.com', + r'modelwerk\.de', + r'nextmodels\.com', + # Add more known Wix sites as discovered + ] + + @classmethod + def can_handle(cls, url): + """Check if this handler can process the URL""" + url_lower = url.lower() + + # Check for known Wix domains + for pattern in cls.KNOWN_WIX_DOMAINS: + if re.search(pattern, url_lower, re.IGNORECASE): + print(f"WixHandler can handle: {url} (known Wix domain: {pattern})") + return True + + # Check for Wix patterns in URL + for pattern in cls.WIX_PATTERNS: + if re.search(pattern, url_lower, re.IGNORECASE): + print(f"WixHandler can handle: {url} (matched pattern: {pattern})") + return True + + return False + + def __init__(self, url, scraper=None): + """Initialize the Wix handler""" + super().__init__(url, scraper) + self.debug_mode = True + self.seen_urls = set() + self.seen_base_ids = set() # Track base image IDs to avoid duplicates + print(f"WixHandler initialized for URL: {url}") + + def prefers_api(self): + """This handler doesn't use APIs""" + return False + + def requires_api(self): + """This handler doesn't require API access""" + return False + + async def extract_media_items_async(self, page): + """ + Extract media items from Wix-powered pages. + """ + print(f"WixHandler: Extracting media from {self.url}") + start_time = time.time() + media_items = [] + + # Get the Playwright page + pw_page = await self._get_playwright_page_async(page) + if not pw_page: + print("No Playwright page available") + return [] + + try: + # Wait for the page to fully load + await pw_page.wait_for_load_state('networkidle', timeout=20000) + + # Give extra time for Wix's lazy loading + await pw_page.wait_for_timeout(2000) + + # Scroll to trigger lazy loading + await self._scroll_page(pw_page) + + # Extract images from multiple sources + media_items = await self._extract_wix_images(pw_page) + + # Extract videos from the page + video_items = await self._extract_wix_videos(pw_page) + media_items.extend(video_items) + + # Also try to get images from network requests + network_images = await self._extract_from_page_resources(pw_page) + + # Merge and deduplicate + for item in network_images: + base_id = self._extract_wix_image_id(item.get('url', '')) + if base_id and base_id not in self.seen_base_ids: + self.seen_base_ids.add(base_id) + media_items.append(item) + + except Exception as e: + print(f"WixHandler error: {e}") + traceback.print_exc() + + print(f"WixHandler: Extracted {len(media_items)} media items in {time.time() - start_time:.2f}s") + return media_items + + async def _scroll_page(self, pw_page): + """Scroll the page to trigger lazy loading""" + try: + # Get page height + scroll_height = await pw_page.evaluate('document.body.scrollHeight') + viewport_height = await pw_page.evaluate('window.innerHeight') + + # Scroll in increments + current_position = 0 + scroll_step = viewport_height * 0.8 + + while current_position < scroll_height: + current_position += scroll_step + await pw_page.evaluate(f'window.scrollTo(0, {current_position})') + await pw_page.wait_for_timeout(500) + + # Check if height increased (more content loaded) + new_height = await pw_page.evaluate('document.body.scrollHeight') + if new_height > scroll_height: + scroll_height = new_height + + # Scroll back to top + await pw_page.evaluate('window.scrollTo(0, 0)') + await pw_page.wait_for_timeout(500) + + except Exception as e: + print(f"Error scrolling page: {e}") + + async def _extract_wix_videos(self, pw_page): + """ + Extract videos from Wix-powered pages. + + Wix videos are hosted at video.wixstatic.com with this pattern: + - https://video.wixstatic.com/video/{video_id}/1080p/mp4/file.mp4 + + Video IDs can be found from: + 1. Direct video src attributes + 2. Gallery items marked as video (poster images with 'f003' suffix) + 3. Data attributes in gallery containers + """ + media_items = [] + seen_video_ids = set() + + try: + # Extract video data from the page + video_data = await pw_page.evaluate(''' + () => { + const results = { + directVideos: [], + galleryVideos: [], + videoContainers: [] + }; + + // 1. Get ALL video elements - check multiple src attributes + const videos = document.querySelectorAll('video'); + for (const video of videos) { + // Check multiple possible source attributes + let src = video.src || video.currentSrc || ''; + const poster = video.poster || ''; + + // Also check source child elements + if (!src) { + const sourceEl = video.querySelector('source'); + if (sourceEl) src = sourceEl.src || ''; + } + + // Any wixstatic video URL (including video.wixstatic.com) + if (src && (src.includes('wixstatic.com') || src.includes('wix.com'))) { + results.directVideos.push({ + src: src, + poster: poster, + type: 'direct', + className: video.className + }); + } + // Poster images can help identify video IDs + if (poster && poster.includes('wixstatic')) { + results.directVideos.push({ + poster: poster, + type: 'poster_reference' + }); + } + } + + // 2. Get gallery items marked as videos + const galleryVideoItems = document.querySelectorAll('[class*="gallery-item-video"]'); + for (const item of galleryVideoItems) { + const img = item.querySelector('img'); + const video = item.querySelector('video'); + + // Get poster/thumbnail image which contains the video ID + if (img && img.src && img.src.includes('wixstatic')) { + // Video poster images typically end with f003 or f000 + const src = img.src; + results.galleryVideos.push({ + posterSrc: src, + hasVideoElement: !!video, + type: 'gallery_video' + }); + } + } + + // 3. Look for video containers with data attributes + const videoContainers = document.querySelectorAll('[class*="video"]'); + for (const container of videoContainers) { + const videoEl = container.querySelector('video[src*="video.wixstatic"]'); + if (videoEl) { + results.videoContainers.push({ + src: videoEl.src, + type: 'container' + }); + } + } + + return results; + } + ''') + + print(f"WixHandler: Found {len(video_data.get('directVideos', []))} direct videos, " + f"{len(video_data.get('galleryVideos', []))} gallery videos, " + f"{len(video_data.get('videoContainers', []))} container videos") + + # Process direct videos (already have full URLs) + for vid in video_data.get('directVideos', []): + src = vid.get('src', '') + print(f"WixHandler: Processing direct video - src: {src[:100] if src else 'None'}...") + + # Check for any wixstatic video URL + if src and ('video.wixstatic.com' in src or 'wixstatic.com' in src): + video_id = self._extract_wix_video_id(src) + print(f"WixHandler: Extracted video ID: {video_id}") + + if video_id and video_id not in seen_video_ids: + seen_video_ids.add(video_id) + # Upgrade to 1080p if not already + full_url = self._upgrade_wix_video_url(src) + print(f"WixHandler: Upgraded video URL to: {full_url}") + media_items.append({ + 'url': full_url, + 'original_url': src, + 'type': 'video', + 'poster': vid.get('poster', ''), + 'source': 'wix_handler_video', + 'page_url': self.url, # For Referer header + 'qualities': { + '1080p': f"https://video.wixstatic.com/video/{video_id}/1080p/mp4/file.mp4", + '720p': f"https://video.wixstatic.com/video/{video_id}/720p/mp4/file.mp4", + '480p': f"https://video.wixstatic.com/video/{video_id}/480p/mp4/file.mp4" + } + }) + + # Process gallery videos (construct URLs from poster images) + for vid in video_data.get('galleryVideos', []): + poster_src = vid.get('posterSrc', '') + if poster_src: + video_id = self._extract_video_id_from_poster(poster_src) + if video_id and video_id not in seen_video_ids: + seen_video_ids.add(video_id) + # Construct direct MP4 URL + video_url = f"https://video.wixstatic.com/video/{video_id}/1080p/mp4/file.mp4" + media_items.append({ + 'url': video_url, + 'type': 'video', + 'poster': poster_src, + 'source': 'wix_handler_gallery_video', + 'page_url': self.url, # For Referer header + 'qualities': { + '1080p': f"https://video.wixstatic.com/video/{video_id}/1080p/mp4/file.mp4", + '720p': f"https://video.wixstatic.com/video/{video_id}/720p/mp4/file.mp4", + '480p': f"https://video.wixstatic.com/video/{video_id}/480p/mp4/file.mp4" + } + }) + print(f"WixHandler: Constructed video URL for ID {video_id}") + + # Process video containers + for vid in video_data.get('videoContainers', []): + src = vid.get('src', '') + if src and 'video.wixstatic.com' in src: + video_id = self._extract_wix_video_id(src) + if video_id and video_id not in seen_video_ids: + seen_video_ids.add(video_id) + full_url = self._upgrade_wix_video_url(src) + media_items.append({ + 'url': full_url, + 'original_url': src, + 'type': 'video', + 'source': 'wix_handler_container_video' + }) + + except Exception as e: + print(f"Error extracting Wix videos: {e}") + traceback.print_exc() + + print(f"WixHandler: Extracted {len(media_items)} videos") + return media_items + + def _extract_wix_video_id(self, url): + """Extract video ID from a Wix video URL""" + if not url: + return None + + # Pattern: video.wixstatic.com/video/{video_id}/quality/mp4/file.mp4 + # Example: video.wixstatic.com/video/b52b2f_19e70a48dd054abfb015bb2598c1ff8d/1080p/mp4/file.mp4 + match = re.search(r'video\.wixstatic\.com/video/([a-f0-9]+_[a-f0-9]+)', url) + if match: + return match.group(1) + return None + + def _extract_video_id_from_poster(self, poster_url): + """ + Extract video ID from a poster image URL. + + Poster images have patterns like: + - b52b2f_0a5d15df38dc4c08a18238b03b87082af003.jpg (f003 suffix = poster frame) + - The video ID is the part before 'f003' or 'f000' + """ + if not poster_url: + return None + + # Pattern: media/{id}f003.jpg or media/{id}f000.jpg + # We need to strip the f00X suffix to get the video ID + patterns = [ + # Match ID with frame suffix (f003, f000, etc.) + r'/media/([a-f0-9]+_[a-f0-9]+?)f\d{3}(?:~mv2)?\.(?:jpg|jpeg|png|webp)', + r'/media/([a-f0-9]+_[a-f0-9]+?)f\d{3}\.(?:jpg|jpeg|png|webp)', + # Direct ID pattern + r'/media/([a-f0-9]+_[a-f0-9]{32})', + ] + + for pattern in patterns: + match = re.search(pattern, poster_url, re.IGNORECASE) + if match: + video_id = match.group(1) + # Clean up any trailing frame indicator + video_id = re.sub(r'f\d{3}$', '', video_id) + return video_id + + return None + + def _upgrade_wix_video_url(self, url): + """Upgrade video URL to highest quality (1080p)""" + if not url: + return url + + # Replace quality specifier with 1080p + upgraded = re.sub(r'/\d{3,4}p/', '/1080p/', url) + return upgraded + + async def _extract_wix_images(self, pw_page): + """Extract and upgrade Wix images from the page""" + media_items = [] + + try: + # Get all images with their sources + images = await pw_page.evaluate(''' + () => { + const results = []; + + // Get all img elements + const imgs = document.querySelectorAll('img'); + for (const img of imgs) { + const src = img.src || img.dataset.src || img.getAttribute('data-src') || ''; + const srcset = img.srcset || ''; + + if (src) { + results.push({ + src: src, + srcset: srcset, + alt: img.alt || '', + width: img.naturalWidth || img.width || 0, + height: img.naturalHeight || img.height || 0 + }); + } + } + + // Also check for background images in styles + const allElements = document.querySelectorAll('*'); + for (const el of allElements) { + const style = window.getComputedStyle(el); + const bgImage = style.backgroundImage; + if (bgImage && bgImage !== 'none' && bgImage.includes('wixstatic')) { + const match = bgImage.match(/url\\(["']?([^"']+)["']?\\)/); + if (match) { + results.push({ + src: match[1], + srcset: '', + alt: '', + width: 0, + height: 0, + isBackground: true + }); + } + } + } + + // Check for Wix gallery data + const galleryData = document.querySelectorAll('[data-image-info]'); + for (const el of galleryData) { + try { + const info = JSON.parse(el.getAttribute('data-image-info')); + if (info && info.imageData && info.imageData.uri) { + results.push({ + src: 'https://static.wixstatic.com/media/' + info.imageData.uri, + srcset: '', + alt: info.imageData.title || '', + width: info.imageData.width || 0, + height: info.imageData.height || 0, + fromGalleryData: true + }); + } + } catch (e) {} + } + + return results; + } + ''') + + print(f"WixHandler: Found {len(images)} image elements") + + for img_data in images: + src = img_data.get('src', '') + srcset = img_data.get('srcset', '') + + # Skip non-wix images and tracking pixels + if not src: + continue + if 'wixstatic' not in src and 'wix.com' not in src: + continue + if img_data.get('width', 0) > 0 and img_data.get('width', 0) < 50: + continue + + # Extract base image ID + base_id = self._extract_wix_image_id(src) + if not base_id: + continue + + # Skip if we've already processed this image + if base_id in self.seen_base_ids: + continue + self.seen_base_ids.add(base_id) + + # Try to get highest resolution from srcset first + best_url = self._get_best_from_srcset(srcset) if srcset else None + + # Upgrade to maximum resolution + if best_url: + full_res_url = self._upgrade_wix_url(best_url) + else: + full_res_url = self._upgrade_wix_url(src) + + if full_res_url in self.seen_urls: + continue + self.seen_urls.add(full_res_url) + + media_item = { + 'url': full_res_url, + 'original_url': src, + 'type': 'image', + 'alt_text': img_data.get('alt', ''), + 'source': 'wix_handler', + 'original_width': img_data.get('width', 0), + 'original_height': img_data.get('height', 0) + } + media_items.append(media_item) + + except Exception as e: + print(f"Error extracting Wix images: {e}") + traceback.print_exc() + + return media_items + + def _extract_wix_image_id(self, url): + """Extract the base image ID from a Wix URL""" + if not url: + return None + + # Pattern: /media/[id]_[hash]~mv2.jpg or /media/[id].jpg + # Example: b52b2f_1f9ee9ba5e6c4b4792257297be983580~mv2.jpg + patterns = [ + r'/media/([a-f0-9]+_[a-f0-9]+[^/]*?)(?:/v1/|\.(?:jpg|jpeg|png|webp|gif))', + r'/media/([a-f0-9]+_[^/]+?)(?:~mv2)?\.(?:jpg|jpeg|png|webp|gif)', + r'/media/([a-f0-9_]+)(?:/|~|\.)', + ] + + for pattern in patterns: + match = re.search(pattern, url, re.IGNORECASE) + if match: + return match.group(1) + + return None + + def _get_best_from_srcset(self, srcset): + """Parse srcset and return the highest resolution URL""" + if not srcset: + return None + + best_url = None + best_width = 0 + + # Parse srcset: "url1 800w, url2 1200w, ..." + parts = srcset.split(',') + for part in parts: + part = part.strip() + match = re.match(r'(\S+)\s+(\d+)w', part) + if match: + url, width = match.groups() + width = int(width) + if width > best_width: + best_width = width + best_url = url + + return best_url + + def _upgrade_wix_url(self, url): + """ + Upgrade a Wix image URL to maximum resolution. + + Strategies: + 1. Request original by removing transforms + 2. Request very large dimensions + 3. Change quality to maximum + """ + if not url: + return url + + # If it's already an original URL (no /v1/fill/ or /v1/fit/), return as-is + if '/v1/fill/' not in url and '/v1/crop/' not in url and '/v1/fit/' not in url: + # Try to ensure we get the raw image + return url + + try: + # Extract the base media URL and transform it + # Pattern: https://static.wixstatic.com/media/[id]/v1/fill/w_X,h_Y,.../filename.ext + # Or: https://static.wixstatic.com/media/[id]/v1/fit/w_X,h_Y,.../filename.ext + + # Strategy 1: Request maximum resolution (4K+) + # Replace dimension parameters with large values + upgraded = url + + # Replace width with large value (Wix typically supports up to 5000) + upgraded = re.sub(r'w_\d+', 'w_4000', upgraded) + + # Replace height with large value + upgraded = re.sub(r'h_\d+', 'h_5000', upgraded) + + # Set quality to maximum (90 is usually best balance, 100 can cause issues) + upgraded = re.sub(r'q_\d+', 'q_95', upgraded) + + # Change fit to fill for better quality (fit may letterbox) + # Actually keep as-is since fit respects aspect ratio + + # Prefer JPEG over AVIF/WebP for better compatibility and quality + upgraded = re.sub(r'enc_avif', 'enc_auto', upgraded) + upgraded = re.sub(r'enc_webp', 'enc_auto', upgraded) + + # Remove quality_auto which adds compression + upgraded = re.sub(r',quality_auto', '', upgraded) + + print(f"WixHandler: Upgraded URL dimensions to 4000x5000") + + return upgraded + + except Exception as e: + print(f"Error upgrading Wix URL: {e}") + return url + + async def _extract_from_page_resources(self, pw_page): + """Extract images from page resources/network requests""" + media_items = [] + + try: + # Get all loaded resources + resources = await pw_page.evaluate(''' + () => { + const resources = []; + + // Check performance entries for loaded resources + if (window.performance && window.performance.getEntriesByType) { + const entries = window.performance.getEntriesByType('resource'); + for (const entry of entries) { + if (entry.initiatorType === 'img' || + entry.name.match(/\\.(jpg|jpeg|png|webp|gif|avif)/i)) { + resources.push(entry.name); + } + } + } + + return resources; + } + ''') + + for url in resources: + if 'wixstatic' not in url: + continue + + base_id = self._extract_wix_image_id(url) + if not base_id or base_id in self.seen_base_ids: + continue + + full_res_url = self._upgrade_wix_url(url) + + if full_res_url not in self.seen_urls: + self.seen_urls.add(full_res_url) + media_items.append({ + 'url': full_res_url, + 'original_url': url, + 'type': 'image', + 'source': 'wix_handler_network' + }) + + except Exception as e: + print(f"Error extracting from page resources: {e}") + + return media_items + + async def _get_playwright_page_async(self, page): + """Get the underlying Playwright page object""" + if not PLAYWRIGHT_AVAILABLE: + return None + + # Handle different page wrapper types + if hasattr(page, '_playwright_page'): + return page._playwright_page + elif hasattr(page, 'page'): + return page.page + elif isinstance(page, AsyncPage) if AsyncPage else False: + return page + elif hasattr(page, 'playwright_page'): + return page.playwright_page + + return page + + async def extract_with_direct_playwright(self, page, **kwargs): + """Direct Playwright extraction method called by the scraper""" + return await self.extract_media_items_async(page) diff --git a/utils/__init__.py b/utils/__init__.py new file mode 100644 index 0000000..a90f885 --- /dev/null +++ b/utils/__init__.py @@ -0,0 +1,19 @@ +""" +Utils Package + +Utility modules for download_tools nodes. +""" + +from .persistent_settings import ( + PersistentSettings, + get_settings_manager, + get_persistent_setting, + set_persistent_setting +) + +__all__ = [ + 'PersistentSettings', + 'get_settings_manager', + 'get_persistent_setting', + 'set_persistent_setting' +] diff --git a/utils/persistent_settings.py b/utils/persistent_settings.py new file mode 100644 index 0000000..af08dfe --- /dev/null +++ b/utils/persistent_settings.py @@ -0,0 +1,167 @@ +""" +Persistent Settings Manager + +Description: Manages persistent settings for download_tools nodes across reloads +Author: Eric Hiss (GitHub: EricRollei) +Contact: eric@historic.camera, eric@rollei.us +License: Dual License (Non-Commercial and Commercial Use) +Copyright (c) 2025 Eric Hiss. All rights reserved. +""" + +import os +import json +from pathlib import Path +from typing import Optional, Any, Dict + + +class PersistentSettings: + """ + Manages persistent settings for download_tools nodes. + Settings are stored in a JSON file and persist across node/workflow reloads. + """ + + _instance = None + _settings: Dict[str, Any] = {} + _settings_file: Path = None + + def __new__(cls): + """Singleton pattern to ensure only one instance exists.""" + if cls._instance is None: + cls._instance = super().__new__(cls) + cls._instance._initialized = False + return cls._instance + + def __init__(self): + """Initialize the settings manager.""" + if self._initialized: + return + + # Determine settings file path + self._settings_file = Path(__file__).parent.parent / "configs" / "node_settings.json" + self._settings = self._load_settings() + self._initialized = True + print(f"PersistentSettings initialized from: {self._settings_file}") + + def _load_settings(self) -> Dict[str, Any]: + """Load settings from the JSON file.""" + default_settings = { + "web_scraper": { + "auth_config_path": "" + }, + "gallery_dl": { + "config_path": "", + "cookie_file": "" + }, + "yt_dlp": { + "config_path": "" + } + } + + try: + if self._settings_file.exists(): + with open(self._settings_file, 'r', encoding='utf-8') as f: + loaded = json.load(f) + # Merge with defaults to ensure all keys exist + for key in default_settings: + if key not in loaded: + loaded[key] = default_settings[key] + elif isinstance(default_settings[key], dict): + for subkey in default_settings[key]: + if subkey not in loaded[key]: + loaded[key][subkey] = default_settings[key][subkey] + return loaded + else: + # Create the file with defaults + self._settings_file.parent.mkdir(parents=True, exist_ok=True) + with open(self._settings_file, 'w', encoding='utf-8') as f: + json.dump(default_settings, f, indent=4) + return default_settings + except Exception as e: + print(f"Error loading persistent settings: {e}") + return default_settings + + def _save_settings(self) -> bool: + """Save current settings to the JSON file.""" + try: + self._settings_file.parent.mkdir(parents=True, exist_ok=True) + with open(self._settings_file, 'w', encoding='utf-8') as f: + json.dump(self._settings, f, indent=4) + return True + except Exception as e: + print(f"Error saving persistent settings: {e}") + return False + + def get(self, node_type: str, key: str, default: Any = "") -> Any: + """ + Get a setting value for a specific node type. + + Args: + node_type: The type of node ('web_scraper', 'gallery_dl', 'yt_dlp') + key: The setting key to retrieve + default: Default value if setting not found + + Returns: + The setting value or default + """ + try: + if node_type in self._settings and key in self._settings[node_type]: + value = self._settings[node_type][key] + return value if value else default + return default + except Exception: + return default + + def set(self, node_type: str, key: str, value: Any) -> bool: + """ + Set a setting value for a specific node type. + + Args: + node_type: The type of node ('web_scraper', 'gallery_dl', 'yt_dlp') + key: The setting key to set + value: The value to store + + Returns: + True if successful, False otherwise + """ + try: + if node_type not in self._settings: + self._settings[node_type] = {} + + # Only update if value is non-empty (don't overwrite with empty values) + if value and str(value).strip(): + self._settings[node_type][key] = str(value).strip() + return self._save_settings() + return True + except Exception as e: + print(f"Error setting persistent setting: {e}") + return False + + def get_all(self, node_type: str) -> Dict[str, Any]: + """Get all settings for a specific node type.""" + return self._settings.get(node_type, {}) + + def reload(self) -> None: + """Reload settings from file (useful if file was edited externally).""" + self._settings = self._load_settings() + + +# Global instance for easy access +_settings_manager: Optional[PersistentSettings] = None + + +def get_settings_manager() -> PersistentSettings: + """Get the global settings manager instance.""" + global _settings_manager + if _settings_manager is None: + _settings_manager = PersistentSettings() + return _settings_manager + + +def get_persistent_setting(node_type: str, key: str, default: Any = "") -> Any: + """Convenience function to get a persistent setting.""" + return get_settings_manager().get(node_type, key, default) + + +def set_persistent_setting(node_type: str, key: str, value: Any) -> bool: + """Convenience function to set a persistent setting.""" + return get_settings_manager().set(node_type, key, value)