From e4c3b0fb6f30086cba7418bc871ea605a1283b37 Mon Sep 17 00:00:00 2001 From: Dariusz L Date: Thu, 25 Jun 2026 02:09:47 +0200 Subject: [PATCH] refactor: consolidate type and progress utilities Add core/type_utils.py with as_dict/as_list helpers, and core/progress.py with a unified report_progress() callback. Replace inline duplicated implementations of _json_safe_metadata(), _as_list(), normalize_filename(), and per-source _report_progress() with calls to the shared modules across downloader, matcher, resolver, and all source modules (CivitAI, CivArchive, HuggingFace, LoRA Manager). Extend filename normalization to include dots as separators. --- core/downloader.py | 11 ++----- core/matcher.py | 4 +-- core/progress.py | 42 ++++++++++++++++++++++++++ core/resolver.py | 9 ++---- core/sources/civarchive.py | 22 ++++++-------- core/sources/civitai.py | 22 ++++++-------- core/sources/huggingface.py | 22 ++++++-------- core/sources/lora_manager_archive.py | 22 ++++++-------- core/sources/model_list.py | 11 +++---- core/type_utils.py | 44 ++++++++++++++++++++++++++++ 10 files changed, 134 insertions(+), 75 deletions(-) create mode 100644 core/progress.py create mode 100644 core/type_utils.py diff --git a/core/downloader.py b/core/downloader.py index 16986fa..3831313 100644 --- a/core/downloader.py +++ b/core/downloader.py @@ -24,6 +24,7 @@ from .log_system.log_funcs import ( ) from .resolver import normalize_sha256 from .path_utils import is_path_within, get_path_identity +from .type_utils import as_dict, as_list try: import folder_paths @@ -180,17 +181,11 @@ def _json_safe_metadata(value: Any, depth: int = 0) -> Any: def _as_dict(value: Any) -> Dict[str, Any]: - return value if isinstance(value, dict) else {} + return as_dict(value) def _as_list(value: Any) -> List[Any]: - if isinstance(value, list): - return [item for item in value if item not in (None, "")] - if isinstance(value, (tuple, set)): - return [item for item in value if item not in (None, "")] - if isinstance(value, str): - return [item.strip() for item in value.split(",") if item.strip()] - return [] + return as_list(value) def _first_present(*values: Any) -> Any: diff --git a/core/matcher.py b/core/matcher.py index bafb9c4..0bd6e8a 100644 --- a/core/matcher.py +++ b/core/matcher.py @@ -37,8 +37,8 @@ def normalize_filename(filename: str) -> str: # Convert to lowercase base = base.lower() - # Normalize separators: replace underscores, hyphens, and spaces with a single space - base = re.sub(r"[_\-\s]+", " ", base) + # Normalize separators: replace underscores, hyphens, dots, and spaces with a single space + base = re.sub(r"[_\-\.\s]+", " ", base) # Strip whitespace base = base.strip() diff --git a/core/progress.py b/core/progress.py new file mode 100644 index 0000000..975b2a6 --- /dev/null +++ b/core/progress.py @@ -0,0 +1,42 @@ +""" +Progress Reporting Module + +Unified utility for reporting progress of asynchronous search and download operations. +""" + +from typing import Dict, Any, Optional, Callable +from .log_system.log_funcs import log_debug + + +def report_progress( + progress_callback: Optional[Callable[[Dict[str, Any]], None]], + stage: str, + message: str, + percent: Optional[float] = None, + error_context: str = "Progress callback", + **extra: Any, +) -> None: + """ + Report progress via callback payload. + + Args: + progress_callback: Callback function to execute + stage: Progress stage name + message: Informational message + percent: Progress percentage (0.0 to 100.0) + error_context: Source name to use in error logging + extra: Additional key-value pairs to add to payload + """ + if not progress_callback: + return + + payload = {"stage": stage, "message": message} + if percent is not None: + payload["percent"] = percent + if extra: + payload.update(extra) + + try: + progress_callback(payload) + except Exception as e: + log_debug(f"{error_context} failed: {e}") diff --git a/core/resolver.py b/core/resolver.py index f4609b6..fd63156 100644 --- a/core/resolver.py +++ b/core/resolver.py @@ -27,6 +27,7 @@ from .workflow_analyzer import ( should_scan_as_model_reference, ) from .matcher import find_matches +from .type_utils import as_dict, as_list from .workflow_updater import update_workflow_nodes from .sources.civitai import resolve_urn from .sources.huggingface import parse_huggingface_url as parse_hf_url @@ -327,15 +328,11 @@ def _metadata_sidecar_candidates(model_path: str) -> List[str]: def _as_dict(value: Any) -> Dict[str, Any]: - return value if isinstance(value, dict) else {} + return as_dict(value) def _as_list(value: Any) -> List[Any]: - if isinstance(value, list): - return value - if isinstance(value, tuple): - return list(value) - return [] + return as_list(value) def _collect_hashes_from_container(value: Any) -> List[str]: diff --git a/core/sources/civarchive.py b/core/sources/civarchive.py index cdfecd3..ac0f2f6 100644 --- a/core/sources/civarchive.py +++ b/core/sources/civarchive.py @@ -21,6 +21,7 @@ from ..matcher import ( base_model_matches as _base_model_matches, base_model_score as _base_model_score, ) +from ..progress import report_progress from ..log_system.log_funcs import ( log_debug, log_info, @@ -98,19 +99,14 @@ def _report_progress( percent: Optional[float] = None, **extra: Any, ) -> None: - if not progress_callback: - return - - payload = {"stage": stage, "message": message} - if percent is not None: - payload["percent"] = percent - if extra: - payload.update(extra) - - try: - progress_callback(payload) - except Exception as e: - log_debug(f"CivArchive progress callback failed: {e}") + report_progress( + progress_callback, + stage, + message, + percent, + error_context="CivArchive progress callback", + **extra, + ) def is_civarchive_available() -> bool: diff --git a/core/sources/civitai.py b/core/sources/civitai.py index 2541aad..319fd97 100644 --- a/core/sources/civitai.py +++ b/core/sources/civitai.py @@ -19,6 +19,7 @@ from ..matcher import ( base_model_matches as _base_model_matches, base_model_score as _base_model_score, ) +from ..progress import report_progress from ..log_system.log_funcs import ( log_debug, log_info, @@ -79,19 +80,14 @@ def _report_progress( percent: Optional[float] = None, **extra: Any, ) -> None: - if not progress_callback: - return - - payload = {"stage": stage, "message": message} - if percent is not None: - payload["percent"] = percent - if extra: - payload.update(extra) - - try: - progress_callback(payload) - except Exception as e: - log_debug(f"CivitAI progress callback failed: {e}") + report_progress( + progress_callback, + stage, + message, + percent, + error_context="CivitAI progress callback", + **extra, + ) def check_civitai_session_token(session_token: Optional[str]) -> Dict[str, Any]: diff --git a/core/sources/huggingface.py b/core/sources/huggingface.py index 20daaae..716e2f1 100644 --- a/core/sources/huggingface.py +++ b/core/sources/huggingface.py @@ -13,6 +13,7 @@ import requests from typing import Dict, Any, Optional, List, Callable from urllib.parse import urlparse, quote +from ..progress import report_progress from ..log_system.log_funcs import ( log_debug, log_info, @@ -54,19 +55,14 @@ def _report_progress( percent: Optional[float] = None, **extra: Any, ) -> None: - if not progress_callback: - return - - payload = {"stage": stage, "message": message} - if percent is not None: - payload["percent"] = percent - if extra: - payload.update(extra) - - try: - progress_callback(payload) - except Exception as e: - log_debug(f"HuggingFace progress callback failed: {e}") + report_progress( + progress_callback, + stage, + message, + percent, + error_context="HuggingFace progress callback", + **extra, + ) def check_huggingface_token(token: Optional[str]) -> Dict[str, Any]: diff --git a/core/sources/lora_manager_archive.py b/core/sources/lora_manager_archive.py index 47cabd4..5900e81 100644 --- a/core/sources/lora_manager_archive.py +++ b/core/sources/lora_manager_archive.py @@ -18,6 +18,7 @@ from ..matcher import ( base_model_matches as _base_model_matches, base_model_score as _base_model_score, ) +from ..progress import report_progress from ..log_system.log_funcs import ( log_debug, log_info, @@ -63,19 +64,14 @@ def _report_progress( percent: Optional[float] = None, **extra: Any, ) -> None: - if not progress_callback: - return - - payload = {"stage": stage, "message": message} - if percent is not None: - payload["percent"] = percent - if extra: - payload.update(extra) - - try: - progress_callback(payload) - except Exception as e: - log_debug(f"LoRA Manager archive progress callback failed: {e}") + report_progress( + progress_callback, + stage, + message, + percent, + error_context="LoRA Manager archive progress callback", + **extra, + ) def get_lora_manager_archive_db_path() -> Optional[str]: diff --git a/core/sources/model_list.py b/core/sources/model_list.py index 863c65d..45c098f 100644 --- a/core/sources/model_list.py +++ b/core/sources/model_list.py @@ -11,9 +11,10 @@ import shutil import tempfile from datetime import datetime, timezone from typing import Dict, Any, Optional, List -from difflib import SequenceMatcher from urllib.request import Request, urlopen +from ..matcher import normalize_filename, calculate_similarity + from ..log_system.log_funcs import ( log_debug, log_info, @@ -219,16 +220,12 @@ def update_model_list_from_remote() -> Dict[str, Any]: def _normalize_filename(filename: str) -> str: """Normalize filename for comparison.""" - # Remove extension - base = os.path.splitext(filename)[0].lower() - # Replace separators with spaces - base = base.replace("-", " ").replace("_", " ").replace(".", " ") - return base + return normalize_filename(filename) def _similarity(a: str, b: str) -> float: """Calculate similarity between two strings.""" - return SequenceMatcher(None, a, b).ratio() + return calculate_similarity(a, b) def search_model_list( diff --git a/core/type_utils.py b/core/type_utils.py new file mode 100644 index 0000000..06d4c52 --- /dev/null +++ b/core/type_utils.py @@ -0,0 +1,44 @@ +""" +Type Utilities Module + +Unified helper functions for safe data type casting and normalization. +""" + +from typing import Any, Dict, List + + +def as_dict(value: Any) -> Dict[str, Any]: + """ + Safely cast value to a dictionary. + + Args: + value: Value to cast + + Returns: + The dictionary if input is a dict, empty dict otherwise. + """ + return value if isinstance(value, dict) else {} + + +def as_list(value: Any) -> List[Any]: + """ + Safely cast value to a list. + + Supports: + - lists (returns list filtered of None/empty strings) + - tuples and sets (casts to list and filters) + - comma-separated strings (splits, strips, and filters) + + Args: + value: Value to cast + + Returns: + A list of elements. + """ + if isinstance(value, list): + return [item for item in value if item not in (None, "")] + if isinstance(value, (tuple, set)): + return [item for item in value if item not in (None, "")] + if isinstance(value, str): + return [item.strip() for item in value.split(",") if item.strip()] + return []