refactor: consolidate type and progress utilities

Add core/type_utils.py with as_dict/as_list helpers, and
core/progress.py with a unified report_progress() callback.

Replace inline duplicated implementations of _json_safe_metadata(),
_as_list(), normalize_filename(), and per-source _report_progress()
with calls to the shared modules across downloader, matcher, resolver,
and all source modules (CivitAI, CivArchive, HuggingFace, LoRA Manager).
Extend filename normalization to include dots as separators.
This commit is contained in:
Dariusz L
2026-06-25 02:09:47 +02:00
parent d49ae0becc
commit e4c3b0fb6f
10 changed files with 134 additions and 75 deletions
+3 -8
View File
@@ -24,6 +24,7 @@ from .log_system.log_funcs import (
)
from .resolver import normalize_sha256
from .path_utils import is_path_within, get_path_identity
from .type_utils import as_dict, as_list
try:
import folder_paths
@@ -180,17 +181,11 @@ def _json_safe_metadata(value: Any, depth: int = 0) -> Any:
def _as_dict(value: Any) -> Dict[str, Any]:
return value if isinstance(value, dict) else {}
return as_dict(value)
def _as_list(value: Any) -> List[Any]:
if isinstance(value, list):
return [item for item in value if item not in (None, "")]
if isinstance(value, (tuple, set)):
return [item for item in value if item not in (None, "")]
if isinstance(value, str):
return [item.strip() for item in value.split(",") if item.strip()]
return []
return as_list(value)
def _first_present(*values: Any) -> Any:
+2 -2
View File
@@ -37,8 +37,8 @@ def normalize_filename(filename: str) -> str:
# Convert to lowercase
base = base.lower()
# Normalize separators: replace underscores, hyphens, and spaces with a single space
base = re.sub(r"[_\-\s]+", " ", base)
# Normalize separators: replace underscores, hyphens, dots, and spaces with a single space
base = re.sub(r"[_\-\.\s]+", " ", base)
# Strip whitespace
base = base.strip()
+42
View File
@@ -0,0 +1,42 @@
"""
Progress Reporting Module
Unified utility for reporting progress of asynchronous search and download operations.
"""
from typing import Dict, Any, Optional, Callable
from .log_system.log_funcs import log_debug
def report_progress(
progress_callback: Optional[Callable[[Dict[str, Any]], None]],
stage: str,
message: str,
percent: Optional[float] = None,
error_context: str = "Progress callback",
**extra: Any,
) -> None:
"""
Report progress via callback payload.
Args:
progress_callback: Callback function to execute
stage: Progress stage name
message: Informational message
percent: Progress percentage (0.0 to 100.0)
error_context: Source name to use in error logging
extra: Additional key-value pairs to add to payload
"""
if not progress_callback:
return
payload = {"stage": stage, "message": message}
if percent is not None:
payload["percent"] = percent
if extra:
payload.update(extra)
try:
progress_callback(payload)
except Exception as e:
log_debug(f"{error_context} failed: {e}")
+3 -6
View File
@@ -27,6 +27,7 @@ from .workflow_analyzer import (
should_scan_as_model_reference,
)
from .matcher import find_matches
from .type_utils import as_dict, as_list
from .workflow_updater import update_workflow_nodes
from .sources.civitai import resolve_urn
from .sources.huggingface import parse_huggingface_url as parse_hf_url
@@ -327,15 +328,11 @@ def _metadata_sidecar_candidates(model_path: str) -> List[str]:
def _as_dict(value: Any) -> Dict[str, Any]:
return value if isinstance(value, dict) else {}
return as_dict(value)
def _as_list(value: Any) -> List[Any]:
if isinstance(value, list):
return value
if isinstance(value, tuple):
return list(value)
return []
return as_list(value)
def _collect_hashes_from_container(value: Any) -> List[str]:
+9 -13
View File
@@ -21,6 +21,7 @@ from ..matcher import (
base_model_matches as _base_model_matches,
base_model_score as _base_model_score,
)
from ..progress import report_progress
from ..log_system.log_funcs import (
log_debug,
log_info,
@@ -98,19 +99,14 @@ def _report_progress(
percent: Optional[float] = None,
**extra: Any,
) -> None:
if not progress_callback:
return
payload = {"stage": stage, "message": message}
if percent is not None:
payload["percent"] = percent
if extra:
payload.update(extra)
try:
progress_callback(payload)
except Exception as e:
log_debug(f"CivArchive progress callback failed: {e}")
report_progress(
progress_callback,
stage,
message,
percent,
error_context="CivArchive progress callback",
**extra,
)
def is_civarchive_available() -> bool:
+9 -13
View File
@@ -19,6 +19,7 @@ from ..matcher import (
base_model_matches as _base_model_matches,
base_model_score as _base_model_score,
)
from ..progress import report_progress
from ..log_system.log_funcs import (
log_debug,
log_info,
@@ -79,19 +80,14 @@ def _report_progress(
percent: Optional[float] = None,
**extra: Any,
) -> None:
if not progress_callback:
return
payload = {"stage": stage, "message": message}
if percent is not None:
payload["percent"] = percent
if extra:
payload.update(extra)
try:
progress_callback(payload)
except Exception as e:
log_debug(f"CivitAI progress callback failed: {e}")
report_progress(
progress_callback,
stage,
message,
percent,
error_context="CivitAI progress callback",
**extra,
)
def check_civitai_session_token(session_token: Optional[str]) -> Dict[str, Any]:
+9 -13
View File
@@ -13,6 +13,7 @@ import requests
from typing import Dict, Any, Optional, List, Callable
from urllib.parse import urlparse, quote
from ..progress import report_progress
from ..log_system.log_funcs import (
log_debug,
log_info,
@@ -54,19 +55,14 @@ def _report_progress(
percent: Optional[float] = None,
**extra: Any,
) -> None:
if not progress_callback:
return
payload = {"stage": stage, "message": message}
if percent is not None:
payload["percent"] = percent
if extra:
payload.update(extra)
try:
progress_callback(payload)
except Exception as e:
log_debug(f"HuggingFace progress callback failed: {e}")
report_progress(
progress_callback,
stage,
message,
percent,
error_context="HuggingFace progress callback",
**extra,
)
def check_huggingface_token(token: Optional[str]) -> Dict[str, Any]:
+9 -13
View File
@@ -18,6 +18,7 @@ from ..matcher import (
base_model_matches as _base_model_matches,
base_model_score as _base_model_score,
)
from ..progress import report_progress
from ..log_system.log_funcs import (
log_debug,
log_info,
@@ -63,19 +64,14 @@ def _report_progress(
percent: Optional[float] = None,
**extra: Any,
) -> None:
if not progress_callback:
return
payload = {"stage": stage, "message": message}
if percent is not None:
payload["percent"] = percent
if extra:
payload.update(extra)
try:
progress_callback(payload)
except Exception as e:
log_debug(f"LoRA Manager archive progress callback failed: {e}")
report_progress(
progress_callback,
stage,
message,
percent,
error_context="LoRA Manager archive progress callback",
**extra,
)
def get_lora_manager_archive_db_path() -> Optional[str]:
+4 -7
View File
@@ -11,9 +11,10 @@ import shutil
import tempfile
from datetime import datetime, timezone
from typing import Dict, Any, Optional, List
from difflib import SequenceMatcher
from urllib.request import Request, urlopen
from ..matcher import normalize_filename, calculate_similarity
from ..log_system.log_funcs import (
log_debug,
log_info,
@@ -219,16 +220,12 @@ def update_model_list_from_remote() -> Dict[str, Any]:
def _normalize_filename(filename: str) -> str:
"""Normalize filename for comparison."""
# Remove extension
base = os.path.splitext(filename)[0].lower()
# Replace separators with spaces
base = base.replace("-", " ").replace("_", " ").replace(".", " ")
return base
return normalize_filename(filename)
def _similarity(a: str, b: str) -> float:
"""Calculate similarity between two strings."""
return SequenceMatcher(None, a, b).ratio()
return calculate_similarity(a, b)
def search_model_list(
+44
View File
@@ -0,0 +1,44 @@
"""
Type Utilities Module
Unified helper functions for safe data type casting and normalization.
"""
from typing import Any, Dict, List
def as_dict(value: Any) -> Dict[str, Any]:
"""
Safely cast value to a dictionary.
Args:
value: Value to cast
Returns:
The dictionary if input is a dict, empty dict otherwise.
"""
return value if isinstance(value, dict) else {}
def as_list(value: Any) -> List[Any]:
"""
Safely cast value to a list.
Supports:
- lists (returns list filtered of None/empty strings)
- tuples and sets (casts to list and filters)
- comma-separated strings (splits, strips, and filters)
Args:
value: Value to cast
Returns:
A list of elements.
"""
if isinstance(value, list):
return [item for item in value if item not in (None, "")]
if isinstance(value, (tuple, set)):
return [item for item in value if item not in (None, "")]
if isinstance(value, str):
return [item.strip() for item in value.split(",") if item.strip()]
return []