refactor: consolidate type and progress utilities
Add core/type_utils.py with as_dict/as_list helpers, and core/progress.py with a unified report_progress() callback. Replace inline duplicated implementations of _json_safe_metadata(), _as_list(), normalize_filename(), and per-source _report_progress() with calls to the shared modules across downloader, matcher, resolver, and all source modules (CivitAI, CivArchive, HuggingFace, LoRA Manager). Extend filename normalization to include dots as separators.
This commit is contained in:
+3
-8
@@ -24,6 +24,7 @@ from .log_system.log_funcs import (
|
||||
)
|
||||
from .resolver import normalize_sha256
|
||||
from .path_utils import is_path_within, get_path_identity
|
||||
from .type_utils import as_dict, as_list
|
||||
|
||||
try:
|
||||
import folder_paths
|
||||
@@ -180,17 +181,11 @@ def _json_safe_metadata(value: Any, depth: int = 0) -> Any:
|
||||
|
||||
|
||||
def _as_dict(value: Any) -> Dict[str, Any]:
|
||||
return value if isinstance(value, dict) else {}
|
||||
return as_dict(value)
|
||||
|
||||
|
||||
def _as_list(value: Any) -> List[Any]:
|
||||
if isinstance(value, list):
|
||||
return [item for item in value if item not in (None, "")]
|
||||
if isinstance(value, (tuple, set)):
|
||||
return [item for item in value if item not in (None, "")]
|
||||
if isinstance(value, str):
|
||||
return [item.strip() for item in value.split(",") if item.strip()]
|
||||
return []
|
||||
return as_list(value)
|
||||
|
||||
|
||||
def _first_present(*values: Any) -> Any:
|
||||
|
||||
+2
-2
@@ -37,8 +37,8 @@ def normalize_filename(filename: str) -> str:
|
||||
# Convert to lowercase
|
||||
base = base.lower()
|
||||
|
||||
# Normalize separators: replace underscores, hyphens, and spaces with a single space
|
||||
base = re.sub(r"[_\-\s]+", " ", base)
|
||||
# Normalize separators: replace underscores, hyphens, dots, and spaces with a single space
|
||||
base = re.sub(r"[_\-\.\s]+", " ", base)
|
||||
|
||||
# Strip whitespace
|
||||
base = base.strip()
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
"""
|
||||
Progress Reporting Module
|
||||
|
||||
Unified utility for reporting progress of asynchronous search and download operations.
|
||||
"""
|
||||
|
||||
from typing import Dict, Any, Optional, Callable
|
||||
from .log_system.log_funcs import log_debug
|
||||
|
||||
|
||||
def report_progress(
|
||||
progress_callback: Optional[Callable[[Dict[str, Any]], None]],
|
||||
stage: str,
|
||||
message: str,
|
||||
percent: Optional[float] = None,
|
||||
error_context: str = "Progress callback",
|
||||
**extra: Any,
|
||||
) -> None:
|
||||
"""
|
||||
Report progress via callback payload.
|
||||
|
||||
Args:
|
||||
progress_callback: Callback function to execute
|
||||
stage: Progress stage name
|
||||
message: Informational message
|
||||
percent: Progress percentage (0.0 to 100.0)
|
||||
error_context: Source name to use in error logging
|
||||
extra: Additional key-value pairs to add to payload
|
||||
"""
|
||||
if not progress_callback:
|
||||
return
|
||||
|
||||
payload = {"stage": stage, "message": message}
|
||||
if percent is not None:
|
||||
payload["percent"] = percent
|
||||
if extra:
|
||||
payload.update(extra)
|
||||
|
||||
try:
|
||||
progress_callback(payload)
|
||||
except Exception as e:
|
||||
log_debug(f"{error_context} failed: {e}")
|
||||
+3
-6
@@ -27,6 +27,7 @@ from .workflow_analyzer import (
|
||||
should_scan_as_model_reference,
|
||||
)
|
||||
from .matcher import find_matches
|
||||
from .type_utils import as_dict, as_list
|
||||
from .workflow_updater import update_workflow_nodes
|
||||
from .sources.civitai import resolve_urn
|
||||
from .sources.huggingface import parse_huggingface_url as parse_hf_url
|
||||
@@ -327,15 +328,11 @@ def _metadata_sidecar_candidates(model_path: str) -> List[str]:
|
||||
|
||||
|
||||
def _as_dict(value: Any) -> Dict[str, Any]:
|
||||
return value if isinstance(value, dict) else {}
|
||||
return as_dict(value)
|
||||
|
||||
|
||||
def _as_list(value: Any) -> List[Any]:
|
||||
if isinstance(value, list):
|
||||
return value
|
||||
if isinstance(value, tuple):
|
||||
return list(value)
|
||||
return []
|
||||
return as_list(value)
|
||||
|
||||
|
||||
def _collect_hashes_from_container(value: Any) -> List[str]:
|
||||
|
||||
@@ -21,6 +21,7 @@ from ..matcher import (
|
||||
base_model_matches as _base_model_matches,
|
||||
base_model_score as _base_model_score,
|
||||
)
|
||||
from ..progress import report_progress
|
||||
from ..log_system.log_funcs import (
|
||||
log_debug,
|
||||
log_info,
|
||||
@@ -98,19 +99,14 @@ def _report_progress(
|
||||
percent: Optional[float] = None,
|
||||
**extra: Any,
|
||||
) -> None:
|
||||
if not progress_callback:
|
||||
return
|
||||
|
||||
payload = {"stage": stage, "message": message}
|
||||
if percent is not None:
|
||||
payload["percent"] = percent
|
||||
if extra:
|
||||
payload.update(extra)
|
||||
|
||||
try:
|
||||
progress_callback(payload)
|
||||
except Exception as e:
|
||||
log_debug(f"CivArchive progress callback failed: {e}")
|
||||
report_progress(
|
||||
progress_callback,
|
||||
stage,
|
||||
message,
|
||||
percent,
|
||||
error_context="CivArchive progress callback",
|
||||
**extra,
|
||||
)
|
||||
|
||||
|
||||
def is_civarchive_available() -> bool:
|
||||
|
||||
+9
-13
@@ -19,6 +19,7 @@ from ..matcher import (
|
||||
base_model_matches as _base_model_matches,
|
||||
base_model_score as _base_model_score,
|
||||
)
|
||||
from ..progress import report_progress
|
||||
from ..log_system.log_funcs import (
|
||||
log_debug,
|
||||
log_info,
|
||||
@@ -79,19 +80,14 @@ def _report_progress(
|
||||
percent: Optional[float] = None,
|
||||
**extra: Any,
|
||||
) -> None:
|
||||
if not progress_callback:
|
||||
return
|
||||
|
||||
payload = {"stage": stage, "message": message}
|
||||
if percent is not None:
|
||||
payload["percent"] = percent
|
||||
if extra:
|
||||
payload.update(extra)
|
||||
|
||||
try:
|
||||
progress_callback(payload)
|
||||
except Exception as e:
|
||||
log_debug(f"CivitAI progress callback failed: {e}")
|
||||
report_progress(
|
||||
progress_callback,
|
||||
stage,
|
||||
message,
|
||||
percent,
|
||||
error_context="CivitAI progress callback",
|
||||
**extra,
|
||||
)
|
||||
|
||||
|
||||
def check_civitai_session_token(session_token: Optional[str]) -> Dict[str, Any]:
|
||||
|
||||
@@ -13,6 +13,7 @@ import requests
|
||||
from typing import Dict, Any, Optional, List, Callable
|
||||
from urllib.parse import urlparse, quote
|
||||
|
||||
from ..progress import report_progress
|
||||
from ..log_system.log_funcs import (
|
||||
log_debug,
|
||||
log_info,
|
||||
@@ -54,19 +55,14 @@ def _report_progress(
|
||||
percent: Optional[float] = None,
|
||||
**extra: Any,
|
||||
) -> None:
|
||||
if not progress_callback:
|
||||
return
|
||||
|
||||
payload = {"stage": stage, "message": message}
|
||||
if percent is not None:
|
||||
payload["percent"] = percent
|
||||
if extra:
|
||||
payload.update(extra)
|
||||
|
||||
try:
|
||||
progress_callback(payload)
|
||||
except Exception as e:
|
||||
log_debug(f"HuggingFace progress callback failed: {e}")
|
||||
report_progress(
|
||||
progress_callback,
|
||||
stage,
|
||||
message,
|
||||
percent,
|
||||
error_context="HuggingFace progress callback",
|
||||
**extra,
|
||||
)
|
||||
|
||||
|
||||
def check_huggingface_token(token: Optional[str]) -> Dict[str, Any]:
|
||||
|
||||
@@ -18,6 +18,7 @@ from ..matcher import (
|
||||
base_model_matches as _base_model_matches,
|
||||
base_model_score as _base_model_score,
|
||||
)
|
||||
from ..progress import report_progress
|
||||
from ..log_system.log_funcs import (
|
||||
log_debug,
|
||||
log_info,
|
||||
@@ -63,19 +64,14 @@ def _report_progress(
|
||||
percent: Optional[float] = None,
|
||||
**extra: Any,
|
||||
) -> None:
|
||||
if not progress_callback:
|
||||
return
|
||||
|
||||
payload = {"stage": stage, "message": message}
|
||||
if percent is not None:
|
||||
payload["percent"] = percent
|
||||
if extra:
|
||||
payload.update(extra)
|
||||
|
||||
try:
|
||||
progress_callback(payload)
|
||||
except Exception as e:
|
||||
log_debug(f"LoRA Manager archive progress callback failed: {e}")
|
||||
report_progress(
|
||||
progress_callback,
|
||||
stage,
|
||||
message,
|
||||
percent,
|
||||
error_context="LoRA Manager archive progress callback",
|
||||
**extra,
|
||||
)
|
||||
|
||||
|
||||
def get_lora_manager_archive_db_path() -> Optional[str]:
|
||||
|
||||
@@ -11,9 +11,10 @@ import shutil
|
||||
import tempfile
|
||||
from datetime import datetime, timezone
|
||||
from typing import Dict, Any, Optional, List
|
||||
from difflib import SequenceMatcher
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from ..matcher import normalize_filename, calculate_similarity
|
||||
|
||||
from ..log_system.log_funcs import (
|
||||
log_debug,
|
||||
log_info,
|
||||
@@ -219,16 +220,12 @@ def update_model_list_from_remote() -> Dict[str, Any]:
|
||||
|
||||
def _normalize_filename(filename: str) -> str:
|
||||
"""Normalize filename for comparison."""
|
||||
# Remove extension
|
||||
base = os.path.splitext(filename)[0].lower()
|
||||
# Replace separators with spaces
|
||||
base = base.replace("-", " ").replace("_", " ").replace(".", " ")
|
||||
return base
|
||||
return normalize_filename(filename)
|
||||
|
||||
|
||||
def _similarity(a: str, b: str) -> float:
|
||||
"""Calculate similarity between two strings."""
|
||||
return SequenceMatcher(None, a, b).ratio()
|
||||
return calculate_similarity(a, b)
|
||||
|
||||
|
||||
def search_model_list(
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
"""
|
||||
Type Utilities Module
|
||||
|
||||
Unified helper functions for safe data type casting and normalization.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List
|
||||
|
||||
|
||||
def as_dict(value: Any) -> Dict[str, Any]:
|
||||
"""
|
||||
Safely cast value to a dictionary.
|
||||
|
||||
Args:
|
||||
value: Value to cast
|
||||
|
||||
Returns:
|
||||
The dictionary if input is a dict, empty dict otherwise.
|
||||
"""
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def as_list(value: Any) -> List[Any]:
|
||||
"""
|
||||
Safely cast value to a list.
|
||||
|
||||
Supports:
|
||||
- lists (returns list filtered of None/empty strings)
|
||||
- tuples and sets (casts to list and filters)
|
||||
- comma-separated strings (splits, strips, and filters)
|
||||
|
||||
Args:
|
||||
value: Value to cast
|
||||
|
||||
Returns:
|
||||
A list of elements.
|
||||
"""
|
||||
if isinstance(value, list):
|
||||
return [item for item in value if item not in (None, "")]
|
||||
if isinstance(value, (tuple, set)):
|
||||
return [item for item in value if item not in (None, "")]
|
||||
if isinstance(value, str):
|
||||
return [item.strip() for item in value.split(",") if item.strip()]
|
||||
return []
|
||||
Reference in New Issue
Block a user