From d1a5ccfedb4cb4fb170292851f940e1f4fa267ee Mon Sep 17 00:00:00 2001 From: Dariusz L Date: Thu, 25 Jun 2026 01:33:25 +0200 Subject: [PATCH] refactor: centralize base model normalization and matching in matcher --- core/matcher.py | 63 +++++++++++++++++++++++++++- core/sources/civarchive.py | 38 ++++------------- core/sources/civitai.py | 38 ++++------------- core/sources/lora_manager_archive.py | 38 ++++------------- core/sources/popular.py | 4 +- 5 files changed, 84 insertions(+), 97 deletions(-) diff --git a/core/matcher.py b/core/matcher.py index 240fdc9..bafb9c4 100644 --- a/core/matcher.py +++ b/core/matcher.py @@ -7,7 +7,7 @@ Implements fuzzy string matching to find similar model names. import os import re import heapq -from typing import List, Dict, Tuple +from typing import List, Dict, Tuple, Optional from difflib import SequenceMatcher from .log_system.log_funcs import log_debug, log_info, log_warn, log_error, log_exception @@ -238,3 +238,64 @@ def find_matches( heapq.heapreplace(best_matches, entry) return [match for _, _, match in sorted(best_matches, key=lambda x: x[0], reverse=True)] + + +def normalize_base_model(value: str) -> str: + """ + Normalize a base model name to standard alphanumeric lowercase format. + + Args: + value: The base model name to normalize + + Returns: + Normalized base model name + """ + return re.sub(r"[^a-z0-9]+", "", str(value or "").lower()) + + +def base_model_matches(candidate: str, preferred: Optional[str]) -> bool: + """ + Check if a candidate base model matches a preferred base model family. + + Args: + candidate: Candidate base model name + preferred: Preferred base model name + + Returns: + True if the candidate base model is a match for the preferred family, False otherwise. + """ + preferred_norm = normalize_base_model(preferred or "") + if not preferred_norm: + return True + + candidate_norm = normalize_base_model(candidate or "") + if not candidate_norm: + return False + + from .sources.popular import load_base_model_aliases + aliases = load_base_model_aliases() + # Exact alias membership check - avoids false positives from substring matching + # e.g. prevents "zimage" (Z-Image alias) from matching "zimagebase" (ZImageBase) + preferred_tokens = aliases.get(preferred_norm, [preferred_norm]) + if candidate_norm in preferred_tokens: + return True + # Symmetric check: if preferred is an alias of candidate's model family + candidate_tokens = aliases.get(candidate_norm, [candidate_norm]) + return preferred_norm in candidate_tokens + + +def base_model_score(candidate: str, preferred: Optional[str]) -> float: + """ + Calculate a score for a base model match (1000.0 if matches, -1000.0 otherwise). + + Args: + candidate: Candidate base model name + preferred: Preferred base model name + + Returns: + Match score + """ + if not preferred: + return 0.0 + return 1000.0 if base_model_matches(candidate, preferred) else -1000.0 + diff --git a/core/sources/civarchive.py b/core/sources/civarchive.py index fd3ba29..cdfecd3 100644 --- a/core/sources/civarchive.py +++ b/core/sources/civarchive.py @@ -14,7 +14,13 @@ from urllib.parse import parse_qs, urljoin, urlparse import requests -from ..matcher import calculate_similarity_with_normalization, normalize_filename +from ..matcher import ( + calculate_similarity_with_normalization, + normalize_filename, + normalize_base_model as _normalize_base_model, + base_model_matches as _base_model_matches, + base_model_score as _base_model_score, +) from ..log_system.log_funcs import ( log_debug, log_info, @@ -1222,36 +1228,6 @@ def _resolve_civarchive_model_title_match( ) -def _normalize_base_model(value: str) -> str: - return re.sub(r"[^a-z0-9]+", "", str(value or "").lower()) - - -def _base_model_matches(candidate: str, preferred: Optional[str]) -> bool: - preferred_norm = _normalize_base_model(preferred or "") - if not preferred_norm: - return True - - candidate_norm = _normalize_base_model(candidate or "") - if not candidate_norm: - return False - - from .popular import load_base_model_aliases - aliases = load_base_model_aliases() - # Exact alias membership check - avoids false positives from substring matching - # e.g. prevents "zimage" (Z-Image alias) from matching "zimagebase" (ZImageBase) - preferred_tokens = aliases.get(preferred_norm, [preferred_norm]) - if candidate_norm in preferred_tokens: - return True - # Symmetric check: if preferred is an alias of candidate's model family - candidate_tokens = aliases.get(candidate_norm, [candidate_norm]) - return preferred_norm in candidate_tokens - - -def _base_model_score(candidate: str, preferred: Optional[str]) -> float: - if not preferred: - return 0.0 - return 1000.0 if _base_model_matches(candidate, preferred) else -1000.0 - def _build_result_from_payload( payload: Dict[str, Any], diff --git a/core/sources/civitai.py b/core/sources/civitai.py index 0dd0bbb..2541aad 100644 --- a/core/sources/civitai.py +++ b/core/sources/civitai.py @@ -12,7 +12,13 @@ import requests from typing import Dict, Any, Optional, List, Callable from urllib.parse import urlparse, parse_qs, quote -from ..matcher import calculate_similarity_with_normalization, normalize_filename +from ..matcher import ( + calculate_similarity_with_normalization, + normalize_filename, + normalize_base_model as _normalize_base_model, + base_model_matches as _base_model_matches, + base_model_score as _base_model_score, +) from ..log_system.log_funcs import ( log_debug, log_info, @@ -441,36 +447,6 @@ def _find_model_title_match_in_model( return None -def _normalize_base_model(value: str) -> str: - return re.sub(r"[^a-z0-9]+", "", str(value or "").lower()) - - -def _base_model_matches(candidate: str, preferred: Optional[str]) -> bool: - preferred_norm = _normalize_base_model(preferred or "") - if not preferred_norm: - return True - - candidate_norm = _normalize_base_model(candidate or "") - if not candidate_norm: - return False - - from .popular import load_base_model_aliases - aliases = load_base_model_aliases() - # Exact alias membership check - avoids false positives from substring matching - # e.g. prevents "zimage" (Z-Image alias) from matching "zimagebase" (ZImageBase) - preferred_tokens = aliases.get(preferred_norm, [preferred_norm]) - if candidate_norm in preferred_tokens: - return True - # Symmetric check: if preferred is an alias of candidate's model family - candidate_tokens = aliases.get(candidate_norm, [candidate_norm]) - return preferred_norm in candidate_tokens - - -def _base_model_score(candidate: str, preferred: Optional[str]) -> float: - if not preferred: - return 0.0 - return 1000.0 if _base_model_matches(candidate, preferred) else -1000.0 - def _find_matching_file_in_versions( versions: List[Dict[str, Any]], diff --git a/core/sources/lora_manager_archive.py b/core/sources/lora_manager_archive.py index 88d63b4..47cabd4 100644 --- a/core/sources/lora_manager_archive.py +++ b/core/sources/lora_manager_archive.py @@ -11,7 +11,13 @@ import re import time from typing import Any, Callable, Dict, List, Optional -from ..matcher import calculate_similarity_with_normalization, normalize_filename +from ..matcher import ( + calculate_similarity_with_normalization, + normalize_filename, + normalize_base_model as _normalize_base_model, + base_model_matches as _base_model_matches, + base_model_score as _base_model_score, +) from ..log_system.log_funcs import ( log_debug, log_info, @@ -175,36 +181,6 @@ def _calculate_confidence( return round(best * 100, 1) -def _normalize_base_model(value: str) -> str: - return re.sub(r"[^a-z0-9]+", "", str(value or "").lower()) - - -def _base_model_matches(candidate: str, preferred: Optional[str]) -> bool: - preferred_norm = _normalize_base_model(preferred or "") - if not preferred_norm: - return True - - candidate_norm = _normalize_base_model(candidate or "") - if not candidate_norm: - return False - - from .popular import load_base_model_aliases - aliases = load_base_model_aliases() - # Exact alias membership check - avoids false positives from substring matching - # e.g. prevents "zimage" (Z-Image alias) from matching "zimagebase" (ZImageBase) - preferred_tokens = aliases.get(preferred_norm, [preferred_norm]) - if candidate_norm in preferred_tokens: - return True - # Symmetric check: if preferred is an alias of candidate's model family - candidate_tokens = aliases.get(candidate_norm, [candidate_norm]) - return preferred_norm in candidate_tokens - - -def _base_model_score(candidate: str, preferred: Optional[str]) -> float: - if not preferred: - return 0.0 - return 1000.0 if _base_model_matches(candidate, preferred) else -1000.0 - def _extract_search_tokens(query: str) -> Dict[str, List[str]]: """Split filename/model query into useful content/version tokens.""" diff --git a/core/sources/popular.py b/core/sources/popular.py index 98c0b92..2233cfa 100644 --- a/core/sources/popular.py +++ b/core/sources/popular.py @@ -74,9 +74,7 @@ def _load_model_aliases() -> Dict[str, List[str]]: return _model_aliases_cache -def _normalize_base_model(value: str) -> str: - import re - return re.sub(r"[^a-z0-9]+", "", str(value or "").lower()) +from ..matcher import normalize_base_model as _normalize_base_model def load_base_model_aliases() -> Dict[str, List[str]]: