feat: add base model filtering across search sources, API routes, and frontend UI
This commit is contained in:
@@ -486,6 +486,34 @@ def _calculate_confidence(
|
||||
return round(best * 100, 1)
|
||||
|
||||
|
||||
def _normalize_base_model(value: str) -> str:
|
||||
return re.sub(r"[^a-z0-9]+", "", str(value or "").lower())
|
||||
|
||||
|
||||
def _base_model_matches(candidate: str, preferred: Optional[str]) -> bool:
|
||||
preferred_norm = _normalize_base_model(preferred or "")
|
||||
if not preferred_norm:
|
||||
return True
|
||||
|
||||
candidate_norm = _normalize_base_model(candidate or "")
|
||||
if not candidate_norm:
|
||||
return False
|
||||
|
||||
from .popular import load_base_model_aliases
|
||||
aliases = load_base_model_aliases()
|
||||
preferred_tokens = aliases.get(preferred_norm, [preferred_norm])
|
||||
return any(
|
||||
token and (token in candidate_norm or candidate_norm in token)
|
||||
for token in preferred_tokens
|
||||
)
|
||||
|
||||
|
||||
def _base_model_score(candidate: str, preferred: Optional[str]) -> float:
|
||||
if not preferred:
|
||||
return 0.0
|
||||
return 1000.0 if _base_model_matches(candidate, preferred) else -1000.0
|
||||
|
||||
|
||||
def _build_result_from_payload(
|
||||
payload: Dict[str, Any],
|
||||
query: str = "",
|
||||
@@ -751,6 +779,7 @@ def search_civarchive(
|
||||
def search_civarchive_for_file(
|
||||
filename: str,
|
||||
model_type: Optional[str] = None,
|
||||
base_model_context: Optional[str] = None,
|
||||
exact_only: bool = False,
|
||||
limit: int = DEFAULT_CIVARCHIVE_CANDIDATE_LIMIT,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
@@ -762,8 +791,9 @@ def search_civarchive_for_file(
|
||||
return None
|
||||
|
||||
limit = max(1, min(int(limit), MAX_CIVARCHIVE_CANDIDATE_LIMIT))
|
||||
base_model_key = _normalize_base_model(base_model_context or "")
|
||||
cache_key = (
|
||||
f"file::{normalized_filename.lower()}::{model_type or ''}::{exact_only}::{limit}"
|
||||
f"file::{normalized_filename.lower()}::{model_type or ''}::{base_model_key}::{exact_only}::{limit}"
|
||||
)
|
||||
if cache_key in _search_cache:
|
||||
return _search_cache[cache_key]
|
||||
@@ -780,6 +810,7 @@ def search_civarchive_for_file(
|
||||
|
||||
best_match = None
|
||||
best_confidence = 0.0
|
||||
best_rank = -9999.0
|
||||
seen = set()
|
||||
|
||||
try:
|
||||
@@ -810,19 +841,29 @@ def search_civarchive_for_file(
|
||||
|
||||
resolved["confidence"] = confidence
|
||||
resolved["match_type"] = "exact" if confidence == 100.0 else "similar"
|
||||
base_model_matches = _base_model_matches(
|
||||
resolved.get("base_model"), base_model_context
|
||||
)
|
||||
rank = confidence + _base_model_score(
|
||||
resolved.get("base_model"), base_model_context
|
||||
)
|
||||
|
||||
if confidence > best_confidence:
|
||||
if rank > best_rank:
|
||||
best_rank = rank
|
||||
best_confidence = confidence
|
||||
best_match = resolved
|
||||
|
||||
if confidence == 100.0:
|
||||
if confidence == 100.0 and base_model_matches:
|
||||
_search_cache[cache_key] = best_match
|
||||
return best_match
|
||||
|
||||
if len(seen) >= limit:
|
||||
break
|
||||
|
||||
if best_match:
|
||||
if best_match and (
|
||||
not base_model_context
|
||||
or _base_model_matches(best_match.get("base_model"), base_model_context)
|
||||
):
|
||||
break
|
||||
except CivArchiveSearchError:
|
||||
raise
|
||||
@@ -832,6 +873,10 @@ def search_civarchive_for_file(
|
||||
|
||||
if best_match and best_confidence < 40:
|
||||
best_match = None
|
||||
if best_match and base_model_context and not _base_model_matches(
|
||||
best_match.get("base_model"), base_model_context
|
||||
):
|
||||
best_match = None
|
||||
|
||||
_search_cache[cache_key] = best_match
|
||||
return best_match
|
||||
|
||||
+48
-6
@@ -261,6 +261,34 @@ def _calculate_filename_confidence(target_filename: str, candidate_filename: str
|
||||
return round(max(similarity, similarity_no_ext) * 100, 1)
|
||||
|
||||
|
||||
def _normalize_base_model(value: str) -> str:
|
||||
return re.sub(r"[^a-z0-9]+", "", str(value or "").lower())
|
||||
|
||||
|
||||
def _base_model_matches(candidate: str, preferred: Optional[str]) -> bool:
|
||||
preferred_norm = _normalize_base_model(preferred or "")
|
||||
if not preferred_norm:
|
||||
return True
|
||||
|
||||
candidate_norm = _normalize_base_model(candidate or "")
|
||||
if not candidate_norm:
|
||||
return False
|
||||
|
||||
from .popular import load_base_model_aliases
|
||||
aliases = load_base_model_aliases()
|
||||
preferred_tokens = aliases.get(preferred_norm, [preferred_norm])
|
||||
return any(
|
||||
token and (token in candidate_norm or candidate_norm in token)
|
||||
for token in preferred_tokens
|
||||
)
|
||||
|
||||
|
||||
def _base_model_score(candidate: str, preferred: Optional[str]) -> float:
|
||||
if not preferred:
|
||||
return 0.0
|
||||
return 1000.0 if _base_model_matches(candidate, preferred) else -1000.0
|
||||
|
||||
|
||||
def _find_matching_file_in_versions(
|
||||
versions: List[Dict[str, Any]],
|
||||
filename: str,
|
||||
@@ -502,6 +530,7 @@ def _find_civitai_file_in_model(
|
||||
api_key: Optional[str] = None,
|
||||
exact_only: bool = False,
|
||||
preferred_version_id: Optional[int] = None,
|
||||
base_model_context: Optional[str] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Load one CivitAI model and search all its versions for the requested file."""
|
||||
filename_lower = filename.lower()
|
||||
@@ -554,7 +583,9 @@ def _find_civitai_file_in_model(
|
||||
if preferred_version_id is not None:
|
||||
resolved = resolve_urn(model_id, preferred_version_id, api_key)
|
||||
if resolved:
|
||||
if resolved_version_matches(resolved):
|
||||
if resolved_version_matches(resolved) and _base_model_matches(
|
||||
resolved.get("base_model"), base_model_context
|
||||
):
|
||||
return build_result_from_resolved_version(resolved, preferred_version_id)
|
||||
|
||||
best_resolved_result = None
|
||||
@@ -589,17 +620,22 @@ def _find_civitai_file_in_model(
|
||||
|
||||
resolved = resolve_urn(model_id, version_id, api_key)
|
||||
if resolved:
|
||||
if resolved_version_matches(resolved):
|
||||
if resolved_version_matches(resolved) and _base_model_matches(
|
||||
resolved.get("base_model"), base_model_context
|
||||
):
|
||||
return build_result_from_resolved_version(resolved, version_id)
|
||||
|
||||
if not exact_only:
|
||||
expected_filename = resolved.get("expected_filename", "")
|
||||
confidence = _calculate_filename_confidence(filename, expected_filename)
|
||||
ranking_score = confidence + _base_model_score(
|
||||
resolved.get("base_model"), base_model_context
|
||||
)
|
||||
log_debug(
|
||||
f"CivitAI resolved version confidence: model_id={model_id}, version_id={version_id}, expected_filename={expected_filename}, confidence={confidence}"
|
||||
)
|
||||
if confidence > best_resolved_confidence:
|
||||
best_resolved_confidence = confidence
|
||||
if ranking_score > best_resolved_confidence:
|
||||
best_resolved_confidence = ranking_score
|
||||
best_resolved_result = build_result_from_resolved_version(
|
||||
resolved, version_id
|
||||
)
|
||||
@@ -639,7 +675,10 @@ def _find_civitai_file_in_model(
|
||||
log_info(
|
||||
f"CivitAI version-list probable match: model_id={model_id}, version_id={result.get('version_id')}, filename={result.get('filename')}, confidence={result['confidence']}"
|
||||
)
|
||||
return result
|
||||
if _base_model_matches(result.get("base_model"), base_model_context):
|
||||
return result
|
||||
if not base_model_context:
|
||||
return result
|
||||
|
||||
return None
|
||||
|
||||
@@ -716,6 +755,7 @@ def search_civitai_for_file(
|
||||
api_key: Optional[str] = None,
|
||||
exact_only: bool = False,
|
||||
model_type: Optional[str] = None,
|
||||
base_model_context: Optional[str] = None,
|
||||
session_token: Optional[str] = None,
|
||||
candidate_limit: int = DEFAULT_CIVITAI_CANDIDATE_LIMIT,
|
||||
use_trpc_search: bool = True,
|
||||
@@ -739,9 +779,10 @@ def search_civitai_for_file(
|
||||
candidate_limit = max(1, min(int(candidate_limit), MAX_CIVITAI_CANDIDATE_LIMIT))
|
||||
session_key = "session" if session_token else "anon"
|
||||
model_type_key = str(model_type or "").lower()
|
||||
base_model_key = _normalize_base_model(base_model_context or "")
|
||||
methods_key = f"trpc{int(bool(use_trpc_search))}_html{int(bool(use_html_fallback))}"
|
||||
cache_key = (
|
||||
f"civit_{filename}_exact{exact_only}_type{model_type_key}_session{session_key}_limit{candidate_limit}_{methods_key}"
|
||||
f"civit_{filename}_exact{exact_only}_type{model_type_key}_base{base_model_key}_session{session_key}_limit{candidate_limit}_{methods_key}"
|
||||
)
|
||||
if cache_key in _search_cache:
|
||||
log_debug(f"CivitAI search cache hit for {filename} (exact_only={exact_only})")
|
||||
@@ -798,6 +839,7 @@ def search_civitai_for_file(
|
||||
api_key=api_key,
|
||||
exact_only=exact_only,
|
||||
preferred_version_id=version_id,
|
||||
base_model_context=base_model_context,
|
||||
)
|
||||
if result:
|
||||
confidence = float(result.get("confidence") or 0.0)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""
|
||||
"""
|
||||
LoRA Manager Archive Source Module
|
||||
|
||||
Search archived CivitAI metadata stored by comfyui-lora-manager.
|
||||
@@ -153,6 +153,34 @@ def _calculate_confidence(
|
||||
return round(best * 100, 1)
|
||||
|
||||
|
||||
def _normalize_base_model(value: str) -> str:
|
||||
return re.sub(r"[^a-z0-9]+", "", str(value or "").lower())
|
||||
|
||||
|
||||
def _base_model_matches(candidate: str, preferred: Optional[str]) -> bool:
|
||||
preferred_norm = _normalize_base_model(preferred or "")
|
||||
if not preferred_norm:
|
||||
return True
|
||||
|
||||
candidate_norm = _normalize_base_model(candidate or "")
|
||||
if not candidate_norm:
|
||||
return False
|
||||
|
||||
from .popular import load_base_model_aliases
|
||||
aliases = load_base_model_aliases()
|
||||
preferred_tokens = aliases.get(preferred_norm, [preferred_norm])
|
||||
return any(
|
||||
token and (token in candidate_norm or candidate_norm in token)
|
||||
for token in preferred_tokens
|
||||
)
|
||||
|
||||
|
||||
def _base_model_score(candidate: str, preferred: Optional[str]) -> float:
|
||||
if not preferred:
|
||||
return 0.0
|
||||
return 1000.0 if _base_model_matches(candidate, preferred) else -1000.0
|
||||
|
||||
|
||||
def _extract_search_tokens(query: str) -> Dict[str, List[str]]:
|
||||
"""Split filename/model query into useful content/version tokens."""
|
||||
base_query = os.path.splitext(os.path.basename(query or ""))[0]
|
||||
@@ -513,7 +541,10 @@ def _build_result_from_row(
|
||||
|
||||
|
||||
def search_lora_manager_archive(
|
||||
query: str, model_type: Optional[str] = None, limit: int = 10
|
||||
query: str,
|
||||
model_type: Optional[str] = None,
|
||||
base_model_context: Optional[str] = None,
|
||||
limit: int = 10,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Search the local comfyui-lora-manager CivitAI archive by model name.
|
||||
@@ -531,7 +562,10 @@ def search_lora_manager_archive(
|
||||
return []
|
||||
|
||||
normalized_type = _normalize_model_type(model_type)
|
||||
cache_key = f"search::{normalized_query.lower()}::{normalized_type}::{limit}"
|
||||
base_model_key = _normalize_base_model(base_model_context or "")
|
||||
cache_key = (
|
||||
f"search::{normalized_query.lower()}::{normalized_type}::{base_model_key}::{limit}"
|
||||
)
|
||||
if cache_key in _search_cache:
|
||||
return list(_search_cache[cache_key])
|
||||
|
||||
@@ -562,22 +596,27 @@ def search_lora_manager_archive(
|
||||
)
|
||||
if confidence < 40:
|
||||
continue
|
||||
scored_candidates.append((confidence, row))
|
||||
rank = confidence + _base_model_score(
|
||||
row["base_model"], base_model_context
|
||||
)
|
||||
scored_candidates.append((rank, confidence, row))
|
||||
|
||||
scored_candidates.sort(
|
||||
key=lambda item: (
|
||||
item[0],
|
||||
1 if item[1]["version_name"] else 0,
|
||||
item[1],
|
||||
1 if item[2]["version_name"] else 0,
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
selected_version_ids = [
|
||||
row["version_id"] for _, row in scored_candidates[: max(limit * 3, limit)]
|
||||
row["version_id"]
|
||||
for _, _, row in scored_candidates[: max(limit * 3, limit)]
|
||||
]
|
||||
full_rows = _load_full_rows_for_versions(conn, selected_version_ids)
|
||||
|
||||
results = []
|
||||
for _, candidate_row in scored_candidates:
|
||||
for _, _, candidate_row in scored_candidates:
|
||||
version_id = candidate_row["version_id"]
|
||||
full_row = full_rows.get(version_id)
|
||||
if not full_row:
|
||||
@@ -585,10 +624,15 @@ def search_lora_manager_archive(
|
||||
result = _build_result_from_row(conn, full_row, normalized_query)
|
||||
if not result or result["confidence"] < 40:
|
||||
continue
|
||||
if base_model_context and not _base_model_matches(
|
||||
result.get("base_model"), base_model_context
|
||||
):
|
||||
continue
|
||||
results.append(result)
|
||||
|
||||
results.sort(
|
||||
key=lambda item: (
|
||||
_base_model_score(item.get("base_model"), base_model_context),
|
||||
item.get("confidence", 0),
|
||||
1 if item.get("match_type") == "exact" else 0,
|
||||
),
|
||||
@@ -611,6 +655,7 @@ def search_lora_manager_archive(
|
||||
def search_lora_manager_archive_for_file(
|
||||
filename: str,
|
||||
model_type: Optional[str] = None,
|
||||
base_model_context: Optional[str] = None,
|
||||
exact_only: bool = False,
|
||||
limit: int = 10,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
@@ -621,29 +666,43 @@ def search_lora_manager_archive_for_file(
|
||||
if not search_query:
|
||||
return None
|
||||
|
||||
base_model_key = _normalize_base_model(base_model_context or "")
|
||||
cache_key = (
|
||||
f"file::{filename.lower()}::{_normalize_model_type(model_type)}::{exact_only}::{limit}"
|
||||
f"file::{filename.lower()}::{_normalize_model_type(model_type)}::{base_model_key}::{exact_only}::{limit}"
|
||||
)
|
||||
if cache_key in _search_cache:
|
||||
return _search_cache[cache_key]
|
||||
|
||||
candidates = search_lora_manager_archive(
|
||||
search_query, model_type=model_type, limit=limit
|
||||
search_query,
|
||||
model_type=model_type,
|
||||
base_model_context=base_model_context,
|
||||
limit=limit,
|
||||
)
|
||||
best_match = None
|
||||
best_confidence = 0.0
|
||||
best_rank = -9999.0
|
||||
|
||||
for candidate in candidates:
|
||||
candidate_filename = candidate.get("filename", "")
|
||||
confidence = _calculate_confidence(filename, candidate.get("name", ""), candidate.get("version_name", ""), candidate_filename)
|
||||
if exact_only and confidence < 100.0:
|
||||
continue
|
||||
if confidence > best_confidence:
|
||||
base_model_matches = _base_model_matches(
|
||||
candidate.get("base_model"), base_model_context
|
||||
)
|
||||
if base_model_context and not base_model_matches:
|
||||
continue
|
||||
rank = confidence + _base_model_score(
|
||||
candidate.get("base_model"), base_model_context
|
||||
)
|
||||
if rank > best_rank:
|
||||
best_rank = rank
|
||||
best_confidence = confidence
|
||||
best_match = dict(candidate)
|
||||
best_match["confidence"] = confidence
|
||||
best_match["match_type"] = "exact" if confidence == 100.0 else "similar"
|
||||
if confidence == 100.0:
|
||||
if confidence == 100.0 and base_model_matches:
|
||||
break
|
||||
|
||||
if best_match is None and exact_only:
|
||||
|
||||
+305
-1
@@ -6,6 +6,10 @@ Curated list of common models with known download URLs.
|
||||
|
||||
import os
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import tempfile
|
||||
from datetime import datetime, timezone
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from ..log_system.log_funcs import (
|
||||
@@ -22,10 +26,12 @@ METADATA_DIR = os.path.join(
|
||||
)
|
||||
POPULAR_MODELS_FILE = os.path.join(METADATA_DIR, "popular-models.json")
|
||||
MODEL_ALIASES_FILE = os.path.join(METADATA_DIR, "model-aliases.json")
|
||||
BASE_MODELS_FILE = os.path.join(METADATA_DIR, "base-models.json")
|
||||
|
||||
# Cache for loaded data
|
||||
_popular_models_cache: Optional[Dict] = None
|
||||
_model_aliases_cache: Optional[Dict] = None
|
||||
_base_models_aliases_cache: Optional[Dict[str, List[str]]] = None
|
||||
|
||||
|
||||
def _load_popular_models() -> Dict[str, Any]:
|
||||
@@ -68,6 +74,71 @@ def _load_model_aliases() -> Dict[str, List[str]]:
|
||||
return _model_aliases_cache
|
||||
|
||||
|
||||
def _normalize_base_model(value: str) -> str:
|
||||
import re
|
||||
return re.sub(r"[^a-z0-9]+", "", str(value or "").lower())
|
||||
|
||||
|
||||
def load_base_model_aliases() -> Dict[str, List[str]]:
|
||||
"""Load and normalize base model aliases from base-models.json."""
|
||||
global _base_models_aliases_cache
|
||||
|
||||
if _base_models_aliases_cache is not None:
|
||||
return _base_models_aliases_cache
|
||||
|
||||
aliases_dict = {}
|
||||
try:
|
||||
if os.path.exists(BASE_MODELS_FILE):
|
||||
with open(BASE_MODELS_FILE, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
base_models = data.get("base_models", [])
|
||||
for model in base_models:
|
||||
name = model.get("name", "")
|
||||
normalized_name = _normalize_base_model(name)
|
||||
if not normalized_name:
|
||||
continue
|
||||
|
||||
aliases = model.get("aliases", [])
|
||||
# Normalize each alias token and deduplicate
|
||||
normalized_tokens = []
|
||||
seen_tokens = set()
|
||||
for alias in aliases:
|
||||
if not alias:
|
||||
continue
|
||||
normalized_alias = _normalize_base_model(alias)
|
||||
if normalized_alias and normalized_alias not in seen_tokens:
|
||||
seen_tokens.add(normalized_alias)
|
||||
normalized_tokens.append(normalized_alias)
|
||||
|
||||
# Ensure normalized name itself is in preferred tokens if not already
|
||||
if normalized_name not in seen_tokens:
|
||||
normalized_tokens.append(normalized_name)
|
||||
|
||||
aliases_dict[normalized_name] = normalized_tokens
|
||||
except Exception as e:
|
||||
log_error(f"Error loading base model aliases: {e}")
|
||||
|
||||
# Fallback to hardcoded defaults if file is missing/corrupted
|
||||
if not aliases_dict:
|
||||
aliases_dict = {
|
||||
"zimage": ["zimage", "zimageturbo"],
|
||||
"pony": ["pony", "ponyxl"],
|
||||
"illustrious": ["illustrious", "illustriousxl"],
|
||||
"sdxl10": ["sdxl", "sdxl10", "sdxl100", "sdxl1"],
|
||||
"sd15": ["sd15", "sd1", "stablediffusion15"],
|
||||
"flux1d": ["flux", "flux1", "flux1d", "fluxdev"],
|
||||
"flux1s": ["flux1s", "fluxschnell"],
|
||||
"qwenimage": ["qwenimage"],
|
||||
"hunyuan1": ["hunyuan", "hunyuan1"],
|
||||
"wanvideo": ["wan", "wanvideo"],
|
||||
"noobai": ["noobai"],
|
||||
"hidream": ["hidream"],
|
||||
}
|
||||
|
||||
_base_models_aliases_cache = aliases_dict
|
||||
return _base_models_aliases_cache
|
||||
|
||||
|
||||
def get_popular_model_url(filename: str) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Look up a model filename in the popular models database.
|
||||
@@ -136,8 +207,241 @@ def get_all_popular_models() -> Dict[str, Any]:
|
||||
|
||||
def reload_databases():
|
||||
"""Force reload of all databases."""
|
||||
global _popular_models_cache, _model_aliases_cache
|
||||
global _popular_models_cache, _model_aliases_cache, _base_models_aliases_cache
|
||||
_popular_models_cache = None
|
||||
_model_aliases_cache = None
|
||||
_base_models_aliases_cache = None
|
||||
_load_popular_models()
|
||||
_load_model_aliases()
|
||||
load_base_model_aliases()
|
||||
|
||||
|
||||
BASE_MODELS_META_FILE = os.path.join(METADATA_DIR, "base-models.meta.json")
|
||||
|
||||
|
||||
def get_base_models_config() -> Dict[str, Any]:
|
||||
"""Return the raw base-models config (base_models array) for the frontend dropdown."""
|
||||
return _read_base_models_file()
|
||||
|
||||
|
||||
def _read_base_models_file() -> Dict[str, Any]:
|
||||
try:
|
||||
if os.path.exists(BASE_MODELS_FILE):
|
||||
with open(BASE_MODELS_FILE, "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
except Exception as e:
|
||||
log_warn(f"Error reading base-models.json: {e}")
|
||||
return {"base_models": []}
|
||||
|
||||
|
||||
def _read_base_models_meta() -> Dict[str, Any]:
|
||||
try:
|
||||
if os.path.exists(BASE_MODELS_META_FILE):
|
||||
with open(BASE_MODELS_META_FILE, "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
except Exception as e:
|
||||
log_warn(f"Error reading base-models.meta.json: {e}")
|
||||
return {}
|
||||
|
||||
|
||||
def generate_aliases(name: str) -> List[str]:
|
||||
aliases = {name.lower()}
|
||||
|
||||
# Replace dots, dashes, underscores with spaces, then normalize spaces
|
||||
normalized = re.sub(r"[\._\-]+", " ", name.lower())
|
||||
normalized = re.sub(r"\s+", " ", normalized).strip()
|
||||
aliases.add(normalized)
|
||||
|
||||
# Remove all spaces/dashes/dots/underscores
|
||||
collapsed = re.sub(r"[\s\._\-]+", "", name.lower())
|
||||
aliases.add(collapsed)
|
||||
|
||||
# If there are dots/dashes/underscores, add specific clean replacements
|
||||
if "." in name:
|
||||
aliases.add(name.lower().replace(".", ""))
|
||||
aliases.add(name.lower().replace(".", " "))
|
||||
if "-" in name:
|
||||
aliases.add(name.lower().replace("-", ""))
|
||||
aliases.add(name.lower().replace("-", " "))
|
||||
if "_":
|
||||
aliases.add(name.lower().replace("_", ""))
|
||||
aliases.add(name.lower().replace("_", " "))
|
||||
|
||||
# Extract first word if it has multiple words
|
||||
words = name.lower().split()
|
||||
if len(words) > 1:
|
||||
first_word = words[0]
|
||||
if len(first_word) > 2 and first_word not in {"stable", "flux", "pony", "sdxl"}:
|
||||
aliases.add(first_word)
|
||||
|
||||
return sorted(list({a for a in aliases if len(a) > 1}))
|
||||
|
||||
|
||||
def get_base_models_status(check_remote: bool = False) -> Dict[str, Any]:
|
||||
"""Return local base-models metadata and optionally compare with CivitAI."""
|
||||
local_data = _read_base_models_file()
|
||||
base_models = local_data.get("base_models", [])
|
||||
meta = _read_base_models_meta()
|
||||
|
||||
local_count = len(base_models)
|
||||
local_updated_at = meta.get("updated_at") or ""
|
||||
|
||||
status = {
|
||||
"local_count": local_count,
|
||||
"local_updated_at": local_updated_at,
|
||||
"remote_checked_at": meta.get("last_checked_at") or "",
|
||||
"update_available": False,
|
||||
}
|
||||
|
||||
if check_remote:
|
||||
try:
|
||||
import requests
|
||||
# Fetch from CivitAI API
|
||||
response = requests.get("https://civitai.com/api/v1/enums", timeout=15)
|
||||
if response.status_code == 200:
|
||||
enums = response.json()
|
||||
remote_models = enums.get("BaseModel", [])
|
||||
|
||||
# Check if there are new models by comparing normalized names and aliases
|
||||
all_existing_normalized = set()
|
||||
for m in base_models:
|
||||
all_existing_normalized.add(_normalize_base_model(m.get("name", "")))
|
||||
for alias in m.get("aliases", []):
|
||||
all_existing_normalized.add(_normalize_base_model(alias))
|
||||
|
||||
new_models_found = False
|
||||
for remote_name in remote_models:
|
||||
norm_remote = _normalize_base_model(remote_name)
|
||||
if norm_remote and norm_remote not in all_existing_normalized:
|
||||
new_models_found = True
|
||||
break
|
||||
|
||||
status.update({
|
||||
"remote_checked_at": datetime.now(timezone.utc).replace(microsecond=0).isoformat(),
|
||||
"update_available": new_models_found
|
||||
})
|
||||
except Exception as e:
|
||||
log_warn(f"Error checking remote base models: {e}")
|
||||
|
||||
return status
|
||||
|
||||
|
||||
def update_base_models_from_remote() -> Dict[str, Any]:
|
||||
"""Fetch live BaseModel enums from CivitAI and merge into base-models.json."""
|
||||
import requests
|
||||
response = requests.get("https://civitai.com/api/v1/enums", timeout=15)
|
||||
if response.status_code != 200:
|
||||
raise ValueError(f"CivitAI enums endpoint returned HTTP {response.status_code}")
|
||||
|
||||
enums = response.json()
|
||||
remote_names = enums.get("BaseModel", [])
|
||||
if not remote_names or not isinstance(remote_names, list):
|
||||
raise ValueError("CivitAI enums did not return a valid BaseModel list")
|
||||
|
||||
local_data = _read_base_models_file()
|
||||
base_models = local_data.get("base_models", [])
|
||||
|
||||
# 1. Build a set of all normalized aliases currently in base-models.json
|
||||
all_known_normalized = set()
|
||||
for m in base_models:
|
||||
for alias in m.get("aliases", []):
|
||||
all_known_normalized.add(_normalize_base_model(alias))
|
||||
all_known_normalized.add(_normalize_base_model(m.get("name", "")))
|
||||
|
||||
updated_models = list(base_models)
|
||||
new_added_count = 0
|
||||
|
||||
for name in remote_names:
|
||||
norm_name = _normalize_base_model(name)
|
||||
if not norm_name:
|
||||
continue
|
||||
|
||||
# If the normalized name is already a known alias or name, skip it
|
||||
if norm_name in all_known_normalized:
|
||||
continue
|
||||
|
||||
# Generate candidate aliases
|
||||
candidates = generate_aliases(name)
|
||||
|
||||
# Filter out aliases that are already registered/known
|
||||
filtered_aliases = []
|
||||
for alias in candidates:
|
||||
norm_alias = _normalize_base_model(alias)
|
||||
if norm_alias not in all_known_normalized:
|
||||
filtered_aliases.append(alias)
|
||||
all_known_normalized.add(norm_alias)
|
||||
|
||||
# If we have valid aliases, add the new base model entry
|
||||
if filtered_aliases:
|
||||
if name.lower() not in [a.lower() for a in filtered_aliases]:
|
||||
filtered_aliases.insert(0, name)
|
||||
all_known_normalized.add(norm_name)
|
||||
|
||||
updated_models.append({
|
||||
"name": name,
|
||||
"aliases": filtered_aliases
|
||||
})
|
||||
new_added_count += 1
|
||||
|
||||
# Write updated models to base-models.json atomically
|
||||
os.makedirs(METADATA_DIR, exist_ok=True)
|
||||
|
||||
# Backup existing
|
||||
if os.path.exists(BASE_MODELS_FILE):
|
||||
shutil.copy2(BASE_MODELS_FILE, f"{BASE_MODELS_FILE}.bak")
|
||||
|
||||
# Write atomically
|
||||
fd, tmp_path = tempfile.mkstemp(
|
||||
prefix=f".{os.path.basename(BASE_MODELS_FILE)}.",
|
||||
suffix=".tmp",
|
||||
dir=os.path.dirname(BASE_MODELS_FILE),
|
||||
text=True,
|
||||
)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
||||
json.dump({"base_models": updated_models}, f, indent=2, ensure_ascii=False)
|
||||
f.write("\n")
|
||||
os.replace(tmp_path, BASE_MODELS_FILE)
|
||||
except Exception:
|
||||
try:
|
||||
os.remove(tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
# Write meta file
|
||||
now_str = datetime.now(timezone.utc).replace(microsecond=0).isoformat()
|
||||
meta = {
|
||||
"updated_at": now_str,
|
||||
"last_checked_at": now_str,
|
||||
"local_count": len(updated_models),
|
||||
"new_models_added": new_added_count
|
||||
}
|
||||
|
||||
fd_meta, tmp_meta_path = tempfile.mkstemp(
|
||||
prefix=f".{os.path.basename(BASE_MODELS_META_FILE)}.",
|
||||
suffix=".tmp",
|
||||
dir=os.path.dirname(BASE_MODELS_META_FILE),
|
||||
text=True,
|
||||
)
|
||||
try:
|
||||
with os.fdopen(fd_meta, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f, indent=2, ensure_ascii=False)
|
||||
f.write("\n")
|
||||
os.replace(tmp_meta_path, BASE_MODELS_META_FILE)
|
||||
except Exception:
|
||||
try:
|
||||
os.remove(tmp_meta_path)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
# Force reload database in-memory cache
|
||||
reload_databases()
|
||||
|
||||
return {
|
||||
"local_count": len(updated_models),
|
||||
"local_updated_at": now_str,
|
||||
"new_models_added": new_added_count,
|
||||
"updated": True
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user