refactor: unify model data contracts
This commit is contained in:
+2309
File diff suppressed because it is too large
Load Diff
+101
-7
@@ -1,7 +1,30 @@
|
||||
"""Shared contract for backend custom-node model adapters."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable, Dict, Optional, Tuple
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Callable, Dict, List, Optional, Protocol, Tuple
|
||||
|
||||
from ..contracts import CustomNodeMetadata, ModelReference, ResolvedModel
|
||||
|
||||
|
||||
class AnalyzeReferencesCallback(Protocol):
|
||||
"""Typed callback used to extract references from a serialized node."""
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
node: Dict[str, Any],
|
||||
available_models: Optional[List[ResolvedModel]] = None,
|
||||
*,
|
||||
is_active: bool,
|
||||
get_widget_name_hint: Callable[[Dict[str, Any], int], str],
|
||||
) -> Optional[List[ModelReference]]: ...
|
||||
|
||||
|
||||
class HasPotentialReferenceCallback(Protocol):
|
||||
"""Typed callback used for cheap custom-node reference detection."""
|
||||
|
||||
def __call__(self, node: Dict[str, Any]) -> bool: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -11,9 +34,80 @@ class CustomNodeModelAdapter:
|
||||
adapter_id: str
|
||||
node_types: Tuple[str, ...]
|
||||
category_hint: Optional[str] = None
|
||||
widget_categories: Dict[int, str] = field(default_factory=dict)
|
||||
analyze_references: Optional[Callable[..., Any]] = None
|
||||
has_potential_reference: Optional[Callable[..., bool]] = None
|
||||
update_model_path: Optional[Callable[..., Optional[bool]]] = None
|
||||
should_skip_existing: Optional[Callable[..., bool]] = None
|
||||
adapt_loaded_model: Optional[Callable[..., Any]] = None
|
||||
widget_categories: Mapping[int, str] = field(default_factory=dict)
|
||||
analyze_references: Optional[AnalyzeReferencesCallback] = None
|
||||
has_potential_reference: Optional[HasPotentialReferenceCallback] = None
|
||||
update_model_path: Optional[
|
||||
Callable[
|
||||
[Dict[str, Any], int, Optional[ResolvedModel], CustomNodeMetadata],
|
||||
Optional[bool],
|
||||
]
|
||||
] = None
|
||||
should_skip_existing: Optional[Callable[[ModelReference], bool]] = None
|
||||
adapt_loaded_model: Optional[
|
||||
Callable[[ModelReference, str, Any], Tuple[str, Any]]
|
||||
] = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not isinstance(self.adapter_id, str):
|
||||
raise TypeError("CustomNodeModelAdapter adapter_id must be a string")
|
||||
adapter_id = self.adapter_id.strip()
|
||||
if not adapter_id:
|
||||
raise ValueError("CustomNodeModelAdapter adapter_id is required")
|
||||
object.__setattr__(self, "adapter_id", adapter_id)
|
||||
|
||||
if not isinstance(self.node_types, (list, tuple)):
|
||||
raise TypeError("CustomNodeModelAdapter node_types must be an array")
|
||||
if any(not isinstance(node_type, str) for node_type in self.node_types):
|
||||
raise TypeError(
|
||||
"CustomNodeModelAdapter node_types must contain only strings"
|
||||
)
|
||||
normalized_node_types = tuple(
|
||||
node_type.strip() for node_type in self.node_types
|
||||
)
|
||||
if any(not node_type for node_type in normalized_node_types):
|
||||
raise ValueError(
|
||||
"CustomNodeModelAdapter node_types cannot contain empty strings"
|
||||
)
|
||||
object.__setattr__(self, "node_types", normalized_node_types)
|
||||
|
||||
if self.category_hint is not None:
|
||||
if not isinstance(self.category_hint, str):
|
||||
raise TypeError(
|
||||
"CustomNodeModelAdapter category_hint must be a string"
|
||||
)
|
||||
object.__setattr__(self, "category_hint", self.category_hint.strip())
|
||||
|
||||
if not isinstance(self.widget_categories, Mapping):
|
||||
raise TypeError(
|
||||
"CustomNodeModelAdapter widget_categories must be an object"
|
||||
)
|
||||
normalized_widget_categories = {}
|
||||
for widget_index, category in self.widget_categories.items():
|
||||
if isinstance(widget_index, bool) or not isinstance(widget_index, int):
|
||||
raise TypeError(
|
||||
"CustomNodeModelAdapter widget category keys must be integers"
|
||||
)
|
||||
if not isinstance(category, str):
|
||||
raise TypeError(
|
||||
"CustomNodeModelAdapter widget category values must be strings"
|
||||
)
|
||||
normalized_widget_categories[widget_index] = category.strip()
|
||||
object.__setattr__(
|
||||
self,
|
||||
"widget_categories",
|
||||
MappingProxyType(normalized_widget_categories),
|
||||
)
|
||||
|
||||
for field_name in (
|
||||
"analyze_references",
|
||||
"has_potential_reference",
|
||||
"update_model_path",
|
||||
"should_skip_existing",
|
||||
"adapt_loaded_model",
|
||||
):
|
||||
callback = getattr(self, field_name)
|
||||
if callback is not None and not callable(callback):
|
||||
raise TypeError(
|
||||
f"CustomNodeModelAdapter {field_name} must be callable"
|
||||
)
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
"""Backend adapter for ComfyUI-Lora-Manager loader nodes."""
|
||||
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
from ..contracts import CustomNodeMetadata, ModelReference, ResolvedModel
|
||||
from ..log_system import create_module_logger
|
||||
from ..type_utils import to_bool
|
||||
from .base import CustomNodeModelAdapter
|
||||
|
||||
log = create_module_logger(__name__)
|
||||
@@ -20,11 +22,11 @@ TEXT_WIDGET_INDEX = 1
|
||||
|
||||
def analyze_references(
|
||||
node: Dict[str, Any],
|
||||
available_models: Optional[List[Dict[str, Any]]] = None,
|
||||
available_models: Optional[List[ResolvedModel]] = None,
|
||||
*,
|
||||
is_active: bool,
|
||||
get_widget_name_hint: Any,
|
||||
) -> Optional[List[Dict[str, Any]]]:
|
||||
get_widget_name_hint: Callable[[Dict[str, Any], int], str],
|
||||
) -> Optional[List[ModelReference]]:
|
||||
"""Extract LoRA references stored in Lora Manager's list widget."""
|
||||
widgets_values = node.get("widgets_values", [])
|
||||
if len(widgets_values) < 3:
|
||||
@@ -36,11 +38,11 @@ def analyze_references(
|
||||
available_models if available_models is not None else get_model_files()
|
||||
)
|
||||
lora_files = [
|
||||
model for model in all_loras if model.get("category") == "loras"
|
||||
model for model in all_loras if model.category == "loras"
|
||||
]
|
||||
lora_lookup: Dict[str, List[Dict[str, Any]]] = {}
|
||||
lora_lookup: Dict[str, List[ResolvedModel]] = {}
|
||||
for lora_file in lora_files:
|
||||
filename = lora_file.get("filename", "")
|
||||
filename = lora_file.filename
|
||||
if not filename:
|
||||
continue
|
||||
base_name = os.path.splitext(filename)[0]
|
||||
@@ -53,7 +55,7 @@ def analyze_references(
|
||||
node_id = node.get("id")
|
||||
node_type = node.get("type", "")
|
||||
node_title = str(node.get("title", "") or "").strip()
|
||||
model_refs: List[Dict[str, Any]] = []
|
||||
model_refs: List[ModelReference] = []
|
||||
for lora_item in lora_list:
|
||||
if not isinstance(lora_item, dict):
|
||||
continue
|
||||
@@ -65,7 +67,7 @@ def analyze_references(
|
||||
lora_exists = False
|
||||
lora_full_path = None
|
||||
if name in lora_lookup:
|
||||
lora_full_path = lora_lookup[name][0].get("path")
|
||||
lora_full_path = lora_lookup[name][0].path
|
||||
lora_exists = (
|
||||
os.path.exists(lora_full_path) if lora_full_path else False
|
||||
)
|
||||
@@ -74,7 +76,7 @@ def analyze_references(
|
||||
test_name = name + extension
|
||||
if test_name not in lora_lookup:
|
||||
continue
|
||||
lora_full_path = lora_lookup[test_name][0].get("path")
|
||||
lora_full_path = lora_lookup[test_name][0].path
|
||||
lora_exists = (
|
||||
os.path.exists(lora_full_path) if lora_full_path else False
|
||||
)
|
||||
@@ -83,27 +85,29 @@ def analyze_references(
|
||||
|
||||
log.debug(f"Lora {name}: exists={lora_exists}, path={lora_full_path}")
|
||||
model_refs.append(
|
||||
{
|
||||
"node_id": node_id,
|
||||
"node_type": node_type,
|
||||
"widget_index": LORA_LIST_WIDGET_INDEX,
|
||||
"widget_name": get_widget_name_hint(
|
||||
node, LORA_LIST_WIDGET_INDEX
|
||||
),
|
||||
"original_path": name,
|
||||
"name": name,
|
||||
"strength": float(lora_item.get("strength", 1.0)),
|
||||
"active": lora_item.get("active", True),
|
||||
"node_title": node_title,
|
||||
"category": "loras",
|
||||
"category_hints": ["loras"],
|
||||
"folder_key_hints": ["loras"],
|
||||
"full_path": lora_full_path,
|
||||
"exists": lora_exists,
|
||||
"is_urn": False,
|
||||
"custom_node_adapter": ADAPTER_ID,
|
||||
"connected": is_active,
|
||||
}
|
||||
ModelReference.from_mapping(
|
||||
{
|
||||
"node_id": node_id,
|
||||
"node_type": node_type,
|
||||
"widget_index": LORA_LIST_WIDGET_INDEX,
|
||||
"widget_name": get_widget_name_hint(
|
||||
node, LORA_LIST_WIDGET_INDEX
|
||||
),
|
||||
"original_path": name,
|
||||
"name": name,
|
||||
"strength": float(lora_item.get("strength", 1.0)),
|
||||
"active": to_bool(lora_item.get("active", True), True),
|
||||
"node_title": node_title,
|
||||
"category": "loras",
|
||||
"category_hints": ["loras"],
|
||||
"folder_key_hints": ["loras"],
|
||||
"full_path": lora_full_path,
|
||||
"exists": lora_exists,
|
||||
"is_urn": False,
|
||||
"custom_node_adapter": ADAPTER_ID,
|
||||
"connected": bool(is_active),
|
||||
}
|
||||
)
|
||||
)
|
||||
return model_refs
|
||||
|
||||
@@ -125,20 +129,16 @@ def has_potential_reference(node: Dict[str, Any]) -> bool:
|
||||
def update_model_path(
|
||||
node: Dict[str, Any],
|
||||
widget_index: int,
|
||||
resolved_model: Optional[Dict[str, Any]],
|
||||
mapping: Optional[Dict[str, Any]],
|
||||
resolved_model: Optional[ResolvedModel],
|
||||
metadata: CustomNodeMetadata,
|
||||
) -> Optional[bool]:
|
||||
"""Update one LoRA name in both the list and formatted text widgets."""
|
||||
mapping = mapping or {}
|
||||
adapter_id = mapping.get("custom_node_adapter")
|
||||
is_legacy_mapping = mapping.get("is_lora_v2") is True
|
||||
adapter_id = metadata.adapter_id
|
||||
is_legacy_mapping = metadata.is_legacy_lora_manager
|
||||
if adapter_id != ADAPTER_ID and not is_legacy_mapping:
|
||||
return None
|
||||
|
||||
original_name = (
|
||||
mapping.get("custom_node_original_identity")
|
||||
or mapping.get("original_lora_name")
|
||||
)
|
||||
original_name = metadata.original_identity
|
||||
if not original_name or widget_index != LORA_LIST_WIDGET_INDEX:
|
||||
return None
|
||||
|
||||
@@ -153,9 +153,7 @@ def update_model_path(
|
||||
|
||||
new_name = None
|
||||
if resolved_model:
|
||||
new_name = resolved_model.get("filename") or resolved_model.get(
|
||||
"name", ""
|
||||
)
|
||||
new_name = resolved_model.filename
|
||||
if new_name and "." in new_name:
|
||||
new_name = new_name.rsplit(".", 1)[0]
|
||||
if not new_name:
|
||||
@@ -204,20 +202,20 @@ def update_model_path(
|
||||
return True
|
||||
|
||||
|
||||
def should_skip_existing(reference: Dict[str, Any]) -> bool:
|
||||
def should_skip_existing(reference: ModelReference) -> bool:
|
||||
"""Existing list entries do not require matching or relinking."""
|
||||
return reference.get("exists") is True
|
||||
return reference.exists is True
|
||||
|
||||
|
||||
def adapt_loaded_model(
|
||||
reference: Dict[str, Any],
|
||||
reference: ModelReference,
|
||||
model_name: str,
|
||||
strength: Any,
|
||||
) -> tuple[str, Any]:
|
||||
"""Use the list entry's display name and strength."""
|
||||
return (
|
||||
reference.get("name", model_name),
|
||||
reference.get("strength", strength),
|
||||
reference.extra_value("name", model_name),
|
||||
reference.extra_value("strength", strength),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1,7 +1,12 @@
|
||||
"""Registry and dispatch helpers for backend custom-node model adapters."""
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
from ..contracts import (
|
||||
CustomNodeMetadata,
|
||||
ModelReference,
|
||||
ResolvedModel,
|
||||
)
|
||||
from .base import CustomNodeModelAdapter
|
||||
from .lora_manager import ADAPTER as LORA_MANAGER_ADAPTER
|
||||
from .rgthree_power_lora_loader import (
|
||||
@@ -35,16 +40,18 @@ def get_custom_node_model_adapter(
|
||||
|
||||
|
||||
def get_custom_node_adapter_for_reference(
|
||||
reference: Dict[str, Any],
|
||||
reference: Optional[ModelReference],
|
||||
) -> Optional[CustomNodeModelAdapter]:
|
||||
"""Resolve an adapter from normalized or legacy reference metadata."""
|
||||
adapter_id = reference.get("custom_node_adapter")
|
||||
if reference is None:
|
||||
return None
|
||||
adapter_id = reference.extra_value("custom_node_adapter")
|
||||
if adapter_id in _ADAPTERS_BY_ID:
|
||||
return _ADAPTERS_BY_ID[adapter_id]
|
||||
adapter = get_custom_node_model_adapter(reference.get("node_type"))
|
||||
adapter = get_custom_node_model_adapter(reference.node_type)
|
||||
if adapter:
|
||||
return adapter
|
||||
if reference.get("is_lora_v2"):
|
||||
if CustomNodeMetadata.from_mapping(reference.extra).is_legacy_lora_manager:
|
||||
return LORA_MANAGER_ADAPTER
|
||||
return None
|
||||
|
||||
@@ -70,38 +77,55 @@ def get_custom_node_widget_categories() -> Dict[str, Dict[int, str]]:
|
||||
|
||||
|
||||
def get_custom_node_resolution_metadata(
|
||||
reference: Dict[str, Any],
|
||||
) -> Dict[str, Any]:
|
||||
"""Normalize adapter metadata carried into workflow update mappings."""
|
||||
reference: Optional[ModelReference],
|
||||
) -> CustomNodeMetadata:
|
||||
"""Normalize adapter metadata carried into workflow update resolutions."""
|
||||
adapter = get_custom_node_adapter_for_reference(reference)
|
||||
if not adapter:
|
||||
return {}
|
||||
original_identity = (
|
||||
reference.get("custom_node_original_identity")
|
||||
or reference.get("original_lora_name")
|
||||
or reference.get("name")
|
||||
or reference.get("original_path")
|
||||
)
|
||||
return {
|
||||
if not adapter or reference is None:
|
||||
return CustomNodeMetadata()
|
||||
metadata = {
|
||||
"custom_node_adapter": adapter.adapter_id,
|
||||
"custom_node_original_identity": original_identity,
|
||||
"custom_node_original_identity": (
|
||||
reference.extra_value("custom_node_original_identity")
|
||||
or reference.extra_value("original_lora_name")
|
||||
or reference.extra_value("name")
|
||||
or reference.original_path
|
||||
),
|
||||
}
|
||||
legacy_v2_flag = reference.extra_value("is_lora_v2")
|
||||
if legacy_v2_flag is not None:
|
||||
metadata["is_lora_v2"] = legacy_v2_flag
|
||||
legacy_flag = reference.extra_value("is_legacy_lora_manager")
|
||||
if legacy_flag is not None:
|
||||
metadata["is_legacy_lora_manager"] = legacy_flag
|
||||
return CustomNodeMetadata.from_mapping(metadata)
|
||||
|
||||
|
||||
def analyze_custom_node_references(
|
||||
node: Dict[str, Any],
|
||||
available_models: Optional[List[Dict[str, Any]]] = None,
|
||||
**context: Any,
|
||||
) -> Optional[List[Dict[str, Any]]]:
|
||||
available_models: Optional[List[ResolvedModel]] = None,
|
||||
*,
|
||||
is_active: bool,
|
||||
get_widget_name_hint: Callable[[Dict[str, Any], int], str],
|
||||
) -> Optional[List[ModelReference]]:
|
||||
"""Dispatch custom serialized model extraction when an adapter handles it."""
|
||||
adapter = get_custom_node_model_adapter(node)
|
||||
if not adapter or not adapter.analyze_references:
|
||||
return None
|
||||
return adapter.analyze_references(
|
||||
raw_references = adapter.analyze_references(
|
||||
node,
|
||||
available_models,
|
||||
**context,
|
||||
is_active=is_active,
|
||||
get_widget_name_hint=get_widget_name_hint,
|
||||
)
|
||||
if raw_references is None:
|
||||
return None
|
||||
return [
|
||||
reference
|
||||
if isinstance(reference, ModelReference)
|
||||
else ModelReference.from_mapping(reference)
|
||||
for reference in raw_references
|
||||
]
|
||||
|
||||
|
||||
def custom_node_has_potential_model_reference(
|
||||
@@ -119,8 +143,8 @@ def custom_node_has_potential_model_reference(
|
||||
def update_custom_node_model_path(
|
||||
node: Dict[str, Any],
|
||||
widget_index: int,
|
||||
resolved_model: Optional[Dict[str, Any]],
|
||||
mapping: Optional[Dict[str, Any]],
|
||||
resolved_model: Optional[ResolvedModel],
|
||||
metadata: CustomNodeMetadata,
|
||||
) -> Optional[bool]:
|
||||
"""Dispatch a custom workflow update, returning None when not handled."""
|
||||
adapter = get_custom_node_model_adapter(node)
|
||||
@@ -130,12 +154,12 @@ def update_custom_node_model_path(
|
||||
node,
|
||||
widget_index,
|
||||
resolved_model,
|
||||
mapping,
|
||||
metadata,
|
||||
)
|
||||
|
||||
|
||||
def should_skip_existing_custom_node_reference(
|
||||
reference: Dict[str, Any],
|
||||
reference: ModelReference,
|
||||
) -> bool:
|
||||
"""Return whether an existing adapter reference needs no resolution."""
|
||||
adapter = get_custom_node_adapter_for_reference(reference)
|
||||
@@ -147,7 +171,7 @@ def should_skip_existing_custom_node_reference(
|
||||
|
||||
|
||||
def adapt_custom_node_loaded_model(
|
||||
reference: Dict[str, Any],
|
||||
reference: ModelReference,
|
||||
model_name: str,
|
||||
strength: Any,
|
||||
) -> tuple[str, Any]:
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import os
|
||||
from typing import Any, List, Optional
|
||||
|
||||
from ..path_utils import normalize_folder_path_values
|
||||
from .dependencies import require_download_dependencies
|
||||
|
||||
|
||||
@@ -90,7 +91,9 @@ def get_download_directory(
|
||||
def _get_folder_paths(folder_name: str) -> List[str]:
|
||||
"""Return paths for one key without rejecting the other aliases."""
|
||||
try:
|
||||
return list(folder_paths.get_folder_paths(folder_name) or [])
|
||||
return normalize_folder_path_values(
|
||||
folder_paths.get_folder_paths(folder_name)
|
||||
)
|
||||
except Exception as exc:
|
||||
facade.log.debug(
|
||||
f"Could not get folder path for {folder_name}: {exc}"
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
"""Shared helpers for enriching local model matches found by SHA-256."""
|
||||
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
from typing import Callable, List, Optional
|
||||
|
||||
SearchLocalMatchesByHash = Callable[..., List[Dict[str, Any]]]
|
||||
from .contracts import ModelMatch
|
||||
|
||||
SearchLocalMatchesByHash = Callable[..., List[ModelMatch]]
|
||||
|
||||
|
||||
def collect_local_hash_matches_for_result(
|
||||
@@ -14,7 +16,7 @@ def collect_local_hash_matches_for_result(
|
||||
force_rescan: bool = False,
|
||||
source: str = "download_source",
|
||||
filename: str = "",
|
||||
) -> List[Dict[str, Any]]:
|
||||
) -> List[ModelMatch]:
|
||||
"""Find local matches and add the shared hash-lookup result fields."""
|
||||
if not sha256:
|
||||
return []
|
||||
@@ -26,11 +28,10 @@ def collect_local_hash_matches_for_result(
|
||||
force_rescan=force_rescan,
|
||||
)
|
||||
return [
|
||||
{
|
||||
**match,
|
||||
"hash_lookup_source": source,
|
||||
"hash_lookup_filename": filename,
|
||||
"hash_lookup_sha256": sha256,
|
||||
}
|
||||
match.with_extra(
|
||||
hash_lookup_source=source,
|
||||
hash_lookup_filename=filename,
|
||||
hash_lookup_sha256=sha256,
|
||||
)
|
||||
for match in matches
|
||||
]
|
||||
|
||||
@@ -186,7 +186,16 @@ class DirectConsoleHandler(logging.Handler):
|
||||
stream = self.stream or _get_console_output_stream()
|
||||
if stream is None:
|
||||
return
|
||||
stream.write(self.format(record) + self.terminator)
|
||||
message = self.format(record) + self.terminator
|
||||
try:
|
||||
stream.write(message)
|
||||
except UnicodeEncodeError:
|
||||
encoding = getattr(stream, "encoding", None) or "utf-8"
|
||||
safe_message = message.encode(
|
||||
encoding,
|
||||
errors="replace",
|
||||
).decode(encoding, errors="replace")
|
||||
stream.write(safe_message)
|
||||
stream.flush()
|
||||
except Exception:
|
||||
self.handleError(record)
|
||||
|
||||
+59
-52
@@ -8,9 +8,11 @@ import heapq
|
||||
import os
|
||||
import re
|
||||
from difflib import SequenceMatcher
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
from .contracts import ModelMatch, ResolvedModel, SearchResult
|
||||
from .log_system import create_module_logger
|
||||
from .metadata_model_utils import normalize_models
|
||||
|
||||
log = create_module_logger(__name__)
|
||||
|
||||
@@ -92,10 +94,10 @@ def calculate_similarity_with_normalization(str1: str, str2: str) -> float:
|
||||
|
||||
def find_matches(
|
||||
target_model: str,
|
||||
candidate_models: List[Dict[str, str]],
|
||||
candidate_models: List[ResolvedModel],
|
||||
threshold: float = 0.0,
|
||||
max_results: int = 10,
|
||||
) -> List[Dict[str, any]]:
|
||||
) -> List[ModelMatch]:
|
||||
"""
|
||||
Find similar models using fuzzy matching.
|
||||
|
||||
@@ -106,13 +108,7 @@ def find_matches(
|
||||
max_results: Maximum number of results to return
|
||||
|
||||
Returns:
|
||||
List of match dictionaries sorted by similarity (highest first):
|
||||
{
|
||||
'model': original model dict from candidates,
|
||||
'filename': model filename,
|
||||
'similarity': similarity score (0.0 to 1.0),
|
||||
'confidence': confidence percentage (0 to 100)
|
||||
}
|
||||
Typed local matches sorted by similarity (highest first).
|
||||
"""
|
||||
# Keep only the best N matches instead of collecting and sorting everything.
|
||||
best_matches = []
|
||||
@@ -129,10 +125,12 @@ def find_matches(
|
||||
# Normalize target filename once for exact match comparisons
|
||||
target_norm = normalize_filename(target_filename)
|
||||
|
||||
for candidate in candidate_models:
|
||||
normalized_candidates = normalize_models(candidate_models)
|
||||
|
||||
for candidate in normalized_candidates:
|
||||
# Get filename from candidate (prefer 'filename' key, fallback to extracting from 'path' or 'relative_path')
|
||||
candidate_filename = candidate.get("filename")
|
||||
candidate_path = candidate.get("path", "") or candidate.get("relative_path", "")
|
||||
candidate_filename = candidate.filename
|
||||
candidate_path = candidate.path or candidate.relative_path
|
||||
|
||||
# If no filename key, try to extract from path or relative_path
|
||||
if not candidate_filename and candidate_path:
|
||||
@@ -143,20 +141,16 @@ def find_matches(
|
||||
|
||||
# Normalize candidate path separators based on current OS
|
||||
# This ensures paths with \ vs / separators are treated as identical
|
||||
candidate_path_normalized = candidate.get("_match_path_norm")
|
||||
if candidate_path_normalized is None:
|
||||
candidate_path_normalized = (
|
||||
os.path.normpath(candidate_path) if candidate_path else ""
|
||||
)
|
||||
candidate["_match_path_norm"] = candidate_path_normalized
|
||||
candidate_path_normalized = (
|
||||
os.path.normpath(candidate_path) if candidate_path else ""
|
||||
)
|
||||
|
||||
candidate_relative_path = candidate.get("relative_path", "")
|
||||
candidate_relative_path_normalized = candidate.get("_match_relative_path_norm")
|
||||
if candidate_relative_path_normalized is None:
|
||||
candidate_relative_path_normalized = (
|
||||
os.path.normpath(candidate_relative_path) if candidate_relative_path else ""
|
||||
)
|
||||
candidate["_match_relative_path_norm"] = candidate_relative_path_normalized
|
||||
candidate_relative_path = candidate.relative_path
|
||||
candidate_relative_path_normalized = (
|
||||
os.path.normpath(candidate_relative_path)
|
||||
if candidate_relative_path
|
||||
else ""
|
||||
)
|
||||
|
||||
# Check if normalized paths are identical (100% match)
|
||||
# This handles cases where paths differ only by separator (e.g., path/to/model vs path\to\model)
|
||||
@@ -176,12 +170,14 @@ def find_matches(
|
||||
if path_match:
|
||||
# Exact path match after normalization = 100% confidence
|
||||
similarity = 1.0
|
||||
match = {
|
||||
"model": candidate,
|
||||
"filename": candidate_filename,
|
||||
"similarity": similarity,
|
||||
"confidence": round(similarity * 100, 1),
|
||||
}
|
||||
match = ModelMatch.from_mapping(
|
||||
{
|
||||
"model": candidate,
|
||||
"filename": candidate_filename,
|
||||
"similarity": similarity,
|
||||
"confidence": round(similarity * 100, 1),
|
||||
}
|
||||
)
|
||||
if max_results <= 0:
|
||||
return []
|
||||
entry = (similarity, match_counter, match)
|
||||
@@ -197,10 +193,7 @@ def find_matches(
|
||||
|
||||
# First check for exact match (after normalization) - should be 100%
|
||||
# Only exact matches should get 100% confidence
|
||||
candidate_norm = candidate.get("_match_filename_norm")
|
||||
if candidate_norm is None:
|
||||
candidate_norm = normalize_filename(candidate_filename)
|
||||
candidate["_match_filename_norm"] = candidate_norm
|
||||
candidate_norm = normalize_filename(candidate_filename)
|
||||
|
||||
if target_norm == candidate_norm:
|
||||
similarity = 1.0
|
||||
@@ -211,12 +204,14 @@ def find_matches(
|
||||
|
||||
# Only include if above threshold
|
||||
if similarity >= threshold:
|
||||
match = {
|
||||
"model": candidate,
|
||||
"filename": candidate_filename,
|
||||
"similarity": similarity,
|
||||
"confidence": confidence,
|
||||
}
|
||||
match = ModelMatch.from_mapping(
|
||||
{
|
||||
"model": candidate,
|
||||
"filename": candidate_filename,
|
||||
"similarity": similarity,
|
||||
"confidence": confidence,
|
||||
}
|
||||
)
|
||||
if max_results <= 0:
|
||||
return []
|
||||
entry = (similarity, match_counter, match)
|
||||
@@ -226,7 +221,10 @@ def find_matches(
|
||||
elif similarity > best_matches[0][0]:
|
||||
heapq.heapreplace(best_matches, entry)
|
||||
|
||||
return [match for _, _, match in sorted(best_matches, key=lambda x: x[0], reverse=True)]
|
||||
return [
|
||||
match
|
||||
for _, _, match in sorted(best_matches, key=lambda x: x[0], reverse=True)
|
||||
]
|
||||
|
||||
|
||||
def normalize_base_model(value: str) -> str:
|
||||
@@ -577,14 +575,14 @@ def match_model_by_title_generic(
|
||||
model_id: int,
|
||||
title_query: str,
|
||||
model_name: str,
|
||||
versions: list,
|
||||
versions: list[Dict[str, Any]],
|
||||
base_model_context: Optional[str],
|
||||
get_base_model_fn,
|
||||
select_file_fn,
|
||||
build_result_fn,
|
||||
hydrate_version_fn=None,
|
||||
get_base_model_fn: Callable[[Any], Optional[str]],
|
||||
select_file_fn: Callable[[Any], Optional[Dict[str, Any]]],
|
||||
build_result_fn: Callable[[Any, Any, float], Optional[SearchResult]],
|
||||
hydrate_version_fn: Optional[Callable[[Any], Any]] = None,
|
||||
log_prefix: str = "ModelResolver",
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
) -> Optional[SearchResult]:
|
||||
"""Generic helper to resolve extensionless workflow values by model title across CivitAI and CivArchive."""
|
||||
|
||||
title_confidence = calculate_model_title_confidence(title_query, model_name)
|
||||
@@ -621,11 +619,20 @@ def match_model_by_title_generic(
|
||||
if not result:
|
||||
continue
|
||||
|
||||
result["confidence"] = title_confidence
|
||||
result["title_confidence"] = title_confidence
|
||||
if not isinstance(result, SearchResult):
|
||||
raise TypeError(
|
||||
"Model title matcher builders must return SearchResult"
|
||||
)
|
||||
|
||||
result = result.with_updates(confidence=title_confidence).with_extra(
|
||||
title_confidence=title_confidence
|
||||
)
|
||||
result_version_id = result.version_id
|
||||
result_filename = result.filename
|
||||
result_base_model = result.base_model
|
||||
|
||||
log.info(
|
||||
f"{log_prefix} model-title match: query={title_query}, model_id={model_id}, model_name={model_name}, version_id={result.get('version_id')}, filename={result.get('filename')}, confidence={title_confidence}, base={result.get('base_model')}"
|
||||
f"{log_prefix} model-title match: query={title_query}, model_id={model_id}, model_name={model_name}, version_id={result_version_id}, filename={result_filename}, confidence={title_confidence}, base={result_base_model}"
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
+11
-10
@@ -9,8 +9,9 @@ import os
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from .contracts import ResolvedModel
|
||||
from .log_system import create_module_logger
|
||||
from .metadata_model_utils import dedupe_models, is_model_file_path
|
||||
from .metadata_model_utils import ModelRecord, dedupe_models, is_model_file_path
|
||||
from .metadata_utils import merge_counted_payload
|
||||
from .path_utils import (
|
||||
find_metadata_sidecar_path,
|
||||
@@ -171,7 +172,7 @@ def _get_batch_size(
|
||||
return max(MIN_AUDIT_BATCH_SIZE, min(MAX_AUDIT_BATCH_SIZE, batch_size))
|
||||
|
||||
|
||||
def _make_batches(models: List[Dict[str, Any]], batch_size: int) -> List[List[Dict[str, Any]]]:
|
||||
def _make_batches(models: List[ResolvedModel], batch_size: int) -> List[List[ResolvedModel]]:
|
||||
if not models:
|
||||
return []
|
||||
safe_batch_size = max(1, int(batch_size or 1))
|
||||
@@ -181,9 +182,9 @@ def _make_batches(models: List[Dict[str, Any]], batch_size: int) -> List[List[Di
|
||||
]
|
||||
|
||||
|
||||
def _audit_one_model(model: Dict[str, Any]) -> Dict[str, Any]:
|
||||
def _audit_one_model(model: ResolvedModel) -> Dict[str, Any]:
|
||||
result = _empty_audit_counts()
|
||||
model_path = str(model.get("path") or "").strip()
|
||||
model_path = model.path.strip()
|
||||
if not model_path:
|
||||
return result
|
||||
|
||||
@@ -254,9 +255,9 @@ def _audit_one_model(model: Dict[str, Any]) -> Dict[str, Any]:
|
||||
result["mismatches"].append(
|
||||
{
|
||||
"filename": model_filename,
|
||||
"relative_path": model.get("relative_path") or model_filename,
|
||||
"category": model.get("category") or "",
|
||||
"base_directory": model.get("base_directory") or "",
|
||||
"relative_path": model.relative_path or model_filename,
|
||||
"category": model.category,
|
||||
"base_directory": model.base_directory,
|
||||
"model_path": model_path,
|
||||
"metadata_path": metadata_path,
|
||||
"metadata_size": metadata_size,
|
||||
@@ -272,7 +273,7 @@ def _audit_one_model(model: Dict[str, Any]) -> Dict[str, Any]:
|
||||
return result
|
||||
|
||||
|
||||
def _audit_model_batch(batch: List[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
def _audit_model_batch(batch: List[ResolvedModel]) -> Dict[str, Any]:
|
||||
result = _empty_audit_counts()
|
||||
for model in batch:
|
||||
merge_counted_payload(
|
||||
@@ -285,7 +286,7 @@ def _audit_model_batch(batch: List[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
|
||||
|
||||
def audit_metadata_sizes(
|
||||
models: Optional[List[Dict[str, Any]]] = None,
|
||||
models: Optional[List[ModelRecord]] = None,
|
||||
*,
|
||||
force_rescan: bool = True,
|
||||
worker_count: Optional[int] = None,
|
||||
@@ -299,7 +300,7 @@ def audit_metadata_sizes(
|
||||
|
||||
models = get_model_files(force_rescan=force_rescan)
|
||||
|
||||
audit_models = dedupe_models(models or [])
|
||||
audit_models = dedupe_models(models)
|
||||
total_models = len(audit_models)
|
||||
resolved_worker_count, cpu_count = resolve_worker_count(
|
||||
total_models,
|
||||
|
||||
+33
-30
@@ -12,8 +12,9 @@ from collections.abc import Iterable
|
||||
from concurrent.futures import CancelledError, ThreadPoolExecutor, as_completed
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
from .contracts import ResolvedModel
|
||||
from .log_system import create_module_logger
|
||||
from .metadata_model_utils import dedupe_models, is_model_file_path
|
||||
from .metadata_model_utils import ModelRecord, dedupe_models, is_model_file_path
|
||||
from .metadata_utils import build_sidecar_file_identity, merge_counted_payload
|
||||
from .path_utils import (
|
||||
HashCalculationCancelled,
|
||||
@@ -89,9 +90,11 @@ def get_metadata_build_capabilities() -> Dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def normalize_metadata_build_mode(value: Any) -> str:
|
||||
def normalize_metadata_build_mode(value: Optional[str]) -> str:
|
||||
"""Return a supported metadata builder source mode."""
|
||||
normalized = str(value or "").strip().lower()
|
||||
if value is not None and not isinstance(value, str):
|
||||
raise TypeError("Metadata build mode must be a string")
|
||||
normalized = (value or "").strip().lower()
|
||||
if normalized in METADATA_BUILD_MODES:
|
||||
return normalized
|
||||
return METADATA_BUILD_MODE_CALCULATE_FRESH
|
||||
@@ -106,7 +109,7 @@ def _trusted_external_sha256(metadata: Dict[str, Any]) -> str:
|
||||
|
||||
def _build_external_metadata_seed(
|
||||
model_path: str,
|
||||
model: Dict[str, Any],
|
||||
model: ResolvedModel,
|
||||
) -> Tuple[Dict[str, Any], str, str]:
|
||||
external_path = find_external_metadata_sidecar_path(model_path)
|
||||
if not external_path:
|
||||
@@ -122,7 +125,7 @@ def _build_external_metadata_seed(
|
||||
seed = build_model_resolver_metadata(
|
||||
model_path,
|
||||
external_raw,
|
||||
str(model.get("category") or ""),
|
||||
model.category,
|
||||
)
|
||||
if trusted_sha256:
|
||||
seed["sha256"] = trusted_sha256
|
||||
@@ -238,7 +241,7 @@ def _model_type_for_category(category: str) -> str:
|
||||
def _build_local_metadata_payload(
|
||||
*,
|
||||
existing: Dict[str, Any],
|
||||
model: Dict[str, Any],
|
||||
model: ResolvedModel,
|
||||
model_path: str,
|
||||
metadata_path: str,
|
||||
file_size: int,
|
||||
@@ -249,7 +252,7 @@ def _build_local_metadata_payload(
|
||||
) -> Tuple[Dict[str, Any], List[str]]:
|
||||
filename = get_filename_from_path(model_path)
|
||||
stem = os.path.splitext(filename)[0]
|
||||
category = str(model.get("category") or "")
|
||||
category = model.category
|
||||
now = time.time()
|
||||
payload = dict(existing or {})
|
||||
changed_fields: List[str] = []
|
||||
@@ -460,7 +463,7 @@ def _build_result_payload(
|
||||
|
||||
|
||||
def _build_metadata_history_item(
|
||||
model: Dict[str, Any],
|
||||
model: ResolvedModel,
|
||||
*,
|
||||
action: str,
|
||||
filename: str,
|
||||
@@ -479,8 +482,8 @@ def _build_metadata_history_item(
|
||||
"action": action,
|
||||
"filename": filename,
|
||||
"relative_path": relative_path,
|
||||
"category": model.get("category") or "",
|
||||
"base_directory": model.get("base_directory") or "",
|
||||
"category": model.category,
|
||||
"base_directory": model.base_directory,
|
||||
"model_path": model_path,
|
||||
"metadata_path": metadata_path,
|
||||
"size": file_size,
|
||||
@@ -498,19 +501,19 @@ def _build_metadata_history_item(
|
||||
return item
|
||||
|
||||
|
||||
def _history_items_for_model(model: Dict[str, Any], result: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
def _history_items_for_model(model: ResolvedModel, result: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
updated_items = result.get("updated") or []
|
||||
if updated_items:
|
||||
return [dict(item) for item in updated_items if isinstance(item, dict)]
|
||||
|
||||
filename = get_filename_from_path(str(model.get("path") or "")) or str(model.get("filename") or "Model")
|
||||
model_path = str(model.get("path") or "").strip()
|
||||
filename = get_filename_from_path(model.path) or model.filename or "Model"
|
||||
model_path = model.path.strip()
|
||||
metadata_path = get_model_resolver_sidecar_path(model_path) if model_path else ""
|
||||
base_item = {
|
||||
"filename": filename,
|
||||
"relative_path": model.get("relative_path") or filename,
|
||||
"category": model.get("category") or "",
|
||||
"base_directory": model.get("base_directory") or "",
|
||||
"relative_path": model.relative_path or filename,
|
||||
"category": model.category,
|
||||
"base_directory": model.base_directory,
|
||||
"model_path": model_path,
|
||||
"metadata_path": metadata_path,
|
||||
}
|
||||
@@ -538,7 +541,7 @@ def _history_items_for_model(model: Dict[str, Any], result: Dict[str, Any]) -> L
|
||||
|
||||
|
||||
def _build_missing_local_metadata_parallel(
|
||||
model_items: List[Dict[str, Any]],
|
||||
model_items: List[ResolvedModel],
|
||||
*,
|
||||
worker_count: int,
|
||||
cpu_count: int,
|
||||
@@ -599,10 +602,10 @@ def _build_missing_local_metadata_parallel(
|
||||
snapshot.update(data)
|
||||
_emit(progress_callback, snapshot)
|
||||
|
||||
def run_one_model(model: Dict[str, Any]) -> Dict[str, Any]:
|
||||
model_path = str(model.get("path") or "").strip()
|
||||
def run_one_model(model: ResolvedModel) -> Dict[str, Any]:
|
||||
model_path = model.path.strip()
|
||||
filename = get_filename_from_path(model_path)
|
||||
model_key = get_model_path_identity(model.get("path")) or model_path
|
||||
model_key = get_model_path_identity(model.path) or model_path
|
||||
|
||||
def child_progress(data: Dict[str, Any]) -> None:
|
||||
if not isinstance(data, dict):
|
||||
@@ -653,9 +656,9 @@ def _build_missing_local_metadata_parallel(
|
||||
try:
|
||||
for future in as_completed(futures):
|
||||
model = futures[future]
|
||||
model_path = str(model.get("path") or "").strip()
|
||||
model_path = model.path.strip()
|
||||
filename = get_filename_from_path(model_path)
|
||||
model_key = get_model_path_identity(model.get("path")) or model_path
|
||||
model_key = get_model_path_identity(model.path) or model_path
|
||||
try:
|
||||
item_result = future.result()
|
||||
except CancelledError:
|
||||
@@ -667,7 +670,7 @@ def _build_missing_local_metadata_parallel(
|
||||
"errors": [
|
||||
{
|
||||
"filename": filename,
|
||||
"relative_path": model.get("relative_path") or filename,
|
||||
"relative_path": model.relative_path or filename,
|
||||
"model_path": model_path,
|
||||
"metadata_path": get_model_resolver_sidecar_path(model_path),
|
||||
"message": str(exc) or "Metadata build failed.",
|
||||
@@ -740,7 +743,7 @@ def _build_missing_local_metadata_parallel(
|
||||
|
||||
|
||||
def build_missing_local_metadata(
|
||||
models: Optional[List[Dict[str, Any]]] = None,
|
||||
models: Optional[List[ModelRecord]] = None,
|
||||
*,
|
||||
force_rescan: bool = True,
|
||||
worker_count: Optional[int] = None,
|
||||
@@ -762,10 +765,10 @@ def build_missing_local_metadata(
|
||||
|
||||
models = get_model_files(force_rescan=force_rescan)
|
||||
|
||||
all_models = dedupe_models(models or [])
|
||||
all_models = dedupe_models(models)
|
||||
model_items = [
|
||||
model for model in all_models
|
||||
if is_model_file_path(str(model.get("path") or "").strip())
|
||||
if is_model_file_path(model.path.strip())
|
||||
]
|
||||
total = len(model_items)
|
||||
resolved_worker_count, cpu_count = resolve_worker_count(
|
||||
@@ -861,9 +864,9 @@ def build_missing_local_metadata(
|
||||
include_updated_count=False,
|
||||
)
|
||||
|
||||
model_path = str(model.get("path") or "").strip()
|
||||
model_path = model.path.strip()
|
||||
filename = get_filename_from_path(model_path)
|
||||
relative_path = model.get("relative_path") or filename
|
||||
relative_path = model.relative_path or filename
|
||||
metadata_path, existing_metadata_path, metadata_exists = _select_metadata_path(model_path)
|
||||
scanned_models += 1
|
||||
|
||||
@@ -1154,8 +1157,8 @@ def build_missing_local_metadata(
|
||||
history_item = {
|
||||
**error_item,
|
||||
"action": "error",
|
||||
"category": model.get("category") or "",
|
||||
"base_directory": model.get("base_directory") or "",
|
||||
"category": model.category,
|
||||
"base_directory": model.base_directory,
|
||||
}
|
||||
history.append(history_item)
|
||||
_emit(
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
"""Shared helpers for collections of local model metadata entries."""
|
||||
|
||||
import os
|
||||
from typing import Any, Dict, List
|
||||
from collections.abc import Iterable, Mapping
|
||||
from typing import Any, List
|
||||
|
||||
from .contracts import ResolvedModel
|
||||
from .path_utils import get_filename_from_path, get_model_path_identity
|
||||
from .type_utils import MODEL_EXTENSIONS
|
||||
|
||||
@@ -19,14 +21,36 @@ def is_model_file_path(path: str) -> bool:
|
||||
return os.path.splitext(filename)[1].lower() in MODEL_EXTENSIONS
|
||||
|
||||
|
||||
def dedupe_models(models: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
"""Keep the first valid model entry for every filesystem identity."""
|
||||
result: List[Dict[str, Any]] = []
|
||||
seen = set()
|
||||
for model in models or []:
|
||||
if not isinstance(model, dict):
|
||||
ModelRecord = ResolvedModel | Mapping[str, Any]
|
||||
|
||||
|
||||
def normalize_models(models: Iterable[ModelRecord] | None) -> List[ResolvedModel]:
|
||||
"""Normalize local model inputs once at the metadata boundary."""
|
||||
result: List[ResolvedModel] = []
|
||||
if models is None:
|
||||
return result
|
||||
if isinstance(models, (str, bytes, Mapping)) or not isinstance(models, Iterable):
|
||||
raise TypeError("models must be an iterable of model records")
|
||||
|
||||
for model in models:
|
||||
if isinstance(model, ResolvedModel):
|
||||
result.append(model)
|
||||
continue
|
||||
identity = get_model_path_identity(model.get("path"))
|
||||
if not isinstance(model, Mapping):
|
||||
continue
|
||||
try:
|
||||
result.append(ResolvedModel.from_mapping(model))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
return result
|
||||
|
||||
|
||||
def dedupe_models(models: Iterable[ModelRecord] | None) -> List[ResolvedModel]:
|
||||
"""Keep the first normalized local model for every filesystem identity."""
|
||||
result: List[ResolvedModel] = []
|
||||
seen = set()
|
||||
for model in normalize_models(models):
|
||||
identity = get_model_path_identity(model.path)
|
||||
if not identity or identity in seen:
|
||||
continue
|
||||
seen.add(identity)
|
||||
|
||||
+15
-5
@@ -6,6 +6,7 @@ import re
|
||||
from collections import Counter, defaultdict
|
||||
from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence
|
||||
|
||||
from .contracts import ResolvedModel
|
||||
from .settings import (
|
||||
normalize_download_category,
|
||||
normalize_relative_subfolder,
|
||||
@@ -251,7 +252,7 @@ def _infer_category_template(records: List[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
|
||||
|
||||
def infer_download_path_templates(
|
||||
models: Iterable[Mapping[str, Any]],
|
||||
models: Iterable[ResolvedModel],
|
||||
base_models_config: Any,
|
||||
) -> Dict[str, Any]:
|
||||
"""Return template and base-model mapping suggestions for local model folders."""
|
||||
@@ -261,15 +262,24 @@ def infer_download_path_templates(
|
||||
mapping_votes: Dict[str, Counter] = defaultdict(Counter)
|
||||
seen = set()
|
||||
|
||||
for model in models or []:
|
||||
category = normalize_download_category(str(model.get("category") or ""))
|
||||
if models is None:
|
||||
model_values = ()
|
||||
elif isinstance(models, (str, bytes)):
|
||||
raise TypeError("models must be an iterable of ResolvedModel values")
|
||||
else:
|
||||
model_values = models
|
||||
|
||||
for model in model_values:
|
||||
if not isinstance(model, ResolvedModel):
|
||||
raise TypeError("models must contain ResolvedModel values")
|
||||
category = normalize_download_category(str(model.category or ""))
|
||||
if category not in TEMPLATE_CATEGORIES:
|
||||
continue
|
||||
|
||||
relative_path = str(model.get("relative_path") or model.get("filename") or "")
|
||||
relative_path = str(model.relative_path or model.filename or "")
|
||||
model_key = (
|
||||
category,
|
||||
str(model.get("path") or ""),
|
||||
str(model.path or ""),
|
||||
relative_path,
|
||||
)
|
||||
if model_key in seen:
|
||||
|
||||
+25
-10
@@ -116,14 +116,22 @@ def _get_folder_paths_module(folder_paths_module: Optional[Any] = None) -> Optio
|
||||
return None
|
||||
|
||||
|
||||
def _coerce_folder_paths(value: Any) -> List[str]:
|
||||
def normalize_string_values(value: Any) -> List[str]:
|
||||
"""Normalize an external string collection without iterating scalar text."""
|
||||
if isinstance(value, str):
|
||||
return [value]
|
||||
if not isinstance(value, (list, tuple)):
|
||||
return [value] if value else []
|
||||
if not isinstance(value, (list, tuple, set, frozenset)):
|
||||
return []
|
||||
if value and isinstance(value[0], (list, tuple, set)):
|
||||
return [str(path) for path in value[0] if path]
|
||||
return [str(path) for path in value if isinstance(path, str) and path]
|
||||
return [item for item in value if isinstance(item, str) and item]
|
||||
|
||||
|
||||
def normalize_folder_path_values(value: Any) -> List[str]:
|
||||
"""Normalize ComfyUI folder-path values across supported registry shapes."""
|
||||
if isinstance(value, (list, tuple)) and value:
|
||||
first_item = value[0]
|
||||
if isinstance(first_item, (list, tuple, set, frozenset)):
|
||||
value = first_item
|
||||
return normalize_string_values(value)
|
||||
|
||||
|
||||
def get_configured_model_roots(
|
||||
@@ -160,12 +168,12 @@ def get_configured_model_roots(
|
||||
try:
|
||||
get_folder_paths = getattr(fp, "get_folder_paths", None)
|
||||
if callable(get_folder_paths):
|
||||
paths = list(get_folder_paths(name) or [])
|
||||
paths = normalize_folder_path_values(get_folder_paths(name))
|
||||
except Exception:
|
||||
paths = []
|
||||
|
||||
if not paths and isinstance(folder_names_and_paths, dict):
|
||||
paths = _coerce_folder_paths(folder_names_and_paths.get(name))
|
||||
paths = normalize_folder_path_values(folder_names_and_paths.get(name))
|
||||
|
||||
for path in paths:
|
||||
if not path:
|
||||
@@ -229,14 +237,21 @@ def prefer_local_base_directory(
|
||||
|
||||
|
||||
def dedupe_local_base_directories(
|
||||
paths: list,
|
||||
paths: Optional[list | tuple] = None,
|
||||
preferred_directory: str = "",
|
||||
comfy_root: str = "",
|
||||
) -> list:
|
||||
"""Deduplicate a list of directory paths, resolving symlinks/junctions."""
|
||||
if paths is None:
|
||||
path_values = ()
|
||||
elif isinstance(paths, (list, tuple)):
|
||||
path_values = paths
|
||||
else:
|
||||
raise TypeError("paths must be a list or tuple")
|
||||
|
||||
by_identity = {}
|
||||
ordered_identities = []
|
||||
for path in paths or []:
|
||||
for path in path_values:
|
||||
if not path or not os.path.isdir(path):
|
||||
continue
|
||||
path_abs = get_path_abs(path)
|
||||
|
||||
@@ -15,6 +15,10 @@ def extract_request_sha256(
|
||||
value = ""
|
||||
for key in keys:
|
||||
value = payload.get(key)
|
||||
if value is None or value == "":
|
||||
continue
|
||||
if not isinstance(value, str):
|
||||
raise TypeError(f"Request {key} must be a string")
|
||||
if value:
|
||||
break
|
||||
return normalize_sha256(value)
|
||||
@@ -31,6 +35,200 @@ async def read_optional_object_payload(request: Any) -> dict[str, Any]:
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def read_text_field(
|
||||
payload: Mapping[str, Any],
|
||||
field_name: str,
|
||||
*,
|
||||
default: str = "",
|
||||
contract_name: str = "Request",
|
||||
) -> str:
|
||||
"""Read a request text field without silently stringifying invalid values."""
|
||||
value = payload.get(field_name, default)
|
||||
if value is None:
|
||||
return default
|
||||
if not isinstance(value, str):
|
||||
raise TypeError(f"{contract_name} {field_name} must be a string")
|
||||
return value.strip()
|
||||
|
||||
|
||||
def read_optional_text_field(
|
||||
payload: Mapping[str, Any],
|
||||
field_name: str,
|
||||
*,
|
||||
default: str | None = None,
|
||||
contract_name: str = "Request",
|
||||
) -> str | None:
|
||||
"""Read an optional request text field without stringifying bad values."""
|
||||
value = payload.get(field_name, default)
|
||||
if value is None:
|
||||
return default
|
||||
if not isinstance(value, str):
|
||||
raise TypeError(f"{contract_name} {field_name} must be a string")
|
||||
return value.strip() or default
|
||||
|
||||
|
||||
def read_first_text_field(
|
||||
payload: Mapping[str, Any],
|
||||
field_names: Tuple[str, ...],
|
||||
*,
|
||||
default: str = "",
|
||||
contract_name: str = "Request",
|
||||
) -> str:
|
||||
"""Read the first non-empty text alias without bypassing invalid values."""
|
||||
for field_name in field_names:
|
||||
if field_name not in payload:
|
||||
continue
|
||||
value = payload[field_name]
|
||||
if value is None:
|
||||
continue
|
||||
if not isinstance(value, str):
|
||||
raise TypeError(f"{contract_name} {field_name} must be a string")
|
||||
value = value.strip()
|
||||
if value:
|
||||
return value
|
||||
return default
|
||||
|
||||
|
||||
def read_first_identifier_field(
|
||||
payload: Mapping[str, Any],
|
||||
field_names: Tuple[str, ...],
|
||||
*,
|
||||
default: int | str | None = None,
|
||||
contract_name: str = "Request",
|
||||
) -> int | str | None:
|
||||
"""Read the first non-empty integer/string identifier alias."""
|
||||
for field_name in field_names:
|
||||
if field_name not in payload:
|
||||
continue
|
||||
value = payload[field_name]
|
||||
if value is None or value == "":
|
||||
continue
|
||||
if isinstance(value, bool) or not isinstance(value, (int, str)):
|
||||
raise TypeError(
|
||||
f"{contract_name} {field_name} must be an integer or string"
|
||||
)
|
||||
if isinstance(value, str):
|
||||
value = value.strip()
|
||||
if not value:
|
||||
continue
|
||||
return value
|
||||
return default
|
||||
|
||||
|
||||
def read_identifier_field(
|
||||
payload: Mapping[str, Any],
|
||||
field_name: str,
|
||||
*,
|
||||
default: int | str | None = None,
|
||||
contract_name: str = "Request",
|
||||
) -> int | str | None:
|
||||
"""Read an optional integer/string identifier without accepting booleans."""
|
||||
value = payload.get(field_name, default)
|
||||
if value is None:
|
||||
return default
|
||||
if isinstance(value, bool) or not isinstance(value, (int, str)):
|
||||
raise TypeError(
|
||||
f"{contract_name} {field_name} must be an integer or string"
|
||||
)
|
||||
if isinstance(value, str):
|
||||
return value.strip() or default
|
||||
return value
|
||||
|
||||
|
||||
def coerce_integer_identifier(
|
||||
value: int | str | None,
|
||||
field_name: str,
|
||||
*,
|
||||
contract_name: str = "Request",
|
||||
) -> int | None:
|
||||
"""Convert a validated identifier to an integer without hiding bad values."""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, bool) or not isinstance(value, (int, str)):
|
||||
raise TypeError(f"{contract_name} {field_name} must be an integer")
|
||||
if isinstance(value, int):
|
||||
return value
|
||||
text = value.strip()
|
||||
if not text:
|
||||
return None
|
||||
try:
|
||||
return int(text)
|
||||
except ValueError as exc:
|
||||
raise ValueError(
|
||||
f"{contract_name} {field_name} must be an integer"
|
||||
) from exc
|
||||
|
||||
|
||||
def read_bool_field(
|
||||
payload: Mapping[str, Any],
|
||||
field_name: str,
|
||||
*,
|
||||
default: bool = False,
|
||||
contract_name: str = "Request",
|
||||
) -> bool:
|
||||
"""Read a request boolean using only the supported JSON representations."""
|
||||
value = payload.get(field_name, default)
|
||||
if value is None:
|
||||
return default
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, (int, float)) and value in (0, 1):
|
||||
return bool(value)
|
||||
if isinstance(value, str):
|
||||
normalized = value.strip().lower()
|
||||
if normalized in {"true", "1", "yes", "on"}:
|
||||
return True
|
||||
if normalized in {"false", "0", "no", "off"}:
|
||||
return False
|
||||
raise TypeError(f"{contract_name} {field_name} must be a boolean")
|
||||
|
||||
|
||||
def read_first_bool_field(
|
||||
payload: Mapping[str, Any],
|
||||
field_names: Tuple[str, ...],
|
||||
*,
|
||||
default: bool = False,
|
||||
contract_name: str = "Request",
|
||||
) -> bool:
|
||||
"""Read the first non-empty boolean alias without coercing bad values."""
|
||||
for field_name in field_names:
|
||||
if field_name not in payload:
|
||||
continue
|
||||
value = payload[field_name]
|
||||
if value is None or value == "":
|
||||
continue
|
||||
return read_bool_field(
|
||||
payload,
|
||||
field_name,
|
||||
default=default,
|
||||
contract_name=contract_name,
|
||||
)
|
||||
return default
|
||||
|
||||
|
||||
def read_int_field(
|
||||
payload: Mapping[str, Any],
|
||||
field_name: str,
|
||||
*,
|
||||
default: int | None = None,
|
||||
contract_name: str = "Request",
|
||||
) -> int | None:
|
||||
"""Read an optional integer without accepting booleans or float truncation."""
|
||||
value = payload.get(field_name, default)
|
||||
if value is None or value == "":
|
||||
return default
|
||||
if isinstance(value, bool) or not isinstance(value, (int, str)):
|
||||
raise TypeError(f"{contract_name} {field_name} must be an integer")
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
return int(value.strip())
|
||||
except ValueError as exc:
|
||||
raise ValueError(
|
||||
f"{contract_name} {field_name} must be an integer"
|
||||
) from exc
|
||||
return value
|
||||
|
||||
|
||||
def validate_workflow_payload(
|
||||
workflow: Any,
|
||||
*,
|
||||
|
||||
+203
-219
@@ -8,12 +8,20 @@ import json
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
from dataclasses import replace
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||
from urllib.parse import unquote
|
||||
|
||||
from .contracts import (
|
||||
MissingModel,
|
||||
ModelMatch,
|
||||
ModelReference,
|
||||
Resolution,
|
||||
ResolvedModel,
|
||||
WorkflowAnalysisResult,
|
||||
)
|
||||
from .custom_nodes import (
|
||||
custom_node_has_potential_model_reference,
|
||||
get_custom_node_resolution_metadata,
|
||||
should_skip_existing_custom_node_reference,
|
||||
)
|
||||
from .log_system import create_module_logger
|
||||
@@ -22,6 +30,7 @@ log = create_module_logger(__name__)
|
||||
|
||||
from .local_hash_matches import collect_local_hash_matches_for_result
|
||||
from .matcher import find_matches, strip_known_model_extension
|
||||
from .metadata_model_utils import normalize_models
|
||||
from .scanner import get_model_files, invalidate_model_files_cache
|
||||
from .type_utils import MODEL_EXTENSIONS as _MODEL_EXTENSIONS
|
||||
from .type_utils import (
|
||||
@@ -63,7 +72,7 @@ from .path_utils import (
|
||||
# Imported from .matcher
|
||||
|
||||
_LOCAL_HASH_MATCH_CACHE_LOCK = threading.Lock()
|
||||
_LOCAL_HASH_MATCH_CACHE: Optional[Dict[str, List[Dict[str, Any]]]] = None
|
||||
_LOCAL_HASH_MATCH_CACHE: Optional[Dict[str, List[ModelMatch]]] = None
|
||||
_ACTIVE_DOWNLOAD_STATUSES = {"starting", "downloading", "paused", "cancelling"}
|
||||
|
||||
|
||||
@@ -80,13 +89,6 @@ def invalidate_model_caches() -> None:
|
||||
invalidate_local_hash_match_cache()
|
||||
|
||||
|
||||
def _clone_hash_match(match: Dict[str, Any]) -> Dict[str, Any]:
|
||||
cloned = dict(match)
|
||||
if isinstance(cloned.get("model"), dict):
|
||||
cloned["model"] = dict(cloned["model"])
|
||||
return cloned
|
||||
|
||||
|
||||
def _get_active_downloads_by_path() -> Dict[str, Dict[str, Any]]:
|
||||
try:
|
||||
from .download.api import get_all_progress
|
||||
@@ -124,9 +126,9 @@ def _get_active_downloads_by_path() -> Dict[str, Dict[str, Any]]:
|
||||
|
||||
|
||||
def annotate_local_matches_with_download_state(
|
||||
matches: List[Dict[str, Any]],
|
||||
matches: List[ModelMatch],
|
||||
active_downloads_by_path: Optional[Dict[str, Dict[str, Any]]] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
) -> List[ModelMatch]:
|
||||
active_downloads = (
|
||||
active_downloads_by_path
|
||||
if active_downloads_by_path is not None
|
||||
@@ -135,18 +137,14 @@ def annotate_local_matches_with_download_state(
|
||||
if not active_downloads:
|
||||
return matches
|
||||
|
||||
enriched_matches: List[Dict[str, Any]] = []
|
||||
enriched_matches: List[ModelMatch] = []
|
||||
for match in matches:
|
||||
if not isinstance(match, dict):
|
||||
enriched_matches.append(match)
|
||||
continue
|
||||
|
||||
model = match.get("model") if isinstance(match.get("model"), dict) else {}
|
||||
model = match.model
|
||||
candidate_paths = [
|
||||
model.get("path"),
|
||||
model.get("resolved_path"),
|
||||
match.get("path"),
|
||||
match.get("resolved_path"),
|
||||
model.path,
|
||||
model.extra_value("resolved_path"),
|
||||
match.extra_value("path"),
|
||||
match.extra_value("resolved_path"),
|
||||
]
|
||||
download_info = None
|
||||
for candidate_path in candidate_paths:
|
||||
@@ -159,24 +157,21 @@ def annotate_local_matches_with_download_state(
|
||||
enriched_matches.append(match)
|
||||
continue
|
||||
|
||||
enriched_match = dict(match)
|
||||
enriched_model = dict(model)
|
||||
download_fields = {
|
||||
**download_info,
|
||||
"is_downloading": True,
|
||||
"downloading": True,
|
||||
}
|
||||
enriched_match.update(download_fields)
|
||||
if enriched_model:
|
||||
enriched_model.update(download_fields)
|
||||
enriched_match["model"] = enriched_model
|
||||
enriched_matches.append(enriched_match)
|
||||
enriched_model = model.with_extra(**download_fields)
|
||||
enriched_matches.append(
|
||||
replace(match, model=enriched_model).with_extra(**download_fields)
|
||||
)
|
||||
|
||||
return enriched_matches
|
||||
|
||||
|
||||
def _is_local_hash_match_candidate(model: Dict[str, Any]) -> bool:
|
||||
model_path = str(model.get("path") or "").strip()
|
||||
def _is_local_hash_match_candidate(model: ResolvedModel) -> bool:
|
||||
model_path = str(model.path or "").strip()
|
||||
if not model_path:
|
||||
return False
|
||||
|
||||
@@ -192,16 +187,18 @@ def _is_local_hash_match_candidate(model: Dict[str, Any]) -> bool:
|
||||
|
||||
|
||||
def _build_local_hash_match_cache(
|
||||
available_models: List[Dict[str, Any]],
|
||||
) -> Dict[str, List[Dict[str, Any]]]:
|
||||
index: Dict[str, List[Dict[str, Any]]] = {}
|
||||
available_models: List[ResolvedModel],
|
||||
) -> Dict[str, List[ModelMatch]]:
|
||||
index: Dict[str, List[ModelMatch]] = {}
|
||||
seen_entries = set()
|
||||
|
||||
for model in available_models:
|
||||
normalized_models = normalize_models(available_models)
|
||||
|
||||
for model in normalized_models:
|
||||
if not _is_local_hash_match_candidate(model):
|
||||
continue
|
||||
|
||||
model_path = model.get("path", "")
|
||||
model_path = model.path
|
||||
if not model_path:
|
||||
continue
|
||||
|
||||
@@ -218,7 +215,7 @@ def _build_local_hash_match_cache(
|
||||
if not metadata_hashes:
|
||||
continue
|
||||
|
||||
model_filename = model.get("filename") or get_filename_from_path(model_path)
|
||||
model_filename = model.filename or get_filename_from_path(model_path)
|
||||
for metadata_hash in metadata_hashes:
|
||||
normalized_hash = normalize_sha256(metadata_hash)
|
||||
if not normalized_hash:
|
||||
@@ -230,29 +227,32 @@ def _build_local_hash_match_cache(
|
||||
continue
|
||||
seen_entries.add(entry_key)
|
||||
|
||||
model_with_metadata = {
|
||||
**model,
|
||||
"sha256": normalized_hash,
|
||||
"metadata_path": metadata_path,
|
||||
}
|
||||
model_with_metadata = model.with_extra(
|
||||
sha256=normalized_hash,
|
||||
metadata_path=metadata_path,
|
||||
)
|
||||
index.setdefault(normalized_hash, []).append(
|
||||
{
|
||||
"model": model_with_metadata,
|
||||
"filename": model_filename,
|
||||
"similarity": 1.0,
|
||||
"confidence": 100.0,
|
||||
"match_type": "hash",
|
||||
"hash_match": True,
|
||||
"hash_source": "metadata",
|
||||
"sha256": normalized_hash,
|
||||
"metadata_path": metadata_path,
|
||||
}
|
||||
ModelMatch.from_mapping(
|
||||
{
|
||||
"model": model_with_metadata,
|
||||
"filename": model_filename,
|
||||
"similarity": 1.0,
|
||||
"confidence": 100.0,
|
||||
"match_type": "hash",
|
||||
"hash_match": True,
|
||||
"hash_source": "metadata",
|
||||
"sha256": normalized_hash,
|
||||
"metadata_path": metadata_path,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
return index
|
||||
|
||||
|
||||
def _get_local_hash_match_cache(force_rescan: bool = False) -> Dict[str, List[Dict[str, Any]]]:
|
||||
def _get_local_hash_match_cache(
|
||||
force_rescan: bool = False,
|
||||
) -> Dict[str, List[ModelMatch]]:
|
||||
global _LOCAL_HASH_MATCH_CACHE
|
||||
if force_rescan:
|
||||
invalidate_local_hash_match_cache()
|
||||
@@ -364,17 +364,17 @@ def extract_workflow_hash_metadata(workflow_json: Dict[str, Any]) -> Dict[str, D
|
||||
|
||||
|
||||
def get_workflow_hash_info_for_ref(
|
||||
workflow_hashes: Dict[str, Dict[str, Any]], model_ref: Dict[str, Any]
|
||||
workflow_hashes: Dict[str, Dict[str, Any]], model_ref: ModelReference
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
if not workflow_hashes:
|
||||
return None
|
||||
original_path = str(model_ref.get("original_path") or "")
|
||||
original_path = model_ref.original_path
|
||||
candidates = [
|
||||
f"{model_ref.get('node_id')}:{model_ref.get('widget_index')}",
|
||||
f"{model_ref.node_id}:{model_ref.widget_index}",
|
||||
original_path,
|
||||
get_filename_from_path(original_path),
|
||||
model_ref.get("filename") or "",
|
||||
model_ref.get("name") or "",
|
||||
model_ref.extra_value("filename", "") or "",
|
||||
model_ref.extra_value("name", "") or "",
|
||||
]
|
||||
for candidate in candidates:
|
||||
if candidate and candidate in workflow_hashes:
|
||||
@@ -452,16 +452,16 @@ def workflow_url_points_to_file(url: str, filename: str) -> bool:
|
||||
return filename in decoded_url or unquote(filename) in decoded_url
|
||||
|
||||
|
||||
def _deduplicate_local_matches(matches: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
def _deduplicate_local_matches(matches: List[ModelMatch]) -> List[ModelMatch]:
|
||||
"""Keep one highest-confidence match for each local model path."""
|
||||
seen_absolute_paths = {}
|
||||
deduplicated_matches = []
|
||||
for match in matches:
|
||||
model_dict = match["model"]
|
||||
absolute_path = model_dict.get("path", "")
|
||||
model = match.model
|
||||
absolute_path = model.path
|
||||
path_identity = get_path_identity(absolute_path) if absolute_path else ""
|
||||
dedupe_key = path_identity or os.path.normcase(
|
||||
model_dict.get("relative_path", "") or match.get("filename", "")
|
||||
model.relative_path or match.filename
|
||||
)
|
||||
|
||||
if dedupe_key not in seen_absolute_paths:
|
||||
@@ -469,7 +469,7 @@ def _deduplicate_local_matches(matches: List[Dict[str, Any]]) -> List[Dict[str,
|
||||
deduplicated_matches.append(match)
|
||||
else:
|
||||
existing_match = seen_absolute_paths[dedupe_key]
|
||||
if match["confidence"] > existing_match["confidence"]:
|
||||
if match.confidence > existing_match.confidence:
|
||||
idx = deduplicated_matches.index(existing_match)
|
||||
deduplicated_matches[idx] = match
|
||||
seen_absolute_paths[dedupe_key] = match
|
||||
@@ -483,7 +483,7 @@ def search_local_matches(
|
||||
similarity_threshold: float = 0.0,
|
||||
max_matches_per_model: int = 10,
|
||||
force_rescan: bool = False,
|
||||
) -> List[Dict[str, Any]]:
|
||||
) -> List[ModelMatch]:
|
||||
"""
|
||||
Search local model files using the same matcher as workflow analysis.
|
||||
|
||||
@@ -497,12 +497,13 @@ def search_local_matches(
|
||||
Deduplicated list of local matches sorted by similarity
|
||||
"""
|
||||
available_models = get_model_files(force_rescan=force_rescan)
|
||||
available_models = normalize_models(available_models)
|
||||
|
||||
candidates = available_models
|
||||
if category and category != "unknown":
|
||||
candidates = [m for m in available_models if m.get("category") == category]
|
||||
candidates = [m for m in available_models if m.category == category]
|
||||
candidates.extend(
|
||||
[m for m in available_models if m.get("category") != category]
|
||||
[m for m in available_models if m.category != category]
|
||||
)
|
||||
|
||||
matches = find_matches(
|
||||
@@ -521,9 +522,13 @@ def _collect_hashes_from_container(value: Any) -> List[str]:
|
||||
return [h] if h else []
|
||||
|
||||
|
||||
def _metadata_file_matches_model(file_info: Dict[str, Any], model: Dict[str, Any]) -> bool:
|
||||
model_filename = str(model.get("filename") or get_filename_from_path(model.get("path", ""))).lower()
|
||||
model_relative = str(model.get("relative_path") or "").replace("\\", "/").lower()
|
||||
def _metadata_file_matches_model(
|
||||
file_info: Dict[str, Any], model: ResolvedModel
|
||||
) -> bool:
|
||||
model_filename = str(
|
||||
model.filename or get_filename_from_path(model.path)
|
||||
).lower()
|
||||
model_relative = str(model.relative_path or "").replace("\\", "/").lower()
|
||||
model_stem = strip_known_model_extension(get_filename_from_path(model_filename)).lower()
|
||||
|
||||
candidates = [
|
||||
@@ -546,7 +551,7 @@ def _metadata_file_matches_model(file_info: Dict[str, Any], model: Dict[str, Any
|
||||
|
||||
|
||||
def _extract_model_sha256_from_metadata(
|
||||
metadata: Dict[str, Any], model: Dict[str, Any]
|
||||
metadata: Dict[str, Any], model: ResolvedModel
|
||||
) -> List[str]:
|
||||
if not isinstance(metadata, dict):
|
||||
return []
|
||||
@@ -580,7 +585,7 @@ def search_local_matches_by_hash(
|
||||
category: Optional[str] = None,
|
||||
max_matches: int = 20,
|
||||
force_rescan: bool = False,
|
||||
) -> List[Dict[str, Any]]:
|
||||
) -> List[ModelMatch]:
|
||||
"""
|
||||
Find local models whose sidecar .metadata.json contains the given SHA256.
|
||||
|
||||
@@ -592,12 +597,12 @@ def search_local_matches_by_hash(
|
||||
return []
|
||||
|
||||
index = _get_local_hash_match_cache(force_rescan=force_rescan)
|
||||
matches = [_clone_hash_match(match) for match in index.get(normalized_hash, [])]
|
||||
matches = list(index.get(normalized_hash, []))
|
||||
|
||||
if category and category != "unknown":
|
||||
matches.sort(
|
||||
key=lambda match: 0
|
||||
if match.get("model", {}).get("category") == category
|
||||
if match.model.category == category
|
||||
else 1
|
||||
)
|
||||
|
||||
@@ -609,7 +614,7 @@ def search_local_matches_by_hash(
|
||||
|
||||
def get_local_model_hash_metadata(
|
||||
model_path: str,
|
||||
model: Optional[Dict[str, Any]] = None,
|
||||
model: Optional[ResolvedModel] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Return SHA256 hashes already stored in sidecar metadata for a local model.
|
||||
@@ -629,12 +634,20 @@ def get_local_model_hash_metadata(
|
||||
file_size = os.path.getsize(normalized_path)
|
||||
except OSError:
|
||||
file_size = 0
|
||||
model_info: Dict[str, Any] = {
|
||||
**(model if isinstance(model, dict) else {}),
|
||||
"path": normalized_path,
|
||||
}
|
||||
model_info.setdefault("filename", get_filename_from_path(normalized_path))
|
||||
model_info.setdefault("relative_path", model_info.get("filename", ""))
|
||||
filename = get_filename_from_path(normalized_path)
|
||||
model_info = (
|
||||
model.with_updates(
|
||||
path=normalized_path,
|
||||
filename=model.filename or filename,
|
||||
relative_path=model.relative_path or model.filename or filename,
|
||||
)
|
||||
if model is not None
|
||||
else ResolvedModel(
|
||||
path=normalized_path,
|
||||
filename=filename,
|
||||
relative_path=filename,
|
||||
)
|
||||
)
|
||||
|
||||
metadata_path = find_metadata_sidecar_path(normalized_path)
|
||||
last_hash_status = ""
|
||||
@@ -821,7 +834,7 @@ def analyze_and_find_matches(
|
||||
progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
|
||||
force_rescan: bool = False,
|
||||
analysis_id: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
) -> WorkflowAnalysisResult:
|
||||
"""
|
||||
Main entry point: analyze workflow and find matches for missing models.
|
||||
|
||||
@@ -833,32 +846,8 @@ def analyze_and_find_matches(
|
||||
analysis_id: Optional identifier used to correlate logs for one request
|
||||
|
||||
Returns:
|
||||
Dictionary with analysis results:
|
||||
{
|
||||
'missing_models': [
|
||||
{
|
||||
'node_id': node ID,
|
||||
'node_type': node type,
|
||||
'widget_index': widget index,
|
||||
'original_path': original path from workflow,
|
||||
'category': model category,
|
||||
'workflow_url': URL from workflow if found,
|
||||
'workflow_directory': directory from workflow if found,
|
||||
'matches': [
|
||||
{
|
||||
'model': model dict from scanner,
|
||||
'filename': model filename,
|
||||
'similarity': similarity score (0.0-1.0),
|
||||
'confidence': confidence percentage (0-100)
|
||||
},
|
||||
...
|
||||
]
|
||||
},
|
||||
...
|
||||
],
|
||||
'total_missing': count of missing models,
|
||||
'total_models_analyzed': count of all models in workflow
|
||||
}
|
||||
Typed workflow analysis result. The HTTP route serializes it only after
|
||||
applying route-level filtering and local download-source enrichment.
|
||||
"""
|
||||
if progress_callback:
|
||||
progress_callback(
|
||||
@@ -883,13 +872,7 @@ def analyze_and_find_matches(
|
||||
"total": 0,
|
||||
}
|
||||
)
|
||||
return {
|
||||
"missing_models": [],
|
||||
"resolved_models": [],
|
||||
"total_resolved": 0,
|
||||
"total_missing": 0,
|
||||
"total_models_analyzed": 0,
|
||||
}
|
||||
return WorkflowAnalysisResult()
|
||||
|
||||
# Extract URLs from workflow (node.properties.models + regex)
|
||||
analysis_context = f" (analysis_id={analysis_id})" if analysis_id else ""
|
||||
@@ -919,16 +902,23 @@ def analyze_and_find_matches(
|
||||
progress_callback=progress_callback,
|
||||
analysis_id=analysis_id,
|
||||
)
|
||||
available_models = inventory["available_models"]
|
||||
all_model_refs = inventory["model_refs"]
|
||||
available_models = inventory.available_models
|
||||
all_model_refs = inventory.model_refs
|
||||
available_models = normalize_models(available_models)
|
||||
all_model_refs = [
|
||||
ref
|
||||
if isinstance(ref, ModelReference)
|
||||
else ModelReference.from_mapping(ref)
|
||||
for ref in all_model_refs
|
||||
]
|
||||
available_models_by_category = {}
|
||||
for model in available_models:
|
||||
model_category = model.get("category", "")
|
||||
model_category = model.category
|
||||
if model_category not in available_models_by_category:
|
||||
available_models_by_category[model_category] = []
|
||||
available_models_by_category[model_category].append(model)
|
||||
|
||||
ordered_candidates_cache: Dict[str, List[Dict[str, Any]]] = {}
|
||||
ordered_candidates_cache: Dict[str, List[ResolvedModel]] = {}
|
||||
|
||||
if progress_callback:
|
||||
progress_callback(
|
||||
@@ -943,35 +933,57 @@ def analyze_and_find_matches(
|
||||
# Identify missing models
|
||||
missing_models = identify_missing_models(all_model_refs, available_models)
|
||||
resolved_model_refs = [
|
||||
model_ref for model_ref in all_model_refs if model_ref.get("exists", False)
|
||||
model_ref for model_ref in all_model_refs if model_ref.exists
|
||||
]
|
||||
|
||||
# Enrich missing models with workflow URLs
|
||||
# Enrich typed missing models with workflow URLs, hash metadata, and URN
|
||||
# hints. Raw mappings are created only for helpers that still consume
|
||||
# external workflow/custom-node metadata.
|
||||
enriched_missing_models = []
|
||||
for missing in missing_models:
|
||||
original_path = missing.get("original_path", "")
|
||||
updates: Dict[str, Any] = {}
|
||||
original_path = missing.reference.original_path
|
||||
filename = get_filename_from_path(original_path)
|
||||
|
||||
url_info = get_workflow_url_info_for_filename(workflow_urls, filename)
|
||||
if url_info:
|
||||
missing["workflow_url"] = url_info.get("url", "")
|
||||
missing["workflow_model_url"] = url_info.get("model_url", "")
|
||||
missing["workflow_directory"] = url_info.get("directory", "")
|
||||
missing["url_source"] = url_info.get("source", "")
|
||||
hash_info = get_workflow_hash_info_for_ref(workflow_hashes, missing)
|
||||
updates.update(
|
||||
{
|
||||
"workflow_url": url_info.get("url", ""),
|
||||
"workflow_model_url": url_info.get("model_url", ""),
|
||||
"workflow_directory": url_info.get("directory", ""),
|
||||
"url_source": url_info.get("source", ""),
|
||||
}
|
||||
)
|
||||
hash_info = get_workflow_hash_info_for_ref(
|
||||
workflow_hashes,
|
||||
missing.reference,
|
||||
)
|
||||
if hash_info:
|
||||
missing["workflow_sha256"] = normalize_sha256(hash_info.get("sha256"))
|
||||
missing["hash_lookup_source"] = hash_info.get("source") or "workflow_metadata"
|
||||
updates.update(
|
||||
{
|
||||
"workflow_sha256": normalize_sha256(hash_info.get("sha256")),
|
||||
"hash_lookup_source": hash_info.get("source")
|
||||
or "workflow_metadata",
|
||||
}
|
||||
)
|
||||
|
||||
# Handle URNs: mark for async resolution by frontend
|
||||
# No sync CivitAI calls here - frontend will fetch asynchronously
|
||||
for missing in missing_models:
|
||||
if missing.get("is_urn"):
|
||||
missing["needs_urn_resolve"] = True
|
||||
urn = missing.get("urn")
|
||||
# Handle URNs: mark for async resolution by frontend. No sync CivitAI
|
||||
# calls happen here; the frontend resolves them asynchronously.
|
||||
if missing.extra_value("is_urn", False):
|
||||
updates["needs_urn_resolve"] = True
|
||||
urn = missing.extra_value("urn")
|
||||
if urn:
|
||||
missing["urn_model_id"] = urn.get("model_id")
|
||||
missing["urn_version_id"] = urn.get("version_id")
|
||||
missing["urn_type"] = urn.get("type", "")
|
||||
updates.update(
|
||||
{
|
||||
"urn_model_id": urn.get("model_id"),
|
||||
"urn_version_id": urn.get("version_id"),
|
||||
"urn_type": urn.get("type", ""),
|
||||
}
|
||||
)
|
||||
|
||||
enriched_missing_models.append(missing.with_extra(**updates))
|
||||
missing_models = enriched_missing_models
|
||||
|
||||
total_matching_models = len(missing_models)
|
||||
if progress_callback:
|
||||
@@ -984,22 +996,24 @@ def analyze_and_find_matches(
|
||||
}
|
||||
)
|
||||
|
||||
local_match_cache: Dict[Tuple[str, str], List[Dict[str, Any]]] = {}
|
||||
local_match_cache: Dict[Tuple[str, str], List[ModelMatch]] = {}
|
||||
active_downloads_by_path = _get_active_downloads_by_path()
|
||||
|
||||
def get_match_target(model_ref: Dict[str, Any]) -> Optional[str]:
|
||||
def get_match_target(model_ref: MissingModel) -> Optional[str]:
|
||||
reference = model_ref.reference
|
||||
target_for_matching = (
|
||||
model_ref.get("original_path")
|
||||
or model_ref.get("expected_filename")
|
||||
or model_ref.get("name")
|
||||
or model_ref.get("filename")
|
||||
or model_ref.get("full_path")
|
||||
reference.original_path
|
||||
or model_ref.extra_value("expected_filename")
|
||||
or model_ref.extra_value("name")
|
||||
or model_ref.extra_value("filename")
|
||||
or model_ref.extra_value("full_path")
|
||||
or ""
|
||||
)
|
||||
|
||||
# For URNs, prefer expected_filename for matching
|
||||
if model_ref.get("is_urn") and model_ref.get("expected_filename"):
|
||||
return model_ref["expected_filename"]
|
||||
expected_filename = model_ref.extra_value("expected_filename")
|
||||
if model_ref.extra_value("is_urn") and expected_filename:
|
||||
return expected_filename
|
||||
|
||||
if isinstance(target_for_matching, str) and target_for_matching.startswith(
|
||||
"urn:air:"
|
||||
@@ -1019,14 +1033,14 @@ def analyze_and_find_matches(
|
||||
|
||||
return target_for_matching
|
||||
|
||||
def get_match_category(model_ref: Dict[str, Any]) -> str:
|
||||
category = model_ref.get("category")
|
||||
def get_match_category(model_ref: MissingModel) -> str:
|
||||
category = model_ref.reference.category
|
||||
if not category or category == "unknown":
|
||||
node_type = model_ref.get("node_type", "")
|
||||
node_type = model_ref.reference.node_type
|
||||
category = NODE_TYPE_TO_CATEGORY_HINTS.get(node_type, "unknown")
|
||||
return category or "unknown"
|
||||
|
||||
def get_candidates_for_category(category: str) -> List[Dict[str, Any]]:
|
||||
def get_candidates_for_category(category: str) -> List[ResolvedModel]:
|
||||
if not category or category == "unknown":
|
||||
return available_models
|
||||
|
||||
@@ -1034,18 +1048,20 @@ def analyze_and_find_matches(
|
||||
if candidates is None:
|
||||
preferred = available_models_by_category.get(category, [])
|
||||
others = [
|
||||
m for m in available_models if m.get("category") != category
|
||||
m for m in available_models if m.category != category
|
||||
]
|
||||
candidates = preferred + others
|
||||
ordered_candidates_cache[category] = candidates
|
||||
|
||||
return candidates
|
||||
|
||||
def find_local_matches_for_ref(model_ref: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
def find_local_matches_for_ref(model_ref: MissingModel) -> List[ModelMatch]:
|
||||
target_for_matching = get_match_target(model_ref)
|
||||
category = get_match_category(model_ref)
|
||||
workflow_sha256 = normalize_sha256(model_ref.get("workflow_sha256"))
|
||||
hash_matches = []
|
||||
workflow_sha256 = normalize_sha256(
|
||||
model_ref.extra_value("workflow_sha256")
|
||||
)
|
||||
hash_matches: List[ModelMatch] = []
|
||||
if workflow_sha256:
|
||||
hash_matches = collect_local_hash_matches_for_result(
|
||||
workflow_sha256,
|
||||
@@ -1053,8 +1069,11 @@ def analyze_and_find_matches(
|
||||
category=category,
|
||||
max_matches=max_matches_per_model,
|
||||
force_rescan=False,
|
||||
source=model_ref.get("hash_lookup_source") or "workflow_metadata",
|
||||
filename=get_filename_from_path(model_ref.get("original_path") or ""),
|
||||
source=model_ref.extra_value("hash_lookup_source")
|
||||
or "workflow_metadata",
|
||||
filename=get_filename_from_path(
|
||||
model_ref.reference.original_path or ""
|
||||
),
|
||||
)
|
||||
|
||||
if not target_for_matching:
|
||||
@@ -1088,41 +1107,35 @@ def analyze_and_find_matches(
|
||||
"message": f"Analyzing model {index} of {total_missing}",
|
||||
"current": index,
|
||||
"total": total_matching_models,
|
||||
"model_name": missing.get("name")
|
||||
or missing.get("original_path", ""),
|
||||
"model_name": missing.extra_value("name")
|
||||
or missing.reference.original_path,
|
||||
}
|
||||
)
|
||||
|
||||
name = missing.get("name") or missing.get("original_path", "")
|
||||
if should_skip_existing_custom_node_reference(missing):
|
||||
name = missing.extra_value("name") or missing.reference.original_path
|
||||
if should_skip_existing_custom_node_reference(missing.reference):
|
||||
log.info(
|
||||
f"Skipping existing custom-node model reference: {name}"
|
||||
)
|
||||
continue
|
||||
|
||||
missing_with_matches.append({
|
||||
**missing,
|
||||
"matches": find_local_matches_for_ref(missing),
|
||||
})
|
||||
missing_with_matches.append(
|
||||
missing.with_matches(
|
||||
find_local_matches_for_ref(missing)
|
||||
)
|
||||
)
|
||||
|
||||
# Existing models already have an exact local path. Fuzzy-matching every
|
||||
# resolved reference against the full local model index is redundant and
|
||||
# makes small workflow edits unnecessarily expensive.
|
||||
resolved_with_matches = [
|
||||
{
|
||||
**resolved,
|
||||
"matches": [],
|
||||
}
|
||||
for resolved in resolved_model_refs
|
||||
]
|
||||
|
||||
result = {
|
||||
"missing_models": missing_with_matches,
|
||||
"resolved_models": resolved_with_matches,
|
||||
"total_resolved": len(resolved_with_matches),
|
||||
"total_missing": len(missing_with_matches),
|
||||
"total_models_analyzed": len(all_model_refs),
|
||||
}
|
||||
resolved_references = tuple(resolved_model_refs)
|
||||
result = WorkflowAnalysisResult(
|
||||
missing_models=tuple(missing_with_matches),
|
||||
resolved_models=resolved_references,
|
||||
total_resolved=len(resolved_references),
|
||||
total_missing=len(missing_with_matches),
|
||||
total_models_analyzed=len(all_model_refs),
|
||||
)
|
||||
|
||||
if progress_callback:
|
||||
progress_callback(
|
||||
@@ -1138,59 +1151,30 @@ def analyze_and_find_matches(
|
||||
|
||||
|
||||
def apply_resolution(
|
||||
workflow_json: Dict[str, Any], resolutions: List[Dict[str, Any]]
|
||||
workflow_json: Dict[str, Any],
|
||||
resolutions: List[Resolution],
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Apply model resolutions to workflow.
|
||||
|
||||
Args:
|
||||
workflow_json: Workflow JSON dictionary (will be modified)
|
||||
resolutions: List of resolution dictionaries:
|
||||
{
|
||||
'node_id': node ID,
|
||||
'widget_index': widget index,
|
||||
'resolved_path': absolute path to resolved model,
|
||||
'category': model category (optional),
|
||||
'resolved_model': model dict from scanner (optional),
|
||||
'nested_key': nested key for dict-type widgets (optional)
|
||||
}
|
||||
resolutions: Validated ``Resolution`` objects.
|
||||
|
||||
Returns:
|
||||
Updated workflow JSON dictionary
|
||||
"""
|
||||
# Prepare mappings for workflow_updater
|
||||
mappings = []
|
||||
typed_resolutions = []
|
||||
for resolution in resolutions:
|
||||
mapping = {
|
||||
"node_id": resolution.get("node_id"),
|
||||
"widget_index": resolution.get("widget_index"),
|
||||
"resolved_path": resolution.get("resolved_path"),
|
||||
"category": resolution.get("category"),
|
||||
"resolved_model": resolution.get("resolved_model"),
|
||||
"subgraph_id": resolution.get(
|
||||
"subgraph_id"
|
||||
), # Include subgraph_id for subgraph nodes
|
||||
"is_top_level": resolution.get(
|
||||
"is_top_level"
|
||||
), # True for top-level nodes, False for nodes in subgraph definitions
|
||||
"nested_key": resolution.get(
|
||||
"nested_key"
|
||||
), # For dict-type widget values
|
||||
"promoted_widget_name": resolution.get("promoted_widget_name"),
|
||||
**get_custom_node_resolution_metadata(resolution),
|
||||
}
|
||||
try:
|
||||
resolution.validate()
|
||||
except (TypeError, ValueError) as exc:
|
||||
log.warning(f"Skipping invalid workflow resolution: {exc}")
|
||||
continue
|
||||
|
||||
# If resolved_model provided, extract path if needed
|
||||
if resolution.get("resolved_model"):
|
||||
resolved_model = resolution["resolved_model"]
|
||||
if "path" in resolved_model and not mapping.get("resolved_path"):
|
||||
mapping["resolved_path"] = resolved_model["path"]
|
||||
if "base_directory" in resolved_model:
|
||||
mapping["base_directory"] = resolved_model["base_directory"]
|
||||
|
||||
mappings.append(mapping)
|
||||
typed_resolutions.append(resolution)
|
||||
|
||||
# Update workflow
|
||||
updated_workflow = update_workflow_nodes(workflow_json, mappings)
|
||||
updated_workflow = update_workflow_nodes(workflow_json, typed_resolutions)
|
||||
|
||||
return updated_workflow
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import asyncio
|
||||
|
||||
from ..request_utils import read_bool_field
|
||||
from ..sources.popular import (
|
||||
get_base_models_config,
|
||||
get_base_models_status,
|
||||
@@ -23,7 +24,14 @@ def register_base_model_routes(routes, web, json_api_endpoint):
|
||||
@json_api_endpoint("base-models status")
|
||||
async def get_base_models_status_route(request):
|
||||
"""Get local and optional remote base-model status."""
|
||||
check_remote = request.query.get("check_remote") == "1"
|
||||
try:
|
||||
check_remote = read_bool_field(
|
||||
request.query,
|
||||
"check_remote",
|
||||
contract_name="Base models status request",
|
||||
)
|
||||
except TypeError as exc:
|
||||
return web.json_response({"error": str(exc)}, status=400)
|
||||
status = await asyncio.to_thread(get_base_models_status, check_remote)
|
||||
return web.json_response(status)
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ import asyncio
|
||||
import threading
|
||||
from functools import wraps
|
||||
|
||||
from ..request_utils import read_optional_object_payload
|
||||
from ..request_utils import read_optional_object_payload, read_text_field
|
||||
|
||||
|
||||
def register_service_route(
|
||||
@@ -170,7 +170,11 @@ def create_route_helpers(web, logger, load_settings, hash_calculation_cancelled)
|
||||
payload = await read_optional_object_payload(request)
|
||||
if "aria2c_path" in payload:
|
||||
settings = dict(settings)
|
||||
settings["aria2c_path"] = payload.get("aria2c_path", "")
|
||||
settings["aria2c_path"] = read_text_field(
|
||||
payload,
|
||||
"aria2c_path",
|
||||
contract_name="Aria2 settings request",
|
||||
)
|
||||
return settings
|
||||
|
||||
return (
|
||||
|
||||
+54
-44
@@ -8,12 +8,19 @@ import os
|
||||
import time
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
from .contracts import ResolvedModel
|
||||
from .log_system import create_module_logger
|
||||
|
||||
log = create_module_logger(__name__)
|
||||
|
||||
|
||||
from .path_utils import get_filename_from_path, get_model_path_identity, get_path_identity
|
||||
from .path_utils import (
|
||||
get_filename_from_path,
|
||||
get_model_path_identity,
|
||||
get_path_identity,
|
||||
normalize_folder_path_values,
|
||||
normalize_string_values,
|
||||
)
|
||||
|
||||
# Import folder_paths lazily - it may not be available until ComfyUI is initialized
|
||||
try:
|
||||
@@ -22,7 +29,7 @@ except ImportError:
|
||||
folder_paths = None
|
||||
log.warning("Model Resolver: folder_paths not available yet - will retry later")
|
||||
|
||||
_MODEL_FILES_CACHE: Optional[List[Dict[str, str]]] = None
|
||||
_MODEL_FILES_CACHE: Optional[List[ResolvedModel]] = None
|
||||
_MODEL_FILES_CACHE_AT: float = 0.0
|
||||
_MODEL_FILES_CACHE_TTL_SECONDS = 2.0
|
||||
|
||||
@@ -53,7 +60,7 @@ def get_model_directories() -> Dict[str, Tuple[List[str], set]]:
|
||||
|
||||
def scan_directory(
|
||||
directory: str, extensions: set, category: str
|
||||
) -> List[Dict[str, str]]:
|
||||
) -> List[ResolvedModel]:
|
||||
"""
|
||||
Recursively scan a single directory for model files.
|
||||
|
||||
@@ -63,16 +70,12 @@ def scan_directory(
|
||||
category: Model category name (e.g., 'checkpoints', 'loras')
|
||||
|
||||
Returns:
|
||||
List of dictionaries with model information:
|
||||
{
|
||||
'filename': 'model.safetensors',
|
||||
'path': 'absolute/path/to/model.safetensors',
|
||||
'relative_path': 'subfolder/model.safetensors' or 'model.safetensors',
|
||||
'category': 'checkpoints',
|
||||
'base_directory': 'absolute/path/to/base'
|
||||
}
|
||||
Typed local model records with path, category and directory metadata.
|
||||
"""
|
||||
models = []
|
||||
if not isinstance(directory, str) or not isinstance(category, str):
|
||||
return models
|
||||
extensions = set(normalize_string_values(extensions))
|
||||
|
||||
if not os.path.exists(directory) or not os.path.isdir(directory):
|
||||
# log.debug(f"Directory does not exist or is not accessible: {directory}")
|
||||
@@ -120,13 +123,13 @@ def scan_directory(
|
||||
relative_path = get_filename_from_path(root)
|
||||
|
||||
models.append(
|
||||
{
|
||||
"filename": get_filename_from_path(root),
|
||||
"path": root,
|
||||
"relative_path": relative_path,
|
||||
"category": category,
|
||||
"base_directory": base_directory,
|
||||
}
|
||||
ResolvedModel(
|
||||
filename=get_filename_from_path(root),
|
||||
path=root,
|
||||
relative_path=relative_path,
|
||||
category=category,
|
||||
base_directory=base_directory,
|
||||
)
|
||||
)
|
||||
|
||||
for filename in files:
|
||||
@@ -152,13 +155,13 @@ def scan_directory(
|
||||
relative_path = filename
|
||||
|
||||
models.append(
|
||||
{
|
||||
"filename": filename,
|
||||
"path": full_path,
|
||||
"relative_path": relative_path,
|
||||
"category": category,
|
||||
"base_directory": base_directory,
|
||||
}
|
||||
ResolvedModel(
|
||||
filename=filename,
|
||||
path=full_path,
|
||||
relative_path=relative_path,
|
||||
category=category,
|
||||
base_directory=base_directory,
|
||||
)
|
||||
)
|
||||
except (OSError, PermissionError) as e:
|
||||
log.warning(f"Error scanning directory {directory}: {e}")
|
||||
@@ -166,12 +169,12 @@ def scan_directory(
|
||||
return models
|
||||
|
||||
|
||||
def scan_all_directories() -> List[Dict[str, str]]:
|
||||
def scan_all_directories() -> List[ResolvedModel]:
|
||||
"""
|
||||
Scan all configured model directories and return list of available models.
|
||||
|
||||
Returns:
|
||||
List of dictionaries with model information (same format as scan_directory)
|
||||
Typed local model records (same fields as scan_directory)
|
||||
"""
|
||||
all_models = []
|
||||
directories = get_model_directories()
|
||||
@@ -188,15 +191,22 @@ def scan_all_directories() -> List[Dict[str, str]]:
|
||||
extensions = set()
|
||||
try:
|
||||
if isinstance(value, (list, tuple)):
|
||||
if len(value) >= 2:
|
||||
paths = value[0] or []
|
||||
raw_exts = value[1]
|
||||
if len(value) >= 2 and (
|
||||
isinstance(value[0], (list, tuple, set, frozenset))
|
||||
or isinstance(value[1], (list, tuple, set, frozenset))
|
||||
):
|
||||
paths = normalize_folder_path_values(value[0])
|
||||
raw_exts = value[1] if len(value) >= 2 else []
|
||||
else:
|
||||
# Unexpected format; treat value as paths
|
||||
paths = list(value)
|
||||
# Unexpected registry format; retain only string paths.
|
||||
paths = normalize_folder_path_values(value)
|
||||
raw_exts = []
|
||||
elif isinstance(value, dict):
|
||||
paths = value.get("paths") or value.get("path") or []
|
||||
paths = normalize_folder_path_values(
|
||||
value.get("paths")
|
||||
if "paths" in value
|
||||
else value.get("path")
|
||||
)
|
||||
raw_exts = value.get("extensions") or []
|
||||
else:
|
||||
# Unknown format; skip category
|
||||
@@ -206,10 +216,10 @@ def scan_all_directories() -> List[Dict[str, str]]:
|
||||
continue
|
||||
|
||||
# Normalize extensions to a set[str]
|
||||
if isinstance(raw_exts, (list, tuple, set)):
|
||||
extensions = {str(e).lower() for e in raw_exts}
|
||||
elif raw_exts:
|
||||
extensions = {str(raw_exts).lower()}
|
||||
extensions = {
|
||||
extension.lower()
|
||||
for extension in normalize_string_values(raw_exts)
|
||||
}
|
||||
except Exception as e:
|
||||
log.warning(f"Error interpreting folder_paths entry for {category}: {e}")
|
||||
continue
|
||||
@@ -225,8 +235,8 @@ def scan_all_directories() -> List[Dict[str, str]]:
|
||||
|
||||
models = scan_directory(directory_path, extensions, category)
|
||||
for model in models:
|
||||
identity = get_model_path_identity(model.get("path"))
|
||||
model_key = (model.get("category", category), identity)
|
||||
identity = get_model_path_identity(model.path)
|
||||
model_key = (model.category or category, identity)
|
||||
if identity and model_key in seen_models:
|
||||
continue
|
||||
if identity:
|
||||
@@ -252,7 +262,7 @@ def invalidate_model_files_cache() -> None:
|
||||
pass
|
||||
|
||||
|
||||
def get_model_files(force_rescan: bool = False) -> List[Dict[str, str]]:
|
||||
def get_model_files(force_rescan: bool = False) -> List[ResolvedModel]:
|
||||
"""
|
||||
Get list of all available model files with metadata.
|
||||
|
||||
@@ -262,7 +272,7 @@ def get_model_files(force_rescan: bool = False) -> List[Dict[str, str]]:
|
||||
force_rescan: If True, bypass the short-lived cache and rescan directories
|
||||
|
||||
Returns:
|
||||
List of model dictionaries (same format as scan_directory)
|
||||
Typed local model records (same fields as scan_directory)
|
||||
"""
|
||||
global _MODEL_FILES_CACHE, _MODEL_FILES_CACHE_AT
|
||||
|
||||
@@ -309,10 +319,10 @@ def find_local_file_path(filename: str, category: Optional[str] = None) -> Optio
|
||||
available_models = get_model_files()
|
||||
for m in available_models:
|
||||
if (
|
||||
m.get("relative_path", "").endswith(filename)
|
||||
or m.get("filename", "") == filename
|
||||
m.relative_path.endswith(filename)
|
||||
or m.filename == filename
|
||||
):
|
||||
file_path = m.get("path")
|
||||
file_path = m.path
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -1,8 +1,16 @@
|
||||
"""CivitAI model search service."""
|
||||
|
||||
from typing import Any, Dict
|
||||
|
||||
from .. import path_utils
|
||||
from ..contracts import SearchResult
|
||||
from ..metadata_utils import build_metadata_source_payload
|
||||
from ..request_utils import extract_request_sha256
|
||||
from ..request_utils import (
|
||||
extract_request_sha256,
|
||||
read_bool_field,
|
||||
read_optional_object_payload,
|
||||
read_text_field,
|
||||
)
|
||||
from ..routes.context import RouteContext
|
||||
from ..sources.common import collect_download_urls
|
||||
from .model_utils import CivitAISearchDependencies, ModelServiceDependencies
|
||||
@@ -31,27 +39,60 @@ class CivitAISearchService(ModelServiceDependencies):
|
||||
request_public_url = self.request_public_url
|
||||
resolve_civarchive_by_hash = self.resolve_civarchive_by_hash
|
||||
search_huggingface_for_file = self.search_huggingface_for_file
|
||||
to_bool = self.to_bool
|
||||
web = self.web
|
||||
write_model_resolver_metadata = self.write_model_resolver_metadata
|
||||
data = await request.json()
|
||||
filename = data.get("filename", "")
|
||||
category = data.get("category", "")
|
||||
resolved_path = data.get("resolved_path", "")
|
||||
local_only = to_bool(data.get("local_only"), False)
|
||||
force_refresh = to_bool(
|
||||
data.get("force_refresh") or data.get("force"), False
|
||||
)
|
||||
provided_hash = extract_request_sha256(
|
||||
data,
|
||||
keys=("sha256", "hash", "file_hash"),
|
||||
)
|
||||
hf_token = data.get("hf_token", "")
|
||||
brave_search_api_key = data.get("brave_search_api_key", "")
|
||||
hf_use_brave_fallback = to_bool(
|
||||
data.get("hf_use_brave_fallback", True),
|
||||
True,
|
||||
)
|
||||
data = await read_optional_object_payload(request)
|
||||
try:
|
||||
filename = read_text_field(
|
||||
data,
|
||||
"filename",
|
||||
contract_name="CivitAI search request",
|
||||
)
|
||||
category = read_text_field(
|
||||
data,
|
||||
"category",
|
||||
contract_name="CivitAI search request",
|
||||
)
|
||||
resolved_path = read_text_field(
|
||||
data,
|
||||
"resolved_path",
|
||||
contract_name="CivitAI search request",
|
||||
)
|
||||
hf_token = read_text_field(
|
||||
data,
|
||||
"hf_token",
|
||||
contract_name="CivitAI search request",
|
||||
)
|
||||
brave_search_api_key = read_text_field(
|
||||
data,
|
||||
"brave_search_api_key",
|
||||
contract_name="CivitAI search request",
|
||||
)
|
||||
local_only = read_bool_field(
|
||||
data,
|
||||
"local_only",
|
||||
contract_name="CivitAI search request",
|
||||
)
|
||||
force_value = dict(data)
|
||||
if force_value.get("force_refresh") is None:
|
||||
force_value["force_refresh"] = force_value.get("force")
|
||||
force_refresh = read_bool_field(
|
||||
force_value,
|
||||
"force_refresh",
|
||||
contract_name="CivitAI search request",
|
||||
)
|
||||
hf_use_brave_fallback = read_bool_field(
|
||||
data,
|
||||
"hf_use_brave_fallback",
|
||||
default=True,
|
||||
contract_name="CivitAI search request",
|
||||
)
|
||||
provided_hash = extract_request_sha256(
|
||||
data,
|
||||
keys=("sha256", "hash", "file_hash"),
|
||||
)
|
||||
except TypeError as exc:
|
||||
return web.json_response({"error": str(exc)}, status=400)
|
||||
|
||||
if not filename:
|
||||
return web.json_response(
|
||||
@@ -292,6 +333,18 @@ class CivitAISearchService(ModelServiceDependencies):
|
||||
return False
|
||||
return not require_filename or result_filename_matches(result)
|
||||
|
||||
def serialize_provider_result(
|
||||
result: SearchResult | None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Cross from typed provider results into metadata response data."""
|
||||
if isinstance(result, SearchResult):
|
||||
return result.to_dict()
|
||||
if result is not None:
|
||||
raise TypeError(
|
||||
f"{type(result).__name__} is not a typed provider result"
|
||||
)
|
||||
return {}
|
||||
|
||||
def huggingface_page_url(result):
|
||||
try:
|
||||
from urllib.parse import quote as _quote
|
||||
@@ -310,7 +363,12 @@ class CivitAISearchService(ModelServiceDependencies):
|
||||
return f"https://huggingface.co/{repo_id}/blob/main/{hf_path}"
|
||||
|
||||
def prepare_remote_result(result, source_name):
|
||||
result = dict(result or {})
|
||||
if isinstance(result, SearchResult):
|
||||
result = result.to_dict()
|
||||
elif not isinstance(result, dict):
|
||||
raise TypeError(
|
||||
f"{type(result).__name__} is not a remote result payload"
|
||||
)
|
||||
source_name = str(source_name or result.get("source") or "").lower()
|
||||
if source_name == "huggingface":
|
||||
download_url = result.get("download_url") or result.get("url")
|
||||
@@ -690,11 +748,13 @@ class CivitAISearchService(ModelServiceDependencies):
|
||||
|
||||
if provided_hash:
|
||||
try:
|
||||
result = resolve_civarchive_by_hash(
|
||||
provided_hash,
|
||||
query=filename,
|
||||
exact_only=False,
|
||||
model_type=infer_model_type_from_category(category),
|
||||
result = serialize_provider_result(
|
||||
resolve_civarchive_by_hash(
|
||||
provided_hash,
|
||||
query=filename,
|
||||
exact_only=False,
|
||||
model_type=infer_model_type_from_category(category),
|
||||
)
|
||||
)
|
||||
if result:
|
||||
if not extract_result_sha256(result):
|
||||
@@ -742,19 +802,21 @@ class CivitAISearchService(ModelServiceDependencies):
|
||||
|
||||
for hf_attempt in hf_attempts:
|
||||
try:
|
||||
result = search_huggingface_for_file(
|
||||
filename,
|
||||
token=hf_token or None,
|
||||
exact_only=True,
|
||||
brave_api_key=brave_search_api_key or None,
|
||||
use_api_search=hf_attempt["use_api_search"],
|
||||
use_comfy_org_fallback=hf_attempt[
|
||||
"use_comfy_org_fallback"
|
||||
],
|
||||
use_brave_fallback=hf_attempt[
|
||||
"use_brave_fallback"
|
||||
],
|
||||
force_refresh=force_refresh,
|
||||
result = serialize_provider_result(
|
||||
search_huggingface_for_file(
|
||||
filename,
|
||||
token=hf_token or None,
|
||||
exact_only=True,
|
||||
brave_api_key=brave_search_api_key or None,
|
||||
use_api_search=hf_attempt["use_api_search"],
|
||||
use_comfy_org_fallback=hf_attempt[
|
||||
"use_comfy_org_fallback"
|
||||
],
|
||||
use_brave_fallback=hf_attempt[
|
||||
"use_brave_fallback"
|
||||
],
|
||||
force_refresh=force_refresh,
|
||||
)
|
||||
)
|
||||
if result and result_hash_matches(
|
||||
result,
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
"""Custom model URL service."""
|
||||
|
||||
from ..contracts import SearchResult
|
||||
from ..local_hash_matches import collect_local_hash_matches_for_result
|
||||
from ..request_utils import (
|
||||
read_first_text_field,
|
||||
read_optional_object_payload,
|
||||
read_text_field,
|
||||
)
|
||||
from ..routes.context import RouteContext
|
||||
from .model_utils import CustomUrlDependencies, ModelServiceDependencies
|
||||
|
||||
@@ -40,6 +46,10 @@ class CustomUrlService(ModelServiceDependencies):
|
||||
def _custom_result_timestamp():
|
||||
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
|
||||
def _as_search_result(value, source=""):
|
||||
if isinstance(value, SearchResult):
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
|
||||
@@ -80,8 +90,11 @@ class CustomUrlService(ModelServiceDependencies):
|
||||
)
|
||||
|
||||
def _collect_custom_url_local_hash_matches(result, category):
|
||||
source_key = str(result.get("source") or "custom").strip().lower()
|
||||
sha256 = normalize_sha256(extract_sha256_from_metadata(result))
|
||||
result_mapping = result.to_dict()
|
||||
source_key = str(result.source or "custom").strip().lower()
|
||||
sha256 = normalize_sha256(
|
||||
extract_sha256_from_metadata(result_mapping)
|
||||
)
|
||||
if not sha256:
|
||||
return []
|
||||
|
||||
@@ -92,8 +105,8 @@ class CustomUrlService(ModelServiceDependencies):
|
||||
category=category or None,
|
||||
max_matches=20,
|
||||
source=source_key,
|
||||
filename=result.get("filename")
|
||||
or result.get("path")
|
||||
filename=result.filename
|
||||
or result.extra_value("path")
|
||||
or "",
|
||||
)
|
||||
except Exception as hash_error:
|
||||
@@ -102,8 +115,40 @@ class CustomUrlService(ModelServiceDependencies):
|
||||
)
|
||||
return []
|
||||
|
||||
data = await request.json()
|
||||
raw_url = str(data.get("url") or data.get("custom_url") or "").strip()
|
||||
data = await read_optional_object_payload(request)
|
||||
try:
|
||||
raw_url = read_first_text_field(
|
||||
data,
|
||||
("url", "custom_url"),
|
||||
contract_name="Custom URL request",
|
||||
)
|
||||
category = read_text_field(
|
||||
data,
|
||||
"category",
|
||||
contract_name="Custom URL request",
|
||||
)
|
||||
filename = read_text_field(
|
||||
data,
|
||||
"filename",
|
||||
contract_name="Custom URL request",
|
||||
)
|
||||
original_path = read_text_field(
|
||||
data,
|
||||
"original_path",
|
||||
contract_name="Custom URL request",
|
||||
)
|
||||
civitai_key = read_text_field(
|
||||
data,
|
||||
"civitai_key",
|
||||
contract_name="Custom URL request",
|
||||
)
|
||||
hf_token = read_text_field(
|
||||
data,
|
||||
"hf_token",
|
||||
contract_name="Custom URL request",
|
||||
)
|
||||
except TypeError as exc:
|
||||
return web.json_response({"error": str(exc)}, status=400)
|
||||
if not raw_url:
|
||||
return web.json_response(
|
||||
{"error": "URL is required"}, status=400
|
||||
@@ -123,14 +168,11 @@ class CustomUrlService(ModelServiceDependencies):
|
||||
status=400,
|
||||
)
|
||||
|
||||
category = data.get("category") or ""
|
||||
expected_filename = (
|
||||
data.get("filename")
|
||||
or get_filename_from_path(data.get("original_path") or "")
|
||||
filename
|
||||
or get_filename_from_path(original_path)
|
||||
or ""
|
||||
)
|
||||
civitai_key = data.get("civitai_key", "")
|
||||
hf_token = data.get("hf_token", "")
|
||||
|
||||
result = None
|
||||
source = ""
|
||||
@@ -145,8 +187,8 @@ class CustomUrlService(ModelServiceDependencies):
|
||||
|
||||
if civitai_parsed:
|
||||
source = "civitai"
|
||||
model_id = civitai_parsed.get("model_id")
|
||||
version_id = civitai_parsed.get("version_id")
|
||||
model_id = civitai_parsed.model_id
|
||||
version_id = civitai_parsed.version_id
|
||||
if model_id:
|
||||
details = await asyncio.to_thread(
|
||||
get_civitai_model_details,
|
||||
@@ -185,17 +227,17 @@ class CustomUrlService(ModelServiceDependencies):
|
||||
)
|
||||
elif civarchive_parsed:
|
||||
source = "civarchive"
|
||||
if civarchive_parsed.get("sha256"):
|
||||
if civarchive_parsed.sha256:
|
||||
result = await asyncio.to_thread(
|
||||
resolve_civarchive_by_hash,
|
||||
civarchive_parsed.get("sha256"),
|
||||
civarchive_parsed.sha256,
|
||||
expected_filename,
|
||||
False,
|
||||
normalize_category_to_model_type(category),
|
||||
)
|
||||
else:
|
||||
model_id = civarchive_parsed.get("model_id")
|
||||
version_id = civarchive_parsed.get("version_id")
|
||||
model_id = civarchive_parsed.model_id
|
||||
version_id = civarchive_parsed.version_id
|
||||
if model_id:
|
||||
result = await asyncio.to_thread(
|
||||
resolve_civarchive_model_version,
|
||||
@@ -217,11 +259,14 @@ class CustomUrlService(ModelServiceDependencies):
|
||||
expected_filename=expected_filename,
|
||||
)
|
||||
if result:
|
||||
result = dict(result)
|
||||
result["source"] = "civarchive"
|
||||
result["details_source"] = "civarchive"
|
||||
result["match_type"] = "custom_url"
|
||||
result["custom_url"] = True
|
||||
result = _as_search_result(result, source="civarchive")
|
||||
if result is not None:
|
||||
result = result.with_updates(
|
||||
source="civarchive",
|
||||
details_source="civarchive",
|
||||
match_type="custom_url",
|
||||
custom_url=True,
|
||||
)
|
||||
else:
|
||||
from urllib.parse import urlparse
|
||||
|
||||
@@ -258,32 +303,44 @@ class CustomUrlService(ModelServiceDependencies):
|
||||
status=400,
|
||||
)
|
||||
|
||||
result = dict(result)
|
||||
result["provided_url"] = normalized_url
|
||||
result["url_source"] = "custom"
|
||||
result["searched_at"] = result.get("searched_at") or _custom_result_timestamp()
|
||||
result.setdefault("category", category)
|
||||
if expected_filename and not result.get("filename"):
|
||||
result["filename"] = expected_filename
|
||||
result = _as_search_result(result, source=source)
|
||||
if result is None:
|
||||
return web.json_response(
|
||||
{"error": "The URL resolved to an invalid model result"},
|
||||
status=400,
|
||||
)
|
||||
|
||||
if not (result.get("download_url") or result.get("url")):
|
||||
result = result.with_extra(
|
||||
provided_url=normalized_url,
|
||||
url_source="custom",
|
||||
searched_at=result.extra_value("searched_at")
|
||||
or _custom_result_timestamp(),
|
||||
category=result.extra_value("category") or category,
|
||||
)
|
||||
if expected_filename and not result.filename:
|
||||
result = result.with_updates(filename=expected_filename)
|
||||
|
||||
if not (result.download_url or result.url):
|
||||
return web.json_response(
|
||||
{"error": "The URL resolved, but no download URL was found"},
|
||||
status=400,
|
||||
)
|
||||
|
||||
source = source or result.get("source") or "custom"
|
||||
source = source or result.source or "custom"
|
||||
local_hash_matches = _collect_custom_url_local_hash_matches(
|
||||
result,
|
||||
category,
|
||||
)
|
||||
result_mapping = result.to_dict()
|
||||
response = {
|
||||
"success": True,
|
||||
"source": source,
|
||||
"result": result,
|
||||
"custom": [result],
|
||||
"result": result_mapping,
|
||||
"custom": [result_mapping],
|
||||
"searched_sources": ["custom"],
|
||||
"local_hash_matches": local_hash_matches,
|
||||
"local_hash_matches": [
|
||||
match.to_dict() for match in local_hash_matches
|
||||
],
|
||||
}
|
||||
response[source] = result
|
||||
response[source] = result_mapping
|
||||
return web.json_response(response)
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import os
|
||||
|
||||
from ..path_utils import normalize_folder_path_values, normalize_string_values
|
||||
from ..routes.context import RouteContext
|
||||
|
||||
|
||||
@@ -99,7 +100,11 @@ class DirectoryService:
|
||||
candidate_keys.append(candidate_key)
|
||||
paths = []
|
||||
for candidate_key in candidate_keys:
|
||||
paths.extend(folder_paths.get_folder_paths(candidate_key) or [])
|
||||
paths.extend(
|
||||
normalize_folder_path_values(
|
||||
folder_paths.get_folder_paths(candidate_key)
|
||||
)
|
||||
)
|
||||
if folder_key == "ultralytics":
|
||||
normalized_ultralytics_paths = []
|
||||
for path in paths:
|
||||
@@ -134,7 +139,9 @@ class DirectoryService:
|
||||
or raw_key in roots
|
||||
):
|
||||
continue
|
||||
raw_paths = folder_paths.get_folder_paths(raw_key) or []
|
||||
raw_paths = normalize_folder_path_values(
|
||||
folder_paths.get_folder_paths(raw_key)
|
||||
)
|
||||
roots[raw_key] = self.dedupe_local_base_directories(
|
||||
raw_paths,
|
||||
comfy_root=comfy_root,
|
||||
@@ -220,7 +227,9 @@ class DirectoryService:
|
||||
|
||||
raw_base_dirs = []
|
||||
for folder_key in available_folder_keys:
|
||||
for base_dir in folder_paths.get_folder_paths(folder_key) or []:
|
||||
for base_dir in normalize_folder_path_values(
|
||||
folder_paths.get_folder_paths(folder_key)
|
||||
):
|
||||
if not base_dir or not os.path.isdir(base_dir):
|
||||
continue
|
||||
raw_base_dirs.append(base_dir)
|
||||
@@ -239,7 +248,9 @@ class DirectoryService:
|
||||
return ""
|
||||
|
||||
for folder_key in available_folder_keys:
|
||||
filenames = folder_paths.get_filename_list(folder_key) or []
|
||||
filenames = normalize_string_values(
|
||||
folder_paths.get_filename_list(folder_key)
|
||||
)
|
||||
for rel_path in filenames:
|
||||
if not isinstance(rel_path, str):
|
||||
continue
|
||||
|
||||
@@ -3,7 +3,15 @@
|
||||
import os
|
||||
from urllib.parse import unquote, urlparse
|
||||
|
||||
from ..request_utils import read_optional_object_payload
|
||||
from ..contracts import DownloadSpec
|
||||
from ..request_utils import (
|
||||
coerce_integer_identifier,
|
||||
read_bool_field,
|
||||
read_first_identifier_field,
|
||||
read_first_text_field,
|
||||
read_optional_object_payload,
|
||||
read_text_field,
|
||||
)
|
||||
from ..routes.context import RouteContext
|
||||
|
||||
|
||||
@@ -18,7 +26,6 @@ class DownloadService:
|
||||
self.clear_completed_downloads_fn = context.require(
|
||||
"clear_completed_downloads"
|
||||
)
|
||||
self.first_non_empty = context.require("first_non_empty")
|
||||
self.get_all_progress_fn = context.require("get_all_progress")
|
||||
self.get_aria2_status_fn = context.require("get_aria2_status")
|
||||
self.get_civarchive_model_details = context.require(
|
||||
@@ -57,8 +64,6 @@ class DownloadService:
|
||||
self.start_aria2_daemon = context.require("start_aria2_daemon")
|
||||
self.start_background_download = context.require("start_background_download")
|
||||
self.stop_aria2_daemon = context.require("stop_aria2_daemon")
|
||||
self.to_bool = context.require("to_bool")
|
||||
self.to_int = context.require("to_int")
|
||||
self.validate_public_http_url = context.require("validate_public_http_url")
|
||||
self.web = context.require("web")
|
||||
|
||||
@@ -68,23 +73,122 @@ class DownloadService:
|
||||
|
||||
async def download_model(self, request):
|
||||
"""Validate and start a background model download."""
|
||||
data = await request.json()
|
||||
url = data.get("url", "")
|
||||
filename = data.get("filename", "")
|
||||
data = await read_optional_object_payload(request)
|
||||
try:
|
||||
url = read_text_field(
|
||||
data,
|
||||
"url",
|
||||
contract_name="Download request",
|
||||
)
|
||||
filename = read_text_field(
|
||||
data,
|
||||
"filename",
|
||||
contract_name="Download request",
|
||||
)
|
||||
category_value = read_text_field(
|
||||
data,
|
||||
"category",
|
||||
default="checkpoints",
|
||||
contract_name="Download request",
|
||||
)
|
||||
subfolder = read_text_field(
|
||||
data,
|
||||
"subfolder",
|
||||
contract_name="Download request",
|
||||
)
|
||||
base_directory = read_text_field(
|
||||
data,
|
||||
"base_directory",
|
||||
contract_name="Download request",
|
||||
)
|
||||
hf_token = read_text_field(
|
||||
data,
|
||||
"hf_token",
|
||||
contract_name="Download request",
|
||||
)
|
||||
civitai_key = read_text_field(
|
||||
data,
|
||||
"civitai_key",
|
||||
contract_name="Download request",
|
||||
)
|
||||
civitai_session_token = read_text_field(
|
||||
data,
|
||||
"civitai_session_token",
|
||||
contract_name="Download request",
|
||||
)
|
||||
except TypeError as exc:
|
||||
return self.web.json_response({"error": str(exc)}, status=400)
|
||||
|
||||
category = self.normalize_download_category(
|
||||
data.get("category", "checkpoints")
|
||||
category_value
|
||||
)
|
||||
subfolder = data.get("subfolder", "")
|
||||
base_directory = data.get("base_directory", "")
|
||||
path_metadata = data.get("path_metadata", {})
|
||||
if not isinstance(path_metadata, dict):
|
||||
if path_metadata is None:
|
||||
path_metadata = {}
|
||||
download_metadata = data.get("download_metadata") or data.get(
|
||||
"metadata", {}
|
||||
)
|
||||
if not isinstance(download_metadata, dict):
|
||||
elif not isinstance(path_metadata, dict):
|
||||
return self.web.json_response(
|
||||
{"error": "Download path metadata must be an object"},
|
||||
status=400,
|
||||
)
|
||||
download_metadata = data.get("download_metadata")
|
||||
if download_metadata is None:
|
||||
download_metadata = data.get("metadata")
|
||||
if download_metadata is None:
|
||||
download_metadata = {}
|
||||
if not isinstance(download_metadata, dict):
|
||||
return self.web.json_response(
|
||||
{"error": "Download metadata must be an object"},
|
||||
status=400,
|
||||
)
|
||||
download_metadata = dict(download_metadata)
|
||||
try:
|
||||
metadata_source = read_first_text_field(
|
||||
download_metadata,
|
||||
("details_source", "source"),
|
||||
contract_name="Download metadata",
|
||||
)
|
||||
if not metadata_source:
|
||||
metadata_source = read_first_text_field(
|
||||
path_metadata,
|
||||
("source",),
|
||||
contract_name="Download path metadata",
|
||||
)
|
||||
|
||||
raw_model_id = read_first_identifier_field(
|
||||
download_metadata,
|
||||
("model_id", "modelId"),
|
||||
contract_name="Download metadata",
|
||||
)
|
||||
if raw_model_id is None:
|
||||
raw_model_id = read_first_identifier_field(
|
||||
path_metadata,
|
||||
("model_id",),
|
||||
contract_name="Download path metadata",
|
||||
)
|
||||
raw_version_id = read_first_identifier_field(
|
||||
download_metadata,
|
||||
("version_id", "versionId"),
|
||||
contract_name="Download metadata",
|
||||
)
|
||||
if raw_version_id is None:
|
||||
raw_version_id = read_first_identifier_field(
|
||||
path_metadata,
|
||||
("version_id",),
|
||||
contract_name="Download path metadata",
|
||||
)
|
||||
|
||||
model_id = coerce_integer_identifier(
|
||||
raw_model_id,
|
||||
"model_id",
|
||||
contract_name="Download metadata",
|
||||
)
|
||||
version_id = coerce_integer_identifier(
|
||||
raw_version_id,
|
||||
"version_id",
|
||||
contract_name="Download metadata",
|
||||
)
|
||||
except (TypeError, ValueError) as exc:
|
||||
return self.web.json_response({"error": str(exc)}, status=400)
|
||||
settings = self.load_resolver_settings()
|
||||
if not base_directory:
|
||||
base_directory = self.get_default_root_for_category(category, settings)
|
||||
@@ -119,7 +223,6 @@ class DownloadService:
|
||||
|
||||
headers = {}
|
||||
if self.host_matches_domain(download_host, "huggingface.co"):
|
||||
hf_token = data.get("hf_token", "")
|
||||
if hf_token:
|
||||
headers["Authorization"] = f"Bearer {hf_token}"
|
||||
elif self.host_matches_domain(
|
||||
@@ -127,12 +230,8 @@ class DownloadService:
|
||||
"civitai.com",
|
||||
"civitai.red",
|
||||
):
|
||||
civitai_key = data.get("civitai_key", "")
|
||||
if civitai_key and "token=" not in url:
|
||||
url += f"{'&' if '?' in url else '?'}token={civitai_key}"
|
||||
civitai_session_token = str(
|
||||
data.get("civitai_session_token", "") or ""
|
||||
).strip()
|
||||
if civitai_session_token:
|
||||
headers["Cookie"] = (
|
||||
f"__Secure-civitai-token={civitai_session_token}"
|
||||
@@ -151,33 +250,9 @@ class DownloadService:
|
||||
download_metadata.setdefault("path_metadata", path_metadata)
|
||||
download_metadata.setdefault(
|
||||
"source",
|
||||
self.first_non_empty(
|
||||
download_metadata.get("details_source"),
|
||||
path_metadata.get("source"),
|
||||
inferred_source,
|
||||
),
|
||||
metadata_source or inferred_source,
|
||||
)
|
||||
|
||||
model_id = self.to_int(
|
||||
self.first_non_empty(
|
||||
download_metadata.get("model_id"),
|
||||
download_metadata.get("modelId"),
|
||||
path_metadata.get("model_id"),
|
||||
)
|
||||
)
|
||||
version_id = self.to_int(
|
||||
self.first_non_empty(
|
||||
download_metadata.get("version_id"),
|
||||
download_metadata.get("versionId"),
|
||||
path_metadata.get("version_id"),
|
||||
)
|
||||
)
|
||||
source_name = str(
|
||||
self.first_non_empty(
|
||||
download_metadata.get("details_source"),
|
||||
download_metadata.get("source"),
|
||||
)
|
||||
).lower()
|
||||
source_name = (metadata_source or inferred_source).lower()
|
||||
try:
|
||||
if (
|
||||
source_name == "civitai"
|
||||
@@ -188,7 +263,7 @@ class DownloadService:
|
||||
self.get_civitai_model_details,
|
||||
model_id,
|
||||
version_id,
|
||||
data.get("civitai_key", ""),
|
||||
civitai_key,
|
||||
)
|
||||
if details:
|
||||
download_metadata["civitai_details"] = details
|
||||
@@ -227,15 +302,21 @@ class DownloadService:
|
||||
target_directory = ""
|
||||
target_path = ""
|
||||
|
||||
download_id = self.start_background_download(
|
||||
url=url,
|
||||
filename=filename,
|
||||
category=category,
|
||||
headers=headers if headers else None,
|
||||
subfolder=subfolder,
|
||||
base_directory=base_directory,
|
||||
metadata=download_metadata,
|
||||
)
|
||||
try:
|
||||
download_spec = DownloadSpec(
|
||||
url=url,
|
||||
filename=filename,
|
||||
category=category,
|
||||
headers=headers,
|
||||
subfolder=subfolder,
|
||||
base_directory=base_directory,
|
||||
metadata=download_metadata,
|
||||
expected_sha256=download_metadata.get("sha256"),
|
||||
)
|
||||
except (TypeError, ValueError) as exc:
|
||||
return self.web.json_response({"error": str(exc)}, status=400)
|
||||
|
||||
download_id = self.start_background_download(**download_spec.to_kwargs())
|
||||
|
||||
return self.web.json_response(
|
||||
{
|
||||
@@ -291,13 +372,19 @@ class DownloadService:
|
||||
|
||||
async def aria2_status(self, request):
|
||||
"""Report aria2 availability using optional request settings."""
|
||||
settings = await self.get_override_settings_from_request(request)
|
||||
try:
|
||||
settings = await self.get_override_settings_from_request(request)
|
||||
except TypeError as exc:
|
||||
return self.web.json_response({"error": str(exc)}, status=400)
|
||||
result = await self.asyncio.to_thread(self.get_aria2_status_fn, settings)
|
||||
return self.web.json_response(result)
|
||||
|
||||
async def aria2_start(self, request):
|
||||
"""Start the aria2 daemon without starting a download."""
|
||||
settings = await self.get_override_settings_from_request(request)
|
||||
try:
|
||||
settings = await self.get_override_settings_from_request(request)
|
||||
except TypeError as exc:
|
||||
return self.web.json_response({"error": str(exc)}, status=400)
|
||||
result = await self.asyncio.to_thread(self.start_aria2_daemon, settings)
|
||||
return self._json_result_with_success_status(result)
|
||||
|
||||
@@ -309,7 +396,14 @@ class DownloadService:
|
||||
async def aria2_install(self, request):
|
||||
"""Install aria2 and persist the selected download backend."""
|
||||
payload = await read_optional_object_payload(request)
|
||||
force = self.to_bool(payload.get("force"), False)
|
||||
try:
|
||||
force = read_bool_field(
|
||||
payload,
|
||||
"force",
|
||||
contract_name="Aria2 install request",
|
||||
)
|
||||
except TypeError as exc:
|
||||
return self.web.json_response({"error": str(exc)}, status=400)
|
||||
try:
|
||||
install_result = await self.asyncio.to_thread(
|
||||
self.install_aria2_engine,
|
||||
|
||||
+120
-58
@@ -4,9 +4,18 @@ import os
|
||||
import time
|
||||
|
||||
from .. import path_utils
|
||||
from ..contracts import ModelReference, ResolvedModel
|
||||
from ..local_hash_matches import collect_local_hash_matches_for_result
|
||||
from ..progress import generate_progress_id
|
||||
from ..request_utils import extract_request_sha256, validate_workflow_payload
|
||||
from ..request_utils import (
|
||||
extract_request_sha256,
|
||||
read_bool_field,
|
||||
read_first_text_field,
|
||||
read_int_field,
|
||||
read_optional_object_payload,
|
||||
read_text_field,
|
||||
validate_workflow_payload,
|
||||
)
|
||||
from ..routes.context import RouteContext
|
||||
|
||||
|
||||
@@ -59,8 +68,6 @@ class HashService:
|
||||
)
|
||||
self.extension = context.require("self")
|
||||
self.time = context.get("time") or time
|
||||
self.to_bool = context.require("to_bool")
|
||||
self.to_int = context.require("to_int")
|
||||
self.web = context.require("web")
|
||||
self.write_json_atomic = context.require("write_json_atomic")
|
||||
|
||||
@@ -84,16 +91,22 @@ class HashService:
|
||||
|
||||
async def local_model_hashes(self, request):
|
||||
"""Return SHA256 hashes already stored in local sidecar metadata."""
|
||||
data = await request.json()
|
||||
data = await read_optional_object_payload(request)
|
||||
model = data.get("model") if isinstance(data.get("model"), dict) else {}
|
||||
path = (
|
||||
data.get("path")
|
||||
or data.get("file_path")
|
||||
or data.get("resolved_path")
|
||||
or model.get("path")
|
||||
or model.get("resolved_path")
|
||||
or ""
|
||||
)
|
||||
try:
|
||||
path = read_first_text_field(
|
||||
data,
|
||||
("path", "file_path", "resolved_path"),
|
||||
contract_name="Local model hashes request",
|
||||
)
|
||||
if not path:
|
||||
path = read_first_text_field(
|
||||
model,
|
||||
("path", "resolved_path"),
|
||||
contract_name="Local model hashes model",
|
||||
)
|
||||
except TypeError as exc:
|
||||
return self.web.json_response({"error": str(exc)}, status=400)
|
||||
|
||||
if not path:
|
||||
return self.web.json_response(
|
||||
@@ -113,7 +126,14 @@ class HashService:
|
||||
|
||||
async def get_model_preview(self, request):
|
||||
"""Serve an adjacent model preview from a configured model directory."""
|
||||
model_path = str(request.query.get("path") or "").strip()
|
||||
try:
|
||||
model_path = read_text_field(
|
||||
request.query,
|
||||
"path",
|
||||
contract_name="Model preview request",
|
||||
)
|
||||
except TypeError as exc:
|
||||
return self.web.Response(text=str(exc), status=400)
|
||||
is_preview_probe = str(getattr(request, "method", "")).upper() == "HEAD"
|
||||
if not model_path:
|
||||
return self.web.Response(text="path is required", status=400)
|
||||
@@ -153,7 +173,7 @@ class HashService:
|
||||
|
||||
async def workflow_model_hashes(self, request):
|
||||
"""Return hash metadata for existing local models used by a workflow."""
|
||||
data = await request.json()
|
||||
data = await read_optional_object_payload(request)
|
||||
workflow_json, workflow_error = validate_workflow_payload(
|
||||
data.get("workflow"),
|
||||
none_is_missing=False,
|
||||
@@ -182,7 +202,16 @@ class HashService:
|
||||
self.get_workflow_model_inventory,
|
||||
workflow_json,
|
||||
)
|
||||
refs = inventory["model_refs"]
|
||||
refs = []
|
||||
for raw_ref in inventory.model_refs:
|
||||
try:
|
||||
refs.append(
|
||||
raw_ref
|
||||
if isinstance(raw_ref, ModelReference)
|
||||
else ModelReference.from_mapping(raw_ref)
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
|
||||
by_node = {}
|
||||
by_path = {}
|
||||
@@ -190,17 +219,17 @@ class HashService:
|
||||
seen = set()
|
||||
|
||||
for ref in refs:
|
||||
if not isinstance(ref, dict) or not ref.get("exists"):
|
||||
if not ref.exists:
|
||||
continue
|
||||
full_path = str(ref.get("full_path") or "").strip()
|
||||
full_path = ref.extra_value("full_path") or ""
|
||||
if not full_path:
|
||||
continue
|
||||
model_info = {
|
||||
"path": full_path,
|
||||
"filename": self.get_filename_from_path(full_path),
|
||||
"relative_path": ref.get("original_path") or "",
|
||||
"category": ref.get("category") or "",
|
||||
}
|
||||
model_info = ResolvedModel(
|
||||
path=full_path,
|
||||
filename=self.get_filename_from_path(full_path),
|
||||
relative_path=ref.original_path or "",
|
||||
category=ref.category or "",
|
||||
)
|
||||
metadata = self.get_local_model_hash_metadata(
|
||||
full_path,
|
||||
model=model_info,
|
||||
@@ -210,15 +239,15 @@ class HashService:
|
||||
continue
|
||||
|
||||
entry = {
|
||||
"node_id": ref.get("node_id"),
|
||||
"node_type": ref.get("node_type") or "",
|
||||
"widget_index": ref.get("widget_index"),
|
||||
"widget_name": ref.get("widget_name") or "",
|
||||
"path": ref.get("original_path") or "",
|
||||
"node_id": ref.node_id,
|
||||
"node_type": ref.node_type or "",
|
||||
"widget_index": ref.widget_index,
|
||||
"widget_name": ref.extra_value("widget_name") or "",
|
||||
"path": ref.original_path or "",
|
||||
"filename": self.get_filename_from_path(
|
||||
ref.get("original_path") or full_path
|
||||
ref.original_path or full_path
|
||||
),
|
||||
"category": ref.get("category") or "",
|
||||
"category": ref.category or "",
|
||||
"sha256": sha256,
|
||||
"size": metadata.get("size") or 0,
|
||||
}
|
||||
@@ -253,30 +282,52 @@ class HashService:
|
||||
|
||||
async def local_matches_by_hash(self, request):
|
||||
"""Search local model metadata sidecars for a remote SHA256."""
|
||||
data = await request.json()
|
||||
sha256 = extract_request_sha256(
|
||||
data,
|
||||
keys=("sha256", "hash", "SHA256"),
|
||||
)
|
||||
data = await read_optional_object_payload(request)
|
||||
try:
|
||||
sha256 = extract_request_sha256(
|
||||
data,
|
||||
keys=("sha256", "hash", "SHA256"),
|
||||
)
|
||||
except TypeError as exc:
|
||||
return self.web.json_response({"error": str(exc)}, status=400)
|
||||
if not sha256:
|
||||
return self.web.json_response(
|
||||
{"error": "sha256 is required"}, status=400
|
||||
)
|
||||
|
||||
category = data.get("category", "")
|
||||
source = str(
|
||||
data.get("source")
|
||||
or data.get("hash_lookup_source")
|
||||
or "download_source"
|
||||
).strip().lower().replace("-", "_")
|
||||
filename = (
|
||||
data.get("filename")
|
||||
or data.get("path")
|
||||
or data.get("model_name")
|
||||
or ""
|
||||
)
|
||||
max_matches = self.to_int(data.get("max_matches"), 20)
|
||||
force_rescan = self.to_bool(data.get("force_rescan"), False)
|
||||
try:
|
||||
category = read_text_field(
|
||||
data,
|
||||
"category",
|
||||
contract_name="Local hash matches request",
|
||||
)
|
||||
source = read_first_text_field(
|
||||
data,
|
||||
("source", "hash_lookup_source"),
|
||||
default="download_source",
|
||||
contract_name="Local hash matches request",
|
||||
).lower().replace("-", "_")
|
||||
filename = read_first_text_field(
|
||||
data,
|
||||
("filename", "path", "model_name"),
|
||||
contract_name="Local hash matches request",
|
||||
)
|
||||
except TypeError as exc:
|
||||
return self.web.json_response({"error": str(exc)}, status=400)
|
||||
try:
|
||||
max_matches = read_int_field(
|
||||
data,
|
||||
"max_matches",
|
||||
default=20,
|
||||
contract_name="Local hash matches request",
|
||||
)
|
||||
force_rescan = read_bool_field(
|
||||
data,
|
||||
"force_rescan",
|
||||
contract_name="Local hash matches request",
|
||||
)
|
||||
except (TypeError, ValueError) as exc:
|
||||
return self.web.json_response({"error": str(exc)}, status=400)
|
||||
|
||||
enriched_matches = collect_local_hash_matches_for_result(
|
||||
sha256,
|
||||
@@ -290,8 +341,10 @@ class HashService:
|
||||
return self.web.json_response(
|
||||
{
|
||||
"sha256": sha256,
|
||||
"local_hash_matches": enriched_matches,
|
||||
"matches": enriched_matches,
|
||||
"local_hash_matches": [
|
||||
match.to_dict() for match in enriched_matches
|
||||
],
|
||||
"matches": [match.to_dict() for match in enriched_matches],
|
||||
}
|
||||
)
|
||||
|
||||
@@ -309,7 +362,17 @@ class HashService:
|
||||
{"success": False, "error": "JSON body must be an object"},
|
||||
status=400,
|
||||
)
|
||||
target_path = data.get("path", "")
|
||||
try:
|
||||
target_path = read_text_field(
|
||||
data,
|
||||
"path",
|
||||
contract_name="Open containing folder request",
|
||||
)
|
||||
except TypeError as exc:
|
||||
return self.web.json_response(
|
||||
{"success": False, "error": str(exc)},
|
||||
status=400,
|
||||
)
|
||||
|
||||
try:
|
||||
normalized_path = self.normalize_file_manager_path(target_path)
|
||||
@@ -366,11 +429,10 @@ class HashService:
|
||||
|
||||
def resolve_hash_file_request(self, data):
|
||||
"""Validate and normalize a file path accepted by hash endpoints."""
|
||||
file_path = (
|
||||
data.get("file_path")
|
||||
or data.get("resolved_path")
|
||||
or data.get("path")
|
||||
or ""
|
||||
file_path = read_first_text_field(
|
||||
data,
|
||||
("file_path", "resolved_path", "path"),
|
||||
contract_name="Hash file request",
|
||||
)
|
||||
if not file_path:
|
||||
return "", "file_path is required"
|
||||
@@ -387,7 +449,7 @@ class HashService:
|
||||
|
||||
async def _get_validated_hash_file_request(self, request):
|
||||
"""Read and validate the file request shared by hash endpoints."""
|
||||
data = await request.json()
|
||||
data = await read_optional_object_payload(request)
|
||||
normalized_path, error = self.resolve_hash_file_request(data)
|
||||
if not error:
|
||||
return normalized_path, None
|
||||
|
||||
@@ -1,6 +1,13 @@
|
||||
"""Loaded model inspection used by the HTTP route adapter."""
|
||||
|
||||
from ..request_utils import validate_workflow_payload
|
||||
from ..contracts import ModelReference
|
||||
from ..metadata_model_utils import normalize_models
|
||||
from ..path_utils import normalize_string_values
|
||||
from ..request_utils import (
|
||||
read_first_text_field,
|
||||
read_optional_object_payload,
|
||||
validate_workflow_payload,
|
||||
)
|
||||
from ..routes.context import RouteContext
|
||||
from ..workflow.traversal import iter_workflow_nodes_with_scope
|
||||
|
||||
@@ -23,14 +30,19 @@ class LoadedModelsService:
|
||||
|
||||
async def get_loaded_models(self, request):
|
||||
"""Get all currently loaded models in the workflow."""
|
||||
data = await request.json()
|
||||
data = await read_optional_object_payload(request)
|
||||
workflow_json, workflow_error = validate_workflow_payload(
|
||||
data.get("workflow"),
|
||||
empty_is_missing=True,
|
||||
)
|
||||
loaded_id = str(
|
||||
data.get("loaded_id") or data.get("progress_id") or ""
|
||||
).strip()
|
||||
try:
|
||||
loaded_id = read_first_text_field(
|
||||
data,
|
||||
("loaded_id", "progress_id"),
|
||||
contract_name="Loaded models request",
|
||||
)
|
||||
except TypeError as exc:
|
||||
return self.web.json_response({"error": str(exc)}, status=400)
|
||||
|
||||
if workflow_error:
|
||||
return self.web.json_response(
|
||||
@@ -80,8 +92,15 @@ class LoadedModelsService:
|
||||
workflow_json,
|
||||
progress_callback=update_workflow_analysis_progress,
|
||||
)
|
||||
available_models = inventory["available_models"]
|
||||
all_model_refs = inventory["model_refs"]
|
||||
available_models = inventory.available_models
|
||||
all_model_refs = inventory.model_refs
|
||||
available_models = normalize_models(available_models)
|
||||
all_model_refs = [
|
||||
ref
|
||||
if isinstance(ref, ModelReference)
|
||||
else ModelReference.from_mapping(ref)
|
||||
for ref in all_model_refs
|
||||
]
|
||||
|
||||
# Create lookup for full paths by filename (with and without extension)
|
||||
path_by_filename = {}
|
||||
@@ -94,19 +113,17 @@ class LoadedModelsService:
|
||||
total=total_local_models,
|
||||
)
|
||||
for index, model_info in enumerate(available_models, start=1):
|
||||
rel_path = model_info.get("relative_path", "")
|
||||
rel_path = model_info.relative_path
|
||||
if rel_path:
|
||||
filename = self.get_filename_from_path(rel_path)
|
||||
path_by_filename[filename] = model_info.get("path")
|
||||
path_by_filename[filename] = model_info.path
|
||||
# Also add without extension for matching (simple approach)
|
||||
if "." in filename:
|
||||
filename_no_ext = filename.rsplit(".", 1)[0]
|
||||
if filename_no_ext not in path_by_filename:
|
||||
path_by_filename[filename_no_ext] = model_info.get(
|
||||
"path"
|
||||
)
|
||||
path_by_filename[filename_no_ext] = model_info.path
|
||||
# Add the full relative path as key too
|
||||
path_by_filename[rel_path] = model_info.get("path")
|
||||
path_by_filename[rel_path] = model_info.path
|
||||
|
||||
if (
|
||||
total_local_models
|
||||
@@ -147,7 +164,9 @@ class LoadedModelsService:
|
||||
total=len(folder_categories),
|
||||
)
|
||||
try:
|
||||
filenames = folder_paths.get_filename_list(category_name)
|
||||
filenames = normalize_string_values(
|
||||
folder_paths.get_filename_list(category_name)
|
||||
)
|
||||
for filename in filenames:
|
||||
full_path = folder_paths.get_full_path(
|
||||
category_name, filename
|
||||
@@ -192,20 +211,18 @@ class LoadedModelsService:
|
||||
},
|
||||
)
|
||||
|
||||
def node_matches_ref(node, ref):
|
||||
def node_matches_ref(node, ref: ModelReference):
|
||||
if not isinstance(node, dict):
|
||||
return False
|
||||
if str(node.get("id")) != str(ref.get("node_id")):
|
||||
if str(node.get("id")) != str(ref.node_id):
|
||||
return False
|
||||
scope = get_node_scope(node)
|
||||
if scope["is_top_level"] != (
|
||||
ref.get("is_top_level") is not False
|
||||
ref.is_top_level is not False
|
||||
):
|
||||
return False
|
||||
if not scope["is_top_level"]:
|
||||
return str(scope["subgraph_id"]) == str(
|
||||
ref.get("subgraph_id") or ""
|
||||
)
|
||||
return str(scope["subgraph_id"]) == str(ref.subgraph_id or "")
|
||||
return True
|
||||
|
||||
# Collect all loaded models with their values
|
||||
@@ -232,18 +249,18 @@ class LoadedModelsService:
|
||||
total=total_refs,
|
||||
)
|
||||
|
||||
original_path = ref.get("original_path", "")
|
||||
node_id = ref.get("node_id")
|
||||
widget_index = ref.get("widget_index")
|
||||
node_type = ref.get("node_type", "")
|
||||
category = ref.get("category", "unknown")
|
||||
original_path = ref.original_path
|
||||
node_id = ref.node_id
|
||||
widget_index = ref.widget_index
|
||||
node_type = ref.node_type
|
||||
category = ref.category or "unknown"
|
||||
|
||||
# Determine model name and strength
|
||||
model_name = self.get_filename_from_path(original_path)
|
||||
strength = None
|
||||
|
||||
if (
|
||||
ref.get("strength") is None
|
||||
ref.extra_value("strength") is None
|
||||
and category in {"lora", "loras"}
|
||||
):
|
||||
for node in nodes:
|
||||
@@ -254,8 +271,8 @@ class LoadedModelsService:
|
||||
)
|
||||
break
|
||||
|
||||
if ref.get("strength") is not None:
|
||||
strength = ref.get("strength")
|
||||
if ref.extra_value("strength") is not None:
|
||||
strength = ref.extra_value("strength")
|
||||
|
||||
model_name, strength = self.adapt_custom_node_loaded_model(
|
||||
ref,
|
||||
@@ -264,11 +281,11 @@ class LoadedModelsService:
|
||||
)
|
||||
|
||||
# Check if model exists locally
|
||||
exists = ref.get("exists", False)
|
||||
exists = ref.exists
|
||||
|
||||
# If URN, resolve to display name
|
||||
if ref.get("is_urn"):
|
||||
urn = ref.get("urn", {})
|
||||
if ref.extra_value("is_urn"):
|
||||
urn = ref.extra_value("urn") or {}
|
||||
# Use model name from URN as display name
|
||||
model_name = (
|
||||
f"urn:{urn.get('type', 'model')}:{urn.get('model_id')}"
|
||||
@@ -284,34 +301,34 @@ class LoadedModelsService:
|
||||
"node_id": node_id,
|
||||
"widget_index": widget_index,
|
||||
"node_type": node_type,
|
||||
"node_title": ref.get("node_title", ""),
|
||||
"subgraph_id": ref.get("subgraph_id") or "",
|
||||
"subgraph_name": ref.get("subgraph_name") or "",
|
||||
"is_top_level": ref.get("is_top_level") is not False,
|
||||
"locate_node_id": ref.get("locate_node_id"),
|
||||
"locate_node_type": ref.get("locate_node_type", ""),
|
||||
"locate_node_title": ref.get("locate_node_title", ""),
|
||||
"node_title": ref.extra_value("node_title", ""),
|
||||
"subgraph_id": ref.subgraph_id or "",
|
||||
"subgraph_name": ref.extra_value("subgraph_name") or "",
|
||||
"is_top_level": ref.is_top_level is not False,
|
||||
"locate_node_id": ref.extra_value("locate_node_id"),
|
||||
"locate_node_type": ref.extra_value("locate_node_type", ""),
|
||||
"locate_node_title": ref.extra_value("locate_node_title", ""),
|
||||
"locate_subgraph_id": (
|
||||
ref.get("locate_subgraph_id") or ""
|
||||
ref.extra_value("locate_subgraph_id") or ""
|
||||
),
|
||||
"locate_subgraph_name": (
|
||||
ref.get("locate_subgraph_name") or ""
|
||||
ref.extra_value("locate_subgraph_name") or ""
|
||||
),
|
||||
"locate_is_top_level": (
|
||||
ref.get("locate_is_top_level") is not False
|
||||
ref.extra_value("locate_is_top_level") is not False
|
||||
),
|
||||
"locate_via_promoted_widget": ref.get(
|
||||
"locate_via_promoted_widget": ref.extra_value(
|
||||
"locate_via_promoted_widget", False
|
||||
),
|
||||
"exists": exists,
|
||||
"strength": strength,
|
||||
"original_path": original_path,
|
||||
"is_urn": ref.get("is_urn", False),
|
||||
"custom_node_adapter": ref.get(
|
||||
"is_urn": ref.extra_value("is_urn", False),
|
||||
"custom_node_adapter": ref.extra_value(
|
||||
"custom_node_adapter"
|
||||
),
|
||||
"active": ref.get("active"),
|
||||
"connected": ref.get("connected", True),
|
||||
"active": ref.extra_value("active"),
|
||||
"connected": ref.extra_value("connected", True),
|
||||
"resolved_path": (
|
||||
path_by_filename.get(model_name)
|
||||
or path_by_filename.get(original_path)
|
||||
|
||||
@@ -1,7 +1,12 @@
|
||||
"""Metadata audit and build orchestration used by HTTP route adapters."""
|
||||
|
||||
from ..progress import generate_progress_id
|
||||
from ..request_utils import read_optional_object_payload
|
||||
from ..request_utils import (
|
||||
read_bool_field,
|
||||
read_int_field,
|
||||
read_optional_object_payload,
|
||||
read_optional_text_field,
|
||||
)
|
||||
from ..routes.context import RouteContext
|
||||
|
||||
|
||||
@@ -26,8 +31,6 @@ class MetadataService:
|
||||
)
|
||||
self.run_in_background_thread = context.require("run_in_background_thread")
|
||||
self.extension = context.require("self")
|
||||
self.to_bool = context.require("to_bool")
|
||||
self.to_int = context.require("to_int")
|
||||
self.web = context.require("web")
|
||||
|
||||
@property
|
||||
@@ -37,13 +40,32 @@ class MetadataService:
|
||||
async def metadata_size_audit(self, request):
|
||||
"""Check local metadata sidecars for stale file-size values."""
|
||||
payload = await read_optional_object_payload(request)
|
||||
|
||||
force_rescan = self.to_bool(payload.get("force_rescan"), True)
|
||||
try:
|
||||
force_rescan = read_bool_field(
|
||||
payload,
|
||||
"force_rescan",
|
||||
default=True,
|
||||
contract_name="Metadata size audit request",
|
||||
)
|
||||
worker_count = read_int_field(
|
||||
payload,
|
||||
"worker_count",
|
||||
default=None,
|
||||
contract_name="Metadata size audit request",
|
||||
)
|
||||
batch_size = read_int_field(
|
||||
payload,
|
||||
"batch_size",
|
||||
default=None,
|
||||
contract_name="Metadata size audit request",
|
||||
)
|
||||
except (TypeError, ValueError) as exc:
|
||||
return self.web.json_response({"error": str(exc)}, status=400)
|
||||
result = await self.asyncio.to_thread(
|
||||
self.audit_metadata_sizes_fn,
|
||||
force_rescan=force_rescan,
|
||||
worker_count=payload.get("worker_count"),
|
||||
batch_size=payload.get("batch_size"),
|
||||
worker_count=worker_count,
|
||||
batch_size=batch_size,
|
||||
)
|
||||
return self.web.json_response(result)
|
||||
|
||||
@@ -56,12 +78,28 @@ class MetadataService:
|
||||
async def metadata_build_start(self, request):
|
||||
"""Start building missing local metadata sidecars in the background."""
|
||||
payload = await read_optional_object_payload(request)
|
||||
|
||||
force_rescan = self.to_bool(payload.get("force_rescan"), True)
|
||||
worker_count = self.to_int(payload.get("worker_count"), 0)
|
||||
metadata_mode = self.normalize_metadata_build_mode(
|
||||
payload.get("metadata_mode")
|
||||
)
|
||||
try:
|
||||
force_rescan = read_bool_field(
|
||||
payload,
|
||||
"force_rescan",
|
||||
default=True,
|
||||
contract_name="Metadata build request",
|
||||
)
|
||||
worker_count = read_int_field(
|
||||
payload,
|
||||
"worker_count",
|
||||
default=0,
|
||||
contract_name="Metadata build request",
|
||||
)
|
||||
metadata_mode = self.normalize_metadata_build_mode(
|
||||
read_optional_text_field(
|
||||
payload,
|
||||
"metadata_mode",
|
||||
contract_name="Metadata build request",
|
||||
)
|
||||
)
|
||||
except (TypeError, ValueError) as exc:
|
||||
return self.web.json_response({"error": str(exc)}, status=400)
|
||||
self.metadata_builder_progress.cleanup()
|
||||
progress_id = generate_progress_id("metadata_build")
|
||||
self.metadata_builder_progress.update(
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
"""Model details service."""
|
||||
|
||||
from ..request_utils import (
|
||||
coerce_integer_identifier,
|
||||
read_identifier_field,
|
||||
read_optional_object_payload,
|
||||
read_text_field,
|
||||
)
|
||||
from ..routes.context import RouteContext
|
||||
from .model_utils import ModelDetailsDependencies, ModelServiceDependencies
|
||||
|
||||
@@ -18,14 +24,45 @@ class ModelDetailsService(ModelServiceDependencies):
|
||||
get_civitai_model_details = self.get_civitai_model_details
|
||||
get_huggingface_model_details = self.get_huggingface_model_details
|
||||
web = self.web
|
||||
data = await request.json()
|
||||
source = str(data.get("source", "")).strip().lower()
|
||||
model_id = data.get("model_id")
|
||||
version_id = data.get("version_id")
|
||||
civitai_key = data.get("civitai_key", "")
|
||||
hf_token = data.get("hf_token", "")
|
||||
file_path = data.get("file_path", "")
|
||||
branch = data.get("branch", "")
|
||||
data = await read_optional_object_payload(request)
|
||||
try:
|
||||
source = read_text_field(
|
||||
data,
|
||||
"source",
|
||||
contract_name="Model details request",
|
||||
).lower()
|
||||
model_id = read_identifier_field(
|
||||
data,
|
||||
"model_id",
|
||||
contract_name="Model details request",
|
||||
)
|
||||
version_id = read_identifier_field(
|
||||
data,
|
||||
"version_id",
|
||||
contract_name="Model details request",
|
||||
)
|
||||
civitai_key = read_text_field(
|
||||
data,
|
||||
"civitai_key",
|
||||
contract_name="Model details request",
|
||||
)
|
||||
hf_token = read_text_field(
|
||||
data,
|
||||
"hf_token",
|
||||
contract_name="Model details request",
|
||||
)
|
||||
file_path = read_text_field(
|
||||
data,
|
||||
"file_path",
|
||||
contract_name="Model details request",
|
||||
)
|
||||
branch = read_text_field(
|
||||
data,
|
||||
"branch",
|
||||
contract_name="Model details request",
|
||||
)
|
||||
except TypeError as exc:
|
||||
return web.json_response({"error": str(exc)}, status=400)
|
||||
|
||||
if not download_available:
|
||||
return web.json_response(
|
||||
@@ -41,27 +78,33 @@ class ModelDetailsService(ModelServiceDependencies):
|
||||
)
|
||||
|
||||
if source == "huggingface":
|
||||
model_id = str(model_id or "").strip()
|
||||
branch = str(branch or version_id or "main").strip() or "main"
|
||||
if model_id is not None and not isinstance(model_id, str):
|
||||
return web.json_response(
|
||||
{"error": "Model details model_id must be a string for Hugging Face"},
|
||||
status=400,
|
||||
)
|
||||
if version_id is not None and not isinstance(version_id, str):
|
||||
return web.json_response(
|
||||
{"error": "Model details version_id must be a string for Hugging Face"},
|
||||
status=400,
|
||||
)
|
||||
model_id = model_id.strip() if isinstance(model_id, str) else ""
|
||||
branch = branch or version_id or "main"
|
||||
version_id = branch
|
||||
else:
|
||||
try:
|
||||
model_id = (
|
||||
int(model_id)
|
||||
if model_id is not None and str(model_id).strip()
|
||||
else None
|
||||
model_id = coerce_integer_identifier(
|
||||
model_id,
|
||||
"model_id",
|
||||
contract_name="Model details request",
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
model_id = None
|
||||
|
||||
try:
|
||||
version_id = (
|
||||
int(version_id)
|
||||
if version_id is not None and str(version_id).strip()
|
||||
else None
|
||||
version_id = coerce_integer_identifier(
|
||||
version_id,
|
||||
"version_id",
|
||||
contract_name="Model details request",
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
version_id = None
|
||||
except (TypeError, ValueError) as exc:
|
||||
return web.json_response({"error": str(exc)}, status=400)
|
||||
|
||||
if not model_id:
|
||||
return web.json_response(
|
||||
|
||||
@@ -3,9 +3,11 @@
|
||||
from dataclasses import dataclass, fields
|
||||
from typing import Any, Dict, Iterable, Optional, Protocol, Tuple
|
||||
|
||||
from ..contracts import ModelMatch, ProviderUrlReference, SearchResult
|
||||
from ..routes.context import RouteContext
|
||||
|
||||
ModelResult = Optional[Dict[str, Any]]
|
||||
SearchResultValue = Optional[SearchResult]
|
||||
|
||||
|
||||
class ExtractSha256Protocol(Protocol):
|
||||
@@ -47,7 +49,7 @@ class NormalizeSha256Protocol(Protocol):
|
||||
|
||||
|
||||
class ModelResultBuilderProtocol(Protocol):
|
||||
def __call__(self, source: str, **fields: Any) -> Dict[str, Any]: ...
|
||||
def __call__(self, source: str, **fields: Any) -> SearchResult: ...
|
||||
|
||||
|
||||
class ReadJsonProtocol(Protocol):
|
||||
@@ -76,7 +78,7 @@ class ResolveCivarchiveByHashProtocol(Protocol):
|
||||
query: str = "",
|
||||
exact_only: bool = False,
|
||||
model_type: Optional[str] = None,
|
||||
) -> ModelResult: ...
|
||||
) -> SearchResultValue: ...
|
||||
|
||||
|
||||
class SearchHuggingFaceProtocol(Protocol):
|
||||
@@ -91,11 +93,7 @@ class SearchHuggingFaceProtocol(Protocol):
|
||||
use_brave_fallback: bool = True,
|
||||
force_refresh: bool = False,
|
||||
progress_callback: Any = None,
|
||||
) -> ModelResult: ...
|
||||
|
||||
|
||||
class ToBoolProtocol(Protocol):
|
||||
def __call__(self, value: Any, default: bool = False) -> bool: ...
|
||||
) -> SearchResultValue: ...
|
||||
|
||||
|
||||
class WriteMetadataProtocol(Protocol):
|
||||
@@ -112,7 +110,7 @@ class WriteMetadataProtocol(Protocol):
|
||||
class BuildCivarchiveCustomResultProtocol(Protocol):
|
||||
def __call__(
|
||||
self, details: Dict[str, Any], expected_filename: str = ""
|
||||
) -> ModelResult: ...
|
||||
) -> SearchResultValue: ...
|
||||
|
||||
|
||||
class BuildCivitaiCustomResultProtocol(Protocol):
|
||||
@@ -121,7 +119,7 @@ class BuildCivitaiCustomResultProtocol(Protocol):
|
||||
details: Dict[str, Any],
|
||||
expected_filename: str = "",
|
||||
api_key: Optional[str] = None,
|
||||
) -> ModelResult: ...
|
||||
) -> SearchResultValue: ...
|
||||
|
||||
|
||||
class BuildHuggingFaceCustomResultProtocol(Protocol):
|
||||
@@ -130,7 +128,7 @@ class BuildHuggingFaceCustomResultProtocol(Protocol):
|
||||
url: str,
|
||||
expected_filename: str = "",
|
||||
token: Optional[str] = None,
|
||||
) -> ModelResult: ...
|
||||
) -> SearchResultValue: ...
|
||||
|
||||
|
||||
class GetCivarchiveModelDetailsProtocol(Protocol):
|
||||
@@ -170,7 +168,7 @@ class HostMatchesDomainProtocol(Protocol):
|
||||
|
||||
|
||||
class ParseProviderUrlProtocol(Protocol):
|
||||
def __call__(self, url: str) -> Optional[Dict[str, Any]]: ...
|
||||
def __call__(self, url: str) -> Optional[ProviderUrlReference]: ...
|
||||
|
||||
|
||||
class ResolveCivarchiveModelVersionProtocol(Protocol):
|
||||
@@ -181,7 +179,7 @@ class ResolveCivarchiveModelVersionProtocol(Protocol):
|
||||
query: str = "",
|
||||
exact_only: bool = False,
|
||||
prefer_page: bool = False,
|
||||
) -> ModelResult: ...
|
||||
) -> SearchResultValue: ...
|
||||
|
||||
|
||||
class ResolveCivitaiVersionProtocol(Protocol):
|
||||
@@ -190,7 +188,7 @@ class ResolveCivitaiVersionProtocol(Protocol):
|
||||
version_id: int,
|
||||
expected_filename: str = "",
|
||||
api_key: Optional[str] = None,
|
||||
) -> ModelResult: ...
|
||||
) -> SearchResultValue: ...
|
||||
|
||||
|
||||
class SearchLocalMatchesByHashProtocol(Protocol):
|
||||
@@ -200,7 +198,7 @@ class SearchLocalMatchesByHashProtocol(Protocol):
|
||||
category: Optional[str] = None,
|
||||
max_matches: int = 20,
|
||||
force_rescan: bool = False,
|
||||
) -> list[Dict[str, Any]]: ...
|
||||
) -> list[ModelMatch]: ...
|
||||
|
||||
|
||||
class ValidatePublicHttpUrlProtocol(Protocol):
|
||||
@@ -227,7 +225,6 @@ class CivitAISearchDependencies:
|
||||
request_public_url: RequestPublicUrlProtocol
|
||||
resolve_civarchive_by_hash: Optional[ResolveCivarchiveByHashProtocol]
|
||||
search_huggingface_for_file: Optional[SearchHuggingFaceProtocol]
|
||||
to_bool: ToBoolProtocol
|
||||
web: Any
|
||||
write_model_resolver_metadata: Optional[WriteMetadataProtocol]
|
||||
|
||||
@@ -265,7 +262,6 @@ class CivitAISearchDependencies:
|
||||
search_huggingface_for_file=context.get(
|
||||
"search_huggingface_for_file"
|
||||
),
|
||||
to_bool=context.require("to_bool"),
|
||||
web=context.require("web"),
|
||||
write_model_resolver_metadata=context.get(
|
||||
"write_model_resolver_metadata"
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
"""Local model scanning orchestration used by HTTP route adapters."""
|
||||
|
||||
from ..metadata_model_utils import normalize_models
|
||||
from ..request_utils import read_first_bool_field
|
||||
from ..routes.context import RouteContext
|
||||
|
||||
|
||||
@@ -15,29 +17,44 @@ class ScannerService:
|
||||
self.invalidate_local_hash_match_cache = context.require(
|
||||
"invalidate_local_hash_match_cache"
|
||||
)
|
||||
self.to_bool = context.require("to_bool")
|
||||
self.web = context.require("web")
|
||||
|
||||
async def get_models(self, request):
|
||||
"""Return the locally available models, optionally forcing a rescan."""
|
||||
force_rescan = self.to_bool(
|
||||
request.query.get("force") or request.query.get("force_rescan"),
|
||||
False,
|
||||
)
|
||||
try:
|
||||
force_rescan = read_first_bool_field(
|
||||
request.query,
|
||||
("force", "force_rescan"),
|
||||
contract_name="Models request",
|
||||
)
|
||||
except TypeError as exc:
|
||||
return self.web.json_response({"error": str(exc)}, status=400)
|
||||
if force_rescan:
|
||||
self.invalidate_local_hash_match_cache()
|
||||
models = self.get_model_files(force_rescan=force_rescan)
|
||||
return self.web.json_response(models)
|
||||
models = normalize_models(
|
||||
self.get_model_files(force_rescan=force_rescan)
|
||||
)
|
||||
return self.web.json_response(
|
||||
[model.to_dict() for model in models]
|
||||
)
|
||||
|
||||
async def get_path_template_suggestions(self, request):
|
||||
"""Infer download path templates from locally available models."""
|
||||
from ..sources.popular import get_base_models_config
|
||||
|
||||
force_rescan = request.query.get("force") == "1"
|
||||
try:
|
||||
force_rescan = read_first_bool_field(
|
||||
request.query,
|
||||
("force", "force_rescan"),
|
||||
contract_name="Path template suggestions request",
|
||||
)
|
||||
except TypeError as exc:
|
||||
return self.web.json_response({"error": str(exc)}, status=400)
|
||||
models = await self.asyncio.to_thread(
|
||||
self.get_model_files,
|
||||
force_rescan,
|
||||
)
|
||||
models = normalize_models(models)
|
||||
base_models_config = get_base_models_config()
|
||||
suggestions = await self.asyncio.to_thread(
|
||||
self.infer_download_path_templates,
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Protocol
|
||||
|
||||
from ..contracts import ModelCatalogEntry, ModelMatch, SearchResult
|
||||
from ..routes.context import RouteContext
|
||||
|
||||
|
||||
@@ -22,16 +23,8 @@ class SearchTrackerProtocol(Protocol):
|
||||
def update(self, *args: Any, **kwargs: Any) -> Any: ...
|
||||
|
||||
|
||||
class BoolParserProtocol(Protocol):
|
||||
def __call__(self, value: Any, default: bool = False) -> bool: ...
|
||||
|
||||
|
||||
class IntParserProtocol(Protocol):
|
||||
def __call__(self, value: Any, default: int = 0) -> int: ...
|
||||
|
||||
|
||||
class ModelResultBuilderProtocol(Protocol):
|
||||
def __call__(self, source: str, **fields: Any) -> dict[str, Any]: ...
|
||||
def __call__(self, source: str, **fields: Any) -> SearchResult: ...
|
||||
|
||||
|
||||
class ClearCacheProtocol(Protocol):
|
||||
@@ -51,15 +44,19 @@ class FormatSizeProtocol(Protocol):
|
||||
|
||||
|
||||
class SearchProviderProtocol(Protocol):
|
||||
def __call__(self, filename: str, **options: Any) -> Any: ...
|
||||
def __call__(
|
||||
self,
|
||||
filename: str,
|
||||
**options: Any,
|
||||
) -> SearchResult | list[SearchResult] | None: ...
|
||||
|
||||
|
||||
class PopularModelUrlProtocol(Protocol):
|
||||
def __call__(self, filename: str) -> dict[str, Any] | None: ...
|
||||
def __call__(self, filename: str) -> ModelCatalogEntry | None: ...
|
||||
|
||||
|
||||
class ModelListSearchProtocol(Protocol):
|
||||
def __call__(self, filename: str) -> dict[str, Any] | None: ...
|
||||
def __call__(self, filename: str) -> SearchResult | None: ...
|
||||
|
||||
|
||||
class LocalHashSearchProtocol(Protocol):
|
||||
@@ -70,25 +67,29 @@ class LocalHashSearchProtocol(Protocol):
|
||||
category: str | None,
|
||||
max_matches: int,
|
||||
force_rescan: bool,
|
||||
) -> list[dict[str, Any]]: ...
|
||||
) -> list[ModelMatch]: ...
|
||||
|
||||
|
||||
class ResolveUrnProtocol(Protocol):
|
||||
def __call__(self, model_id: Any, version_id: Any) -> dict[str, Any] | None: ...
|
||||
def __call__(
|
||||
self,
|
||||
model_id: int | str,
|
||||
version_id: int | str,
|
||||
) -> dict[str, Any] | None: ...
|
||||
|
||||
|
||||
class ResolveArchiveVersionProtocol(Protocol):
|
||||
def __call__(
|
||||
self,
|
||||
model_id: Any,
|
||||
version_id: Any,
|
||||
model_id: int | str,
|
||||
version_id: int | str,
|
||||
*,
|
||||
query: str,
|
||||
) -> dict[str, Any] | None: ...
|
||||
) -> SearchResult | None: ...
|
||||
|
||||
|
||||
class GetDownloadUrlProtocol(Protocol):
|
||||
def __call__(self, version_id: Any) -> str | None: ...
|
||||
def __call__(self, version_id: int | str) -> str | None: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@@ -120,8 +121,6 @@ class SearchDependencies:
|
||||
search_local_matches_by_hash: LocalHashSearchProtocol
|
||||
search_lora_manager_archive_for_file: SearchProviderProtocol
|
||||
search_model_list: ModelListSearchProtocol
|
||||
to_bool: BoolParserProtocol
|
||||
to_int: IntParserProtocol
|
||||
web: Any
|
||||
|
||||
@staticmethod
|
||||
@@ -184,7 +183,5 @@ class SearchDependencies:
|
||||
"search_lora_manager_archive_for_file"
|
||||
),
|
||||
search_model_list=context.require("search_model_list"),
|
||||
to_bool=context.require("to_bool"),
|
||||
to_int=context.require("to_int"),
|
||||
web=context.require("web"),
|
||||
)
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
"""Request orchestration for model source searches."""
|
||||
|
||||
from ..contracts import SearchResult
|
||||
from ..local_hash_matches import collect_local_hash_matches_for_result
|
||||
from ..request_utils import extract_request_sha256
|
||||
from ..request_utils import read_optional_object_payload
|
||||
from ..routes.context import RouteContext
|
||||
from .search_cache import SearchResultCache
|
||||
from .search_dependencies import SearchDependencies
|
||||
@@ -53,8 +54,6 @@ class SearchOrchestrator:
|
||||
dependencies.search_lora_manager_archive_for_file
|
||||
)
|
||||
self.search_model_list = dependencies.search_model_list
|
||||
self.to_bool = dependencies.to_bool
|
||||
self.to_int = dependencies.to_int
|
||||
self.web = dependencies.web
|
||||
self.search_cache = SearchResultCache(self.search_result_timestamps)
|
||||
self.provider_runner = SearchProviderRunner(self)
|
||||
@@ -83,6 +82,35 @@ class SearchOrchestrator:
|
||||
parts.append(f"{key}={formatted}")
|
||||
return " ".join(parts)
|
||||
|
||||
@staticmethod
|
||||
def normalize_search_result(source_key, result):
|
||||
"""Validate that a provider returned the shared result contract."""
|
||||
if result is None or isinstance(result, SearchResult):
|
||||
return result
|
||||
if isinstance(result, list) and all(
|
||||
isinstance(item, SearchResult) for item in result
|
||||
):
|
||||
return result
|
||||
raise TypeError(
|
||||
f"Search source '{source_key}' returned an untyped result: "
|
||||
f"{type(result).__name__}"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def serialize_search_result(result):
|
||||
"""Serialize canonical results at the HTTP/cache boundary."""
|
||||
if result is None:
|
||||
return None
|
||||
if isinstance(result, SearchResult):
|
||||
return result.to_dict()
|
||||
if isinstance(result, list) and all(
|
||||
isinstance(item, SearchResult) for item in result
|
||||
):
|
||||
return [item.to_dict() for item in result]
|
||||
raise TypeError(
|
||||
f"Cannot serialize untyped search result: {type(result).__name__}"
|
||||
)
|
||||
|
||||
def normalize_result_extra(self, extra):
|
||||
if not extra:
|
||||
return {}
|
||||
@@ -100,6 +128,8 @@ class SearchOrchestrator:
|
||||
return normalized
|
||||
|
||||
def format_result_details(self, result, extra=None):
|
||||
if isinstance(result, SearchResult):
|
||||
result = result.to_dict()
|
||||
if isinstance(result, list):
|
||||
return self.format_log_fields(count=len(result))
|
||||
if not isinstance(result, dict):
|
||||
@@ -136,62 +166,27 @@ class SearchOrchestrator:
|
||||
async def search_sources(self, request):
|
||||
"""Execute all requested sources and return a JSON response."""
|
||||
try:
|
||||
data = await request.json()
|
||||
filename = str(data.get("filename", "") or "").strip()
|
||||
sha256 = extract_request_sha256(
|
||||
data,
|
||||
keys=("sha256", "hash", "file_hash"),
|
||||
)
|
||||
category = data.get("category", "")
|
||||
base_model_context = data.get("base_model_context", "")
|
||||
progress_id = str(data.get("progress_id") or "").strip()
|
||||
progress_source = str(data.get("progress_source") or "").strip()
|
||||
civitai_candidate_limit = self.to_int(
|
||||
data.get("civitai_candidate_limit"),
|
||||
5,
|
||||
)
|
||||
civitai_candidate_limit = max(1, min(civitai_candidate_limit, 20))
|
||||
civarchive_candidate_limit = self.to_int(
|
||||
data.get("civarchive_candidate_limit"),
|
||||
10,
|
||||
)
|
||||
civarchive_candidate_limit = max(
|
||||
1,
|
||||
min(civarchive_candidate_limit, 30),
|
||||
)
|
||||
|
||||
is_urn = self.to_bool(data.get("is_urn", False), False)
|
||||
civitai_key = data.get("civitai_key", "")
|
||||
civitai_session_token = data.get("civitai_session_token", "")
|
||||
hf_token = data.get("hf_token", "")
|
||||
brave_search_api_key = data.get("brave_search_api_key", "")
|
||||
civitai_use_trpc_search = self.to_bool(
|
||||
data.get("civitai_use_trpc_search", True),
|
||||
True,
|
||||
)
|
||||
civitai_use_api_search = self.to_bool(
|
||||
data.get("civitai_use_api_search", True),
|
||||
True,
|
||||
)
|
||||
civitai_use_html_fallback = self.to_bool(
|
||||
data.get("civitai_use_html_fallback", True),
|
||||
True,
|
||||
)
|
||||
hf_use_api_search = self.to_bool(
|
||||
data.get("hf_use_api_search", True),
|
||||
True,
|
||||
)
|
||||
hf_use_comfy_org_fallback = self.to_bool(
|
||||
data.get("hf_use_comfy_org_fallback", True),
|
||||
True,
|
||||
)
|
||||
hf_use_brave_fallback = self.to_bool(
|
||||
data.get("hf_use_brave_fallback", True),
|
||||
True,
|
||||
)
|
||||
|
||||
model_id = data.get("model_id")
|
||||
version_id = data.get("version_id")
|
||||
data = await read_optional_object_payload(request)
|
||||
try:
|
||||
search_request = SearchRequest.from_mapping(
|
||||
data,
|
||||
)
|
||||
except (TypeError, ValueError) as exc:
|
||||
return self.web.json_response(
|
||||
{"error": str(exc)},
|
||||
status=400,
|
||||
)
|
||||
filename = search_request.filename
|
||||
sha256 = search_request.sha256
|
||||
category = search_request.category
|
||||
base_model_context = search_request.base_model_context
|
||||
progress_id = search_request.progress_id
|
||||
progress_source = search_request.progress_source
|
||||
model_id = search_request.model_id
|
||||
version_id = search_request.version_id
|
||||
is_urn = search_request.is_urn
|
||||
normalized_sources = search_request.normalized_sources
|
||||
force_search = search_request.force_search
|
||||
if not filename and not sha256 and not (is_urn and model_id and version_id):
|
||||
return self.web.json_response(
|
||||
{
|
||||
@@ -203,61 +198,6 @@ class SearchOrchestrator:
|
||||
status=400,
|
||||
)
|
||||
|
||||
raw_sources = data.get("sources", ["all"])
|
||||
if isinstance(raw_sources, str):
|
||||
raw_sources = [raw_sources]
|
||||
elif not isinstance(raw_sources, list):
|
||||
raw_sources = ["all"]
|
||||
|
||||
normalized_sources = frozenset(
|
||||
str(source).strip().lower()
|
||||
for source in raw_sources
|
||||
if str(source).strip()
|
||||
)
|
||||
if not normalized_sources:
|
||||
normalized_sources = frozenset({"all"})
|
||||
if "all" in normalized_sources:
|
||||
normalized_sources = frozenset(
|
||||
{
|
||||
"local",
|
||||
"huggingface",
|
||||
"civitai",
|
||||
"civarchive",
|
||||
"lora_manager_archive",
|
||||
}
|
||||
)
|
||||
|
||||
if not progress_source:
|
||||
progress_source = (
|
||||
next(iter(normalized_sources))
|
||||
if len(normalized_sources) == 1
|
||||
else "all"
|
||||
)
|
||||
force_search = self.to_bool(data.get("force_search"), False)
|
||||
search_request = SearchRequest(
|
||||
data=data,
|
||||
filename=filename,
|
||||
category=category,
|
||||
base_model_context=base_model_context,
|
||||
progress_id=progress_id,
|
||||
progress_source=progress_source,
|
||||
civitai_candidate_limit=civitai_candidate_limit,
|
||||
civarchive_candidate_limit=civarchive_candidate_limit,
|
||||
is_urn=is_urn,
|
||||
civitai_key=civitai_key,
|
||||
civitai_session_token=civitai_session_token,
|
||||
hf_token=hf_token,
|
||||
brave_search_api_key=brave_search_api_key,
|
||||
civitai_use_trpc_search=civitai_use_trpc_search,
|
||||
civitai_use_api_search=civitai_use_api_search,
|
||||
civitai_use_html_fallback=civitai_use_html_fallback,
|
||||
hf_use_api_search=hf_use_api_search,
|
||||
hf_use_comfy_org_fallback=hf_use_comfy_org_fallback,
|
||||
hf_use_brave_fallback=hf_use_brave_fallback,
|
||||
force_search=force_search,
|
||||
normalized_sources=normalized_sources,
|
||||
sha256=sha256,
|
||||
)
|
||||
raise_if_search_cancelled = (
|
||||
self.provider_runner.raise_if_search_cancelled
|
||||
)
|
||||
@@ -334,10 +274,17 @@ class SearchOrchestrator:
|
||||
def iter_result_items(result):
|
||||
if isinstance(result, list):
|
||||
for item in result:
|
||||
if isinstance(item, dict):
|
||||
yield item
|
||||
elif isinstance(result, dict):
|
||||
yield result
|
||||
if not isinstance(item, SearchResult):
|
||||
raise TypeError(
|
||||
"Search result lists must contain SearchResult values"
|
||||
)
|
||||
yield item.to_dict()
|
||||
elif isinstance(result, SearchResult):
|
||||
yield result.to_dict()
|
||||
elif result is not None:
|
||||
raise TypeError(
|
||||
"Search result sources must contain SearchResult values"
|
||||
)
|
||||
|
||||
def collect_local_hash_matches(payload):
|
||||
matches = []
|
||||
@@ -405,12 +352,10 @@ class SearchOrchestrator:
|
||||
continue
|
||||
|
||||
for match in hash_matches:
|
||||
model_path = (
|
||||
match.get("model", {}).get("path")
|
||||
or match.get("path")
|
||||
or ""
|
||||
model_path = match.model.path or str(
|
||||
match.extra_value("path") or ""
|
||||
)
|
||||
path_key = model_path.lower()
|
||||
path_key = model_path.casefold()
|
||||
if path_key and path_key in seen_match_paths:
|
||||
continue
|
||||
if path_key:
|
||||
@@ -451,14 +396,20 @@ class SearchOrchestrator:
|
||||
elif source_key == "source_status":
|
||||
results["source_status"].update(source_result or {})
|
||||
elif source_result:
|
||||
results[source_key] = source_result
|
||||
results[source_key] = self.normalize_search_result(
|
||||
source_key,
|
||||
source_result,
|
||||
)
|
||||
if source_found:
|
||||
results["found"] = True
|
||||
|
||||
raise_if_search_cancelled(search_request, progress_source)
|
||||
results["local_hash_matches"] = collect_local_hash_matches(results)
|
||||
if results["local_hash_matches"]:
|
||||
local_hash_matches = collect_local_hash_matches(results)
|
||||
if local_hash_matches:
|
||||
results["found"] = True
|
||||
results["local_hash_matches"] = [
|
||||
match.to_dict() for match in local_hash_matches
|
||||
]
|
||||
raise_if_search_cancelled(search_request, progress_source)
|
||||
|
||||
self.logger.info(
|
||||
@@ -473,6 +424,17 @@ class SearchOrchestrator:
|
||||
100,
|
||||
status="completed",
|
||||
)
|
||||
for source_key in (
|
||||
"popular",
|
||||
"model_list",
|
||||
"huggingface",
|
||||
"civitai",
|
||||
"civarchive",
|
||||
"lora_manager_archive",
|
||||
):
|
||||
results[source_key] = self.serialize_search_result(
|
||||
results[source_key]
|
||||
)
|
||||
self.search_cache.stamp_results(
|
||||
results,
|
||||
force_search=force_search,
|
||||
|
||||
@@ -1,8 +1,17 @@
|
||||
"""Provider-specific search execution and progress handling."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from ..contracts import SearchResult
|
||||
from ..request_utils import (
|
||||
extract_request_sha256,
|
||||
read_bool_field,
|
||||
read_identifier_field,
|
||||
read_int_field,
|
||||
read_text_field,
|
||||
)
|
||||
from ..sources.civarchive import build_civarchive_failure_status
|
||||
from ..sources.civitai import build_civitai_result_payload
|
||||
from ..type_utils import select_primary_model_file
|
||||
@@ -16,7 +25,6 @@ class SearchCancelled(BaseException):
|
||||
class SearchRequest:
|
||||
"""Normalized values shared by the provider search tasks."""
|
||||
|
||||
data: dict[str, Any]
|
||||
filename: str
|
||||
category: str
|
||||
base_model_context: str
|
||||
@@ -38,6 +46,240 @@ class SearchRequest:
|
||||
force_search: bool
|
||||
normalized_sources: frozenset[str]
|
||||
sha256: str = ""
|
||||
model_id: int | str | None = None
|
||||
version_id: int | str | None = None
|
||||
|
||||
_TEXT_FIELDS = (
|
||||
"filename",
|
||||
"category",
|
||||
"base_model_context",
|
||||
"progress_id",
|
||||
"progress_source",
|
||||
"civitai_key",
|
||||
"civitai_session_token",
|
||||
"hf_token",
|
||||
"brave_search_api_key",
|
||||
"sha256",
|
||||
)
|
||||
_BOOLEAN_FIELDS = (
|
||||
"is_urn",
|
||||
"civitai_use_trpc_search",
|
||||
"civitai_use_api_search",
|
||||
"civitai_use_html_fallback",
|
||||
"hf_use_api_search",
|
||||
"hf_use_comfy_org_fallback",
|
||||
"hf_use_brave_fallback",
|
||||
"force_search",
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_mapping(
|
||||
cls,
|
||||
value: Mapping[str, Any],
|
||||
) -> "SearchRequest":
|
||||
"""Adapt one HTTP payload into the typed provider request."""
|
||||
if not isinstance(value, Mapping):
|
||||
raise TypeError("Search request must be an object")
|
||||
data = dict(value)
|
||||
|
||||
filename = read_text_field(
|
||||
data,
|
||||
"filename",
|
||||
contract_name="SearchRequest",
|
||||
)
|
||||
sha256 = extract_request_sha256(
|
||||
data,
|
||||
keys=("sha256", "hash", "file_hash"),
|
||||
)
|
||||
category = read_text_field(
|
||||
data,
|
||||
"category",
|
||||
contract_name="SearchRequest",
|
||||
)
|
||||
base_model_context = read_text_field(
|
||||
data,
|
||||
"base_model_context",
|
||||
contract_name="SearchRequest",
|
||||
)
|
||||
progress_id = read_text_field(
|
||||
data,
|
||||
"progress_id",
|
||||
contract_name="SearchRequest",
|
||||
)
|
||||
progress_source = read_text_field(
|
||||
data,
|
||||
"progress_source",
|
||||
contract_name="SearchRequest",
|
||||
)
|
||||
|
||||
civitai_candidate_limit = read_int_field(
|
||||
data,
|
||||
"civitai_candidate_limit",
|
||||
default=5,
|
||||
contract_name="SearchRequest",
|
||||
)
|
||||
civarchive_candidate_limit = read_int_field(
|
||||
data,
|
||||
"civarchive_candidate_limit",
|
||||
default=10,
|
||||
contract_name="SearchRequest",
|
||||
)
|
||||
civitai_candidate_limit = max(1, min(civitai_candidate_limit, 20))
|
||||
civarchive_candidate_limit = max(1, min(civarchive_candidate_limit, 30))
|
||||
|
||||
def read_request_bool(field_name: str, default: bool) -> bool:
|
||||
return read_bool_field(
|
||||
data,
|
||||
field_name,
|
||||
default=default,
|
||||
contract_name="SearchRequest",
|
||||
)
|
||||
|
||||
is_urn = read_request_bool("is_urn", False)
|
||||
raw_sources = data.get("sources", ["all"])
|
||||
if isinstance(raw_sources, str):
|
||||
raw_sources = [raw_sources]
|
||||
elif raw_sources is None or not isinstance(raw_sources, list):
|
||||
raise TypeError("SearchRequest sources must be a string or array")
|
||||
if any(not isinstance(source, str) for source in raw_sources):
|
||||
raise TypeError("SearchRequest sources must contain only strings")
|
||||
|
||||
normalized_sources = frozenset(
|
||||
source.strip().lower()
|
||||
for source in raw_sources
|
||||
if source.strip()
|
||||
)
|
||||
if not normalized_sources:
|
||||
normalized_sources = frozenset({"all"})
|
||||
if "all" in normalized_sources:
|
||||
normalized_sources = frozenset(
|
||||
{
|
||||
"local",
|
||||
"huggingface",
|
||||
"civitai",
|
||||
"civarchive",
|
||||
"lora_manager_archive",
|
||||
}
|
||||
)
|
||||
|
||||
if not progress_source:
|
||||
progress_source = (
|
||||
next(iter(normalized_sources))
|
||||
if len(normalized_sources) == 1
|
||||
else "all"
|
||||
)
|
||||
|
||||
return cls(
|
||||
filename=filename,
|
||||
category=category,
|
||||
base_model_context=base_model_context,
|
||||
progress_id=progress_id,
|
||||
progress_source=progress_source,
|
||||
civitai_candidate_limit=civitai_candidate_limit,
|
||||
civarchive_candidate_limit=civarchive_candidate_limit,
|
||||
is_urn=is_urn,
|
||||
civitai_key=read_text_field(
|
||||
data,
|
||||
"civitai_key",
|
||||
contract_name="SearchRequest",
|
||||
),
|
||||
civitai_session_token=read_text_field(
|
||||
data,
|
||||
"civitai_session_token",
|
||||
contract_name="SearchRequest",
|
||||
),
|
||||
hf_token=read_text_field(
|
||||
data,
|
||||
"hf_token",
|
||||
contract_name="SearchRequest",
|
||||
),
|
||||
brave_search_api_key=read_text_field(
|
||||
data,
|
||||
"brave_search_api_key",
|
||||
contract_name="SearchRequest",
|
||||
),
|
||||
civitai_use_trpc_search=read_request_bool(
|
||||
"civitai_use_trpc_search",
|
||||
True,
|
||||
),
|
||||
civitai_use_api_search=read_request_bool(
|
||||
"civitai_use_api_search",
|
||||
True,
|
||||
),
|
||||
civitai_use_html_fallback=read_request_bool(
|
||||
"civitai_use_html_fallback",
|
||||
True,
|
||||
),
|
||||
hf_use_api_search=read_request_bool("hf_use_api_search", True),
|
||||
hf_use_comfy_org_fallback=read_request_bool(
|
||||
"hf_use_comfy_org_fallback",
|
||||
True,
|
||||
),
|
||||
hf_use_brave_fallback=read_request_bool(
|
||||
"hf_use_brave_fallback",
|
||||
True,
|
||||
),
|
||||
force_search=read_request_bool("force_search", False),
|
||||
normalized_sources=normalized_sources,
|
||||
sha256=sha256,
|
||||
model_id=read_identifier_field(
|
||||
data,
|
||||
"model_id",
|
||||
contract_name="SearchRequest",
|
||||
),
|
||||
version_id=read_identifier_field(
|
||||
data,
|
||||
"version_id",
|
||||
contract_name="SearchRequest",
|
||||
),
|
||||
)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
"""Validate the normalized request before provider execution."""
|
||||
for field_name in self._TEXT_FIELDS:
|
||||
value = getattr(self, field_name)
|
||||
if value is None:
|
||||
value = ""
|
||||
if not isinstance(value, str):
|
||||
raise TypeError(f"SearchRequest {field_name} must be a string")
|
||||
object.__setattr__(self, field_name, value)
|
||||
|
||||
for field_name in self._BOOLEAN_FIELDS:
|
||||
value = getattr(self, field_name)
|
||||
if not isinstance(value, bool):
|
||||
raise TypeError(f"SearchRequest {field_name} must be a boolean")
|
||||
|
||||
for field_name in (
|
||||
"civitai_candidate_limit",
|
||||
"civarchive_candidate_limit",
|
||||
):
|
||||
value = getattr(self, field_name)
|
||||
if isinstance(value, bool) or not isinstance(value, int):
|
||||
raise TypeError(
|
||||
f"SearchRequest {field_name} must be a positive integer"
|
||||
)
|
||||
if value < 1:
|
||||
raise ValueError(
|
||||
f"SearchRequest {field_name} must be a positive integer"
|
||||
)
|
||||
|
||||
for field_name in ("model_id", "version_id"):
|
||||
value = getattr(self, field_name)
|
||||
if value is None:
|
||||
continue
|
||||
if isinstance(value, bool) or not isinstance(value, (int, str)):
|
||||
raise TypeError(
|
||||
f"SearchRequest {field_name} must be an integer or string"
|
||||
)
|
||||
if isinstance(value, str):
|
||||
object.__setattr__(self, field_name, value.strip() or None)
|
||||
|
||||
if not isinstance(self.normalized_sources, frozenset):
|
||||
raise TypeError("SearchRequest normalized_sources must be a frozenset")
|
||||
if any(not isinstance(source, str) for source in self.normalized_sources):
|
||||
raise TypeError(
|
||||
"SearchRequest normalized_sources must contain only strings"
|
||||
)
|
||||
|
||||
@property
|
||||
def search_local(self):
|
||||
@@ -171,19 +413,22 @@ class SearchProviderRunner:
|
||||
raise
|
||||
|
||||
def mark_any_model_fallback(self, request: SearchRequest, result):
|
||||
if result is None:
|
||||
return None
|
||||
if isinstance(result, list):
|
||||
return [
|
||||
self.mark_any_model_fallback(request, item)
|
||||
for item in result
|
||||
]
|
||||
if not isinstance(result, dict):
|
||||
return result
|
||||
|
||||
marked = dict(result)
|
||||
marked["any_model_match"] = True
|
||||
marked["base_model_fallback"] = True
|
||||
marked["requested_base_model"] = request.base_model_context
|
||||
return marked
|
||||
if isinstance(result, SearchResult):
|
||||
return result.with_extra(
|
||||
any_model_match=True,
|
||||
base_model_fallback=True,
|
||||
requested_base_model=request.base_model_context,
|
||||
)
|
||||
raise TypeError(
|
||||
f"{type(result).__name__} is not a typed search result"
|
||||
)
|
||||
|
||||
def execute_search_with_fallback(
|
||||
self,
|
||||
@@ -248,7 +493,6 @@ class SearchProviderRunner:
|
||||
return source_results, source_found
|
||||
|
||||
popular_info = self.owner.get_popular_model_url(request.filename)
|
||||
self.owner.log_search_result("popular", popular_info)
|
||||
self.owner.search_tracker.update(
|
||||
request.progress_id,
|
||||
"local",
|
||||
@@ -262,30 +506,44 @@ class SearchProviderRunner:
|
||||
model_list_result,
|
||||
{
|
||||
"confidence": (
|
||||
model_list_result.get("confidence")
|
||||
model_list_result.confidence
|
||||
if model_list_result
|
||||
else None
|
||||
)
|
||||
},
|
||||
)
|
||||
if popular_info:
|
||||
popular_result = {
|
||||
"source": "popular",
|
||||
"filename": request.filename,
|
||||
**popular_info,
|
||||
}
|
||||
popular_result = self.owner.build_model_result(
|
||||
"popular",
|
||||
filename=request.filename,
|
||||
name=popular_info.name or request.filename,
|
||||
type=popular_info.model_type,
|
||||
url=(
|
||||
popular_info.url
|
||||
or popular_info.download_url
|
||||
or ""
|
||||
),
|
||||
download_url=popular_info.download_url,
|
||||
size=popular_info.size,
|
||||
match_type="exact",
|
||||
directory=popular_info.directory,
|
||||
**dict(popular_info.extra),
|
||||
)
|
||||
if (
|
||||
model_list_result
|
||||
and model_list_result.get("filename", "").lower()
|
||||
and model_list_result.filename.lower()
|
||||
== request.filename.lower()
|
||||
and model_list_result.get("size")
|
||||
and model_list_result.size is not None
|
||||
):
|
||||
popular_result["size"] = model_list_result.get("size")
|
||||
popular_result = popular_result.with_updates(
|
||||
size=model_list_result.size
|
||||
)
|
||||
source_results["popular"] = popular_result
|
||||
self.owner.log_search_result("popular", popular_result)
|
||||
source_found = True
|
||||
|
||||
if model_list_result:
|
||||
confidence = model_list_result.get("confidence", 0)
|
||||
confidence = model_list_result.confidence
|
||||
if (request.is_urn and confidence >= 70) or not request.is_urn:
|
||||
source_results["model_list"] = model_list_result
|
||||
source_found = True
|
||||
@@ -340,8 +598,8 @@ class SearchProviderRunner:
|
||||
source_found = False
|
||||
|
||||
if request.is_urn:
|
||||
model_id = request.data.get("model_id")
|
||||
version_id = request.data.get("version_id")
|
||||
model_id = request.model_id
|
||||
version_id = request.version_id
|
||||
|
||||
if model_id and version_id:
|
||||
self.owner.search_tracker.update(
|
||||
@@ -418,20 +676,7 @@ class SearchProviderRunner:
|
||||
{"results_count": len(civitai_results)},
|
||||
)
|
||||
if civitai_results:
|
||||
first_result = civitai_results[0]
|
||||
source_results["civitai"] = self.owner.build_model_result(
|
||||
"civitai",
|
||||
model_id=first_result.get("model_id"),
|
||||
version_id=first_result.get("version_id"),
|
||||
name=first_result.get("name"),
|
||||
filename=first_result.get("filename"),
|
||||
type=first_result.get("type"),
|
||||
download_url=first_result.get("download_url"),
|
||||
url=first_result.get("url"),
|
||||
size=first_result.get("size"),
|
||||
base_model=first_result.get("base_model"),
|
||||
tags=first_result.get("tags", []),
|
||||
)
|
||||
source_results["civitai"] = civitai_results[0]
|
||||
source_found = True
|
||||
else:
|
||||
civitai_result = self.execute_search_with_fallback(
|
||||
@@ -478,8 +723,8 @@ class SearchProviderRunner:
|
||||
source_found = False
|
||||
|
||||
if request.is_urn:
|
||||
model_id = request.data.get("model_id")
|
||||
version_id = request.data.get("version_id")
|
||||
model_id = request.model_id
|
||||
version_id = request.version_id
|
||||
if model_id and version_id:
|
||||
self.owner.search_tracker.update(
|
||||
request.progress_id,
|
||||
@@ -628,8 +873,8 @@ class SearchProviderRunner:
|
||||
or getattr(request, "sha256", "")
|
||||
or (
|
||||
request.is_urn
|
||||
and request.data.get("model_id")
|
||||
and request.data.get("version_id")
|
||||
and request.model_id
|
||||
and request.version_id
|
||||
)
|
||||
):
|
||||
tasks.append(
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
"""Search cache, credentials, and index operations for HTTP routes."""
|
||||
|
||||
from ..request_utils import (
|
||||
read_bool_field,
|
||||
read_optional_object_payload,
|
||||
read_text_field,
|
||||
)
|
||||
from ..routes.context import RouteContext
|
||||
|
||||
|
||||
@@ -56,10 +61,16 @@ class SearchSupportService:
|
||||
log_name,
|
||||
):
|
||||
try:
|
||||
data = await request.json()
|
||||
value = data.get(payload_key, "")
|
||||
data = await read_optional_object_payload(request)
|
||||
value = read_text_field(
|
||||
data,
|
||||
payload_key,
|
||||
contract_name=f"{log_name} request",
|
||||
)
|
||||
result = await self.asyncio.to_thread(check_func, value)
|
||||
return self.web.json_response(result)
|
||||
except TypeError as error:
|
||||
return self.web.json_response({"error": str(error)}, status=400)
|
||||
except Exception as error:
|
||||
self.extension.logger.exception(f"{log_name} check error: {error}")
|
||||
return self.web.json_response({"error": str(error)}, status=500)
|
||||
@@ -108,8 +119,15 @@ class SearchSupportService:
|
||||
|
||||
async def refresh_author_index(self, request):
|
||||
"""Refresh HuggingFace author fallback index."""
|
||||
data = await request.json()
|
||||
hf_token = data.get("hf_token", "")
|
||||
data = await read_optional_object_payload(request)
|
||||
try:
|
||||
hf_token = read_text_field(
|
||||
data,
|
||||
"hf_token",
|
||||
contract_name="Author index refresh request",
|
||||
)
|
||||
except TypeError as error:
|
||||
return self.web.json_response({"error": str(error)}, status=400)
|
||||
result = await self.asyncio.to_thread(
|
||||
self.refresh_known_author_fallback_indexes,
|
||||
hf_token or None,
|
||||
@@ -119,11 +137,14 @@ class SearchSupportService:
|
||||
|
||||
async def model_list_status(self, request):
|
||||
"""Return local model-list status and optionally compare with GitHub."""
|
||||
check_remote = str(request.query.get("check_remote", "")).lower() in {
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
}
|
||||
try:
|
||||
check_remote = read_bool_field(
|
||||
request.query,
|
||||
"check_remote",
|
||||
contract_name="Model list status request",
|
||||
)
|
||||
except TypeError as exc:
|
||||
return self.web.json_response({"error": str(exc)}, status=400)
|
||||
return self.web.json_response(
|
||||
self.get_model_list_update_status(check_remote=check_remote)
|
||||
)
|
||||
|
||||
@@ -1,6 +1,13 @@
|
||||
"""Workflow analysis service."""
|
||||
|
||||
from ..request_utils import validate_workflow_payload
|
||||
from ..contracts import MissingModel, Resolution, WorkflowAnalysisResult
|
||||
from ..custom_nodes import get_custom_node_resolution_metadata
|
||||
from ..request_utils import (
|
||||
read_bool_field,
|
||||
read_optional_object_payload,
|
||||
read_text_field,
|
||||
validate_workflow_payload,
|
||||
)
|
||||
from ..routes.context import RouteContext
|
||||
|
||||
|
||||
@@ -25,7 +32,6 @@ class WorkflowService:
|
||||
self.should_skip_existing_custom_node_reference = context.require(
|
||||
"should_skip_existing_custom_node_reference"
|
||||
)
|
||||
self.to_bool = context.require("to_bool")
|
||||
self.web = context.require("web")
|
||||
|
||||
async def analyze_workflow(self, request):
|
||||
@@ -39,15 +45,25 @@ class WorkflowService:
|
||||
invalidate_local_hash_match_cache = self.invalidate_local_hash_match_cache
|
||||
search_model_list = self.search_model_list
|
||||
should_skip_existing_custom_node_reference = self.should_skip_existing_custom_node_reference
|
||||
to_bool = self.to_bool
|
||||
web = self.web
|
||||
try:
|
||||
data = await request.json()
|
||||
data = await read_optional_object_payload(request)
|
||||
workflow_json, workflow_error = validate_workflow_payload(
|
||||
data.get("workflow")
|
||||
)
|
||||
analysis_id = str(data.get("analysis_id") or "").strip()
|
||||
force_rescan = to_bool(data.get("force_rescan"), False)
|
||||
try:
|
||||
analysis_id = read_text_field(
|
||||
data,
|
||||
"analysis_id",
|
||||
contract_name="Workflow analysis request",
|
||||
)
|
||||
force_rescan = read_bool_field(
|
||||
data,
|
||||
"force_rescan",
|
||||
contract_name="Workflow analysis request",
|
||||
)
|
||||
except TypeError as exc:
|
||||
return web.json_response({"error": str(exc)}, status=400)
|
||||
if force_rescan:
|
||||
invalidate_local_hash_match_cache()
|
||||
|
||||
@@ -72,7 +88,7 @@ class WorkflowService:
|
||||
self._update_analysis_progress(analysis_id, payload)
|
||||
|
||||
# Analyze and find matches
|
||||
result = await asyncio.to_thread(
|
||||
analysis_result = await asyncio.to_thread(
|
||||
analyze_and_find_matches,
|
||||
workflow_json,
|
||||
0.0,
|
||||
@@ -82,38 +98,36 @@ class WorkflowService:
|
||||
analysis_id=analysis_id,
|
||||
)
|
||||
|
||||
missing_models = result.get("missing_models", [])
|
||||
filtered_missing = []
|
||||
for missing in missing_models:
|
||||
name = missing.get("name") or missing.get("original_path", "")
|
||||
if should_skip_existing_custom_node_reference(missing):
|
||||
filtered_missing: list[MissingModel] = []
|
||||
for missing in analysis_result.missing_models:
|
||||
name = (
|
||||
missing.extra_value("name")
|
||||
or missing.reference.original_path
|
||||
)
|
||||
if should_skip_existing_custom_node_reference(missing.reference):
|
||||
self.logger.info(
|
||||
"Filtered existing custom-node model "
|
||||
f"reference: {name}"
|
||||
)
|
||||
continue
|
||||
filtered_missing.append(missing)
|
||||
result["missing_models"] = filtered_missing
|
||||
result["total_missing"] = len(filtered_missing)
|
||||
|
||||
# If download available, check for download sources only from LOCAL sources
|
||||
# (workflow_url, popular, model-list.json) - skip automatic online search
|
||||
# Online search is now only triggered on-demand via search button
|
||||
if download_available:
|
||||
for missing in result.get("missing_models", []):
|
||||
for index, missing in enumerate(filtered_missing):
|
||||
# Check if there's a 100% local match
|
||||
matches = missing.get("matches", [])
|
||||
has_perfect_match = any(
|
||||
m.get("confidence", 0) == 100 for m in matches
|
||||
)
|
||||
matches = missing.matches or ()
|
||||
has_perfect_match = any(match.confidence == 100 for match in matches)
|
||||
|
||||
if not has_perfect_match:
|
||||
filename = get_filename_from_path(
|
||||
missing.get("original_path", "")
|
||||
missing.reference.original_path
|
||||
)
|
||||
|
||||
# 0. Check workflow URL first (highest priority - directly from workflow)
|
||||
workflow_url = missing.get("workflow_url", "")
|
||||
workflow_url = missing.extra_value("workflow_url", "")
|
||||
if workflow_url:
|
||||
# Determine source from URL
|
||||
if "huggingface.co" in workflow_url:
|
||||
@@ -126,21 +140,24 @@ class WorkflowService:
|
||||
# Try to get file size using cached remote helper
|
||||
file_size = fetch_remote_file_size_cached(workflow_url, timeout=5)
|
||||
|
||||
missing["download_source"] = {
|
||||
"source": source,
|
||||
"url": workflow_url,
|
||||
"model_url": missing.get(
|
||||
"workflow_model_url", workflow_url
|
||||
),
|
||||
"filename": filename,
|
||||
"directory": missing.get(
|
||||
"workflow_directory", ""
|
||||
)
|
||||
or missing.get("category", "checkpoints"),
|
||||
"match_type": "exact",
|
||||
"url_source": "workflow",
|
||||
"size": file_size,
|
||||
}
|
||||
filtered_missing[index] = missing.with_extra(
|
||||
download_source={
|
||||
"source": source,
|
||||
"url": workflow_url,
|
||||
"model_url": missing.extra_value(
|
||||
"workflow_model_url", workflow_url
|
||||
),
|
||||
"filename": filename,
|
||||
"directory": missing.extra_value(
|
||||
"workflow_directory", ""
|
||||
)
|
||||
or missing.reference.category
|
||||
or "checkpoints",
|
||||
"match_type": "exact",
|
||||
"url_source": "workflow",
|
||||
"size": file_size,
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
# 1. Check popular models (always exact match)
|
||||
@@ -149,20 +166,27 @@ class WorkflowService:
|
||||
popular_model_list_result = search_model_list(
|
||||
filename, exact_only=True
|
||||
)
|
||||
missing["download_source"] = {
|
||||
"source": "popular",
|
||||
"url": popular_info.get("url"),
|
||||
"filename": filename,
|
||||
"type": popular_info.get("type"),
|
||||
"directory": popular_info.get("directory"),
|
||||
"size": (
|
||||
popular_model_list_result.get("size")
|
||||
filtered_missing[index] = missing.with_extra(
|
||||
download_source={
|
||||
"source": "popular",
|
||||
"url": popular_info.url
|
||||
or popular_info.download_url,
|
||||
"filename": popular_info.filename or filename,
|
||||
"type": popular_info.model_type,
|
||||
"directory": popular_info.directory,
|
||||
"size": (
|
||||
popular_model_list_result.size
|
||||
if popular_model_list_result
|
||||
and popular_model_list_result.size
|
||||
is not None
|
||||
else None
|
||||
)
|
||||
if popular_model_list_result
|
||||
else None
|
||||
)
|
||||
or popular_info.get("size"),
|
||||
"match_type": "exact",
|
||||
}
|
||||
and popular_model_list_result.size is not None
|
||||
else popular_info.size,
|
||||
"match_type": "exact",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
# 2. Check model list (ComfyUI Manager database)
|
||||
@@ -171,38 +195,46 @@ class WorkflowService:
|
||||
filename, exact_only=True
|
||||
)
|
||||
if model_list_result:
|
||||
missing["download_source"] = {
|
||||
"source": "model_list",
|
||||
"url": model_list_result.get("url"),
|
||||
"filename": model_list_result.get("filename"),
|
||||
"name": model_list_result.get("name"),
|
||||
"type": model_list_result.get("type"),
|
||||
"directory": model_list_result.get("directory"),
|
||||
"size": model_list_result.get("size"),
|
||||
"match_type": model_list_result.get(
|
||||
"match_type"
|
||||
),
|
||||
"confidence": model_list_result.get(
|
||||
"confidence"
|
||||
),
|
||||
}
|
||||
filtered_missing[index] = missing.with_extra(
|
||||
download_source={
|
||||
"source": "model_list",
|
||||
"url": model_list_result.url,
|
||||
"filename": model_list_result.filename,
|
||||
"name": model_list_result.name,
|
||||
"type": model_list_result.model_type,
|
||||
"directory": model_list_result.extra_value(
|
||||
"directory"
|
||||
),
|
||||
"size": model_list_result.size,
|
||||
"match_type": model_list_result.match_type,
|
||||
"confidence": model_list_result.confidence,
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
# NOTE: Search for online sources (HuggingFace, CivitAI) is
|
||||
# now done on-demand via /model_resolver/search endpoint
|
||||
# when user clicks "Search Online" button, not automatically
|
||||
|
||||
result = WorkflowAnalysisResult(
|
||||
missing_models=tuple(filtered_missing),
|
||||
resolved_models=analysis_result.resolved_models,
|
||||
total_resolved=analysis_result.total_resolved,
|
||||
total_missing=len(filtered_missing),
|
||||
total_models_analyzed=analysis_result.total_models_analyzed,
|
||||
)
|
||||
|
||||
if analysis_id:
|
||||
self.analysis_progress.update(
|
||||
analysis_id,
|
||||
status="completed",
|
||||
stage="completed",
|
||||
message="Analysis complete",
|
||||
current=result.get("total_missing", 0),
|
||||
total=result.get("total_missing", 0),
|
||||
current=result.total_missing,
|
||||
total=result.total_missing,
|
||||
)
|
||||
|
||||
return web.json_response(result)
|
||||
return web.json_response(result.to_dict())
|
||||
except Exception as e:
|
||||
if "analysis_id" in locals() and analysis_id:
|
||||
self.analysis_progress.update(
|
||||
@@ -220,24 +252,42 @@ class WorkflowService:
|
||||
"""Apply model resolution and return updated workflow."""
|
||||
apply_resolution = self.apply_resolution
|
||||
web = self.web
|
||||
data = await request.json()
|
||||
data = await read_optional_object_payload(request)
|
||||
workflow_json, workflow_error = validate_workflow_payload(
|
||||
data.get("workflow"),
|
||||
empty_is_missing=True,
|
||||
require_object=False,
|
||||
)
|
||||
resolutions = data.get("resolutions", [])
|
||||
raw_resolutions = data.get("resolutions", [])
|
||||
|
||||
if workflow_error:
|
||||
return web.json_response(
|
||||
{"error": workflow_error}, status=400
|
||||
)
|
||||
|
||||
if not resolutions:
|
||||
if not isinstance(raw_resolutions, list) or not raw_resolutions:
|
||||
return web.json_response(
|
||||
{"error": "Resolutions array is required"}, status=400
|
||||
)
|
||||
|
||||
try:
|
||||
resolutions = []
|
||||
for raw_resolution in raw_resolutions:
|
||||
if not isinstance(raw_resolution, dict):
|
||||
raise ValueError("Each resolution must be an object")
|
||||
resolution = Resolution.from_mapping(raw_resolution)
|
||||
resolution.validate()
|
||||
resolutions.append(
|
||||
resolution.with_custom_node_metadata(
|
||||
get_custom_node_resolution_metadata(resolution.reference)
|
||||
)
|
||||
)
|
||||
except (TypeError, ValueError) as exc:
|
||||
return web.json_response(
|
||||
{"error": f"Invalid resolution: {exc}"},
|
||||
status=400,
|
||||
)
|
||||
|
||||
# Apply resolutions
|
||||
updated_workflow = apply_resolution(workflow_json, resolutions)
|
||||
|
||||
@@ -249,12 +299,26 @@ class WorkflowService:
|
||||
"""Search local model files by filename/path."""
|
||||
invalidate_local_hash_match_cache = self.invalidate_local_hash_match_cache
|
||||
search_local_matches = self.search_local_matches
|
||||
to_bool = self.to_bool
|
||||
web = self.web
|
||||
data = await request.json()
|
||||
filename = data.get("filename", "")
|
||||
category = data.get("category", "")
|
||||
force_rescan = to_bool(data.get("force_rescan"), False)
|
||||
data = await read_optional_object_payload(request)
|
||||
try:
|
||||
filename = read_text_field(
|
||||
data,
|
||||
"filename",
|
||||
contract_name="Local matches request",
|
||||
)
|
||||
category = read_text_field(
|
||||
data,
|
||||
"category",
|
||||
contract_name="Local matches request",
|
||||
)
|
||||
force_rescan = read_bool_field(
|
||||
data,
|
||||
"force_rescan",
|
||||
contract_name="Local matches request",
|
||||
)
|
||||
except TypeError as exc:
|
||||
return web.json_response({"error": str(exc)}, status=400)
|
||||
if force_rescan:
|
||||
invalidate_local_hash_match_cache()
|
||||
|
||||
@@ -270,4 +334,6 @@ class WorkflowService:
|
||||
max_matches_per_model=10,
|
||||
force_rescan=force_rescan,
|
||||
)
|
||||
return web.json_response({"matches": matches})
|
||||
return web.json_response(
|
||||
{"matches": [match.to_dict() for match in matches]}
|
||||
)
|
||||
|
||||
+95
-75
@@ -13,6 +13,7 @@ from urllib.parse import urljoin, urlparse
|
||||
|
||||
import requests
|
||||
|
||||
from ..contracts import ProviderUrlReference, SearchResult
|
||||
from ..log_system import create_module_logger
|
||||
from ..matcher import (
|
||||
base_model_matches as _base_model_matches,
|
||||
@@ -588,7 +589,7 @@ def _search_page(
|
||||
return [result for result in results if isinstance(result, dict)]
|
||||
|
||||
|
||||
def parse_civarchive_url(url: str) -> Optional[Dict[str, Any]]:
|
||||
def parse_civarchive_url(url: str) -> Optional[ProviderUrlReference]:
|
||||
"""Parse CivArchive URLs into sha256 or model/version identifiers."""
|
||||
from ..type_utils import parse_provider_model_url
|
||||
if not url:
|
||||
@@ -604,7 +605,7 @@ def _extract_sha256(value: str) -> Optional[str]:
|
||||
if normalized:
|
||||
return normalized
|
||||
parsed = parse_civarchive_url(value)
|
||||
return parsed.get("sha256") if parsed else None
|
||||
return parsed.sha256 if parsed else None
|
||||
|
||||
|
||||
|
||||
@@ -1024,29 +1025,34 @@ def get_civarchive_model_details(
|
||||
resolved = resolve_civarchive_model_version(model_id, version_id)
|
||||
if not resolved:
|
||||
return None
|
||||
resolved_mapping = resolved.to_dict()
|
||||
fallback_version = {
|
||||
"id": resolved.get("version_id"),
|
||||
"name": resolved.get("version_name"),
|
||||
"baseModel": resolved.get("base_model"),
|
||||
"trainedWords": resolved.get("trained_words", []),
|
||||
"images": resolved.get("images", []),
|
||||
"id": resolved_mapping.get("version_id"),
|
||||
"name": resolved_mapping.get("version_name"),
|
||||
"baseModel": resolved_mapping.get("base_model"),
|
||||
"trainedWords": resolved_mapping.get("trained_words", []),
|
||||
"images": resolved_mapping.get("images", []),
|
||||
"files": [
|
||||
{
|
||||
"name": resolved.get("filename"),
|
||||
"type": resolved.get("type"),
|
||||
"downloadUrl": resolved.get("download_url"),
|
||||
"sizeKB": (resolved.get("size") or 0) / 1024 if resolved.get("size") else None,
|
||||
"name": resolved_mapping.get("filename"),
|
||||
"type": resolved_mapping.get("type"),
|
||||
"downloadUrl": resolved_mapping.get("download_url"),
|
||||
"sizeKB": (
|
||||
(resolved_mapping.get("size") or 0) / 1024
|
||||
if resolved_mapping.get("size")
|
||||
else None
|
||||
),
|
||||
"primary": True,
|
||||
}
|
||||
],
|
||||
}
|
||||
context = {
|
||||
"id": model_id,
|
||||
"name": resolved.get("name"),
|
||||
"type": resolved.get("type"),
|
||||
"tags": resolved.get("tags", []),
|
||||
"creator": resolved.get("creator", {}),
|
||||
"platform": resolved.get("platform"),
|
||||
"name": resolved_mapping.get("name"),
|
||||
"type": resolved_mapping.get("type"),
|
||||
"tags": resolved_mapping.get("tags", []),
|
||||
"creator": resolved_mapping.get("creator", {}),
|
||||
"platform": resolved_mapping.get("platform"),
|
||||
}
|
||||
selected = _normalize_archive_version(fallback_version, context)
|
||||
return {
|
||||
@@ -1189,7 +1195,7 @@ def _build_civarchive_result(
|
||||
sha256: Optional[str] = None,
|
||||
hashes: Optional[Dict[str, Any]] = None,
|
||||
hash_value: Any = _RESULT_HASH_UNSET,
|
||||
) -> Dict[str, Any]:
|
||||
) -> SearchResult:
|
||||
extra_fields = {
|
||||
"civitai_model_id": civitai_model_id,
|
||||
"civitai_model_version_id": civitai_model_version_id,
|
||||
@@ -1231,7 +1237,7 @@ def _build_result_from_normalized_version(
|
||||
version: Dict[str, Any],
|
||||
file_info: Dict[str, Any],
|
||||
match_type: str,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
) -> Optional[SearchResult]:
|
||||
download_urls = _collect_archive_download_urls(
|
||||
file_info,
|
||||
**_NORMALIZED_DOWNLOAD_URL_OPTIONS,
|
||||
@@ -1313,7 +1319,7 @@ def _find_model_title_match_in_model_details(
|
||||
model_details: Dict[str, Any],
|
||||
title_query: str,
|
||||
base_model_context: Optional[str] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
) -> Optional[SearchResult]:
|
||||
"""For extensionless workflow values, resolve by CivArchive model page title."""
|
||||
from ..matcher import match_model_by_title_generic
|
||||
|
||||
@@ -1366,7 +1372,7 @@ def _build_result_from_model_details(
|
||||
model_details: Dict[str, Any],
|
||||
query: str = "",
|
||||
exact_only: bool = False,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
) -> Optional[SearchResult]:
|
||||
selected_version = model_details.get("selected_version")
|
||||
if not isinstance(selected_version, dict):
|
||||
return None
|
||||
@@ -1417,11 +1423,12 @@ def _build_result_from_model_details(
|
||||
|
||||
hashes = selected_file.get("hashes") if isinstance(selected_file.get("hashes"), dict) else {}
|
||||
sha256 = extract_file_sha256(selected_file)
|
||||
result["confidence"] = best_confidence
|
||||
result["sha256"] = sha256
|
||||
result["hash"] = sha256
|
||||
result["hashes"] = hashes
|
||||
return result
|
||||
result = result.with_updates(
|
||||
confidence=best_confidence,
|
||||
sha256=sha256,
|
||||
hashes=hashes,
|
||||
)
|
||||
return result.with_extra(hash=sha256)
|
||||
|
||||
|
||||
def _extract_hash_page_files(html_text: str) -> List[Dict[str, Any]]:
|
||||
@@ -1580,7 +1587,7 @@ def _build_result_from_hash_page_file(
|
||||
file_info: Dict[str, Any],
|
||||
query: str,
|
||||
sha256: str = "",
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
) -> Optional[SearchResult]:
|
||||
model_id = to_int(model_details.get("model_id") or file_info.get("model_id"))
|
||||
version_id = to_int(version.get("id") or file_info.get("model_version_id"))
|
||||
normalized_file = _normalize_archive_file(file_info, model_id, version_id)
|
||||
@@ -1599,13 +1606,15 @@ def _build_result_from_hash_page_file(
|
||||
if not result:
|
||||
return None
|
||||
|
||||
result["confidence"] = confidence
|
||||
result = result.with_updates(confidence=confidence)
|
||||
if sha256:
|
||||
result["sha256"] = sha256
|
||||
result["hash"] = sha256
|
||||
hashes = result.get("hashes") if isinstance(result.get("hashes"), dict) else {}
|
||||
result = result.with_updates(sha256=sha256)
|
||||
result = result.with_extra(hash=sha256)
|
||||
hashes = dict(result.hashes or {})
|
||||
if not hashes.get("SHA256") and not hashes.get("sha256"):
|
||||
result["hashes"] = {**hashes, "SHA256": sha256}
|
||||
result = result.with_updates(
|
||||
hashes={**hashes, "SHA256": sha256}
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
@@ -1614,7 +1623,7 @@ def _build_result_from_payload(
|
||||
query: str = "",
|
||||
preferred_filename: str = "",
|
||||
exact_only: bool = False,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
) -> Optional[SearchResult]:
|
||||
context, version, top_files = _extract_model_context(payload)
|
||||
if not version:
|
||||
return None
|
||||
@@ -1728,7 +1737,7 @@ def resolve_civarchive_by_hash(
|
||||
query: str = "",
|
||||
exact_only: bool = False,
|
||||
model_type: Optional[str] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
) -> Optional[SearchResult]:
|
||||
"""Resolve a model by SHA256 hash through CivArchive."""
|
||||
if not sha256:
|
||||
return None
|
||||
@@ -1760,7 +1769,7 @@ def resolve_civarchive_model_version(
|
||||
query: str = "",
|
||||
exact_only: bool = False,
|
||||
prefer_page: bool = False,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
) -> Optional[SearchResult]:
|
||||
"""Resolve a model/version through CivArchive."""
|
||||
if model_id is None:
|
||||
return None
|
||||
@@ -1796,9 +1805,9 @@ def _candidate_identity(candidate: Dict[str, Any]) -> Tuple[Any, Any, Any]:
|
||||
parsed = parse_civarchive_url(candidate.get("url", ""))
|
||||
if parsed:
|
||||
return (
|
||||
parsed.get("sha256"),
|
||||
parsed.get("model_id"),
|
||||
parsed.get("version_id"),
|
||||
parsed.sha256,
|
||||
parsed.model_id,
|
||||
parsed.version_id,
|
||||
)
|
||||
return (candidate.get("id"), candidate.get("url"), None)
|
||||
|
||||
@@ -1816,7 +1825,7 @@ def _build_result_from_search_candidate(
|
||||
query: str,
|
||||
page_text: str = "",
|
||||
parsed: Optional[Dict[str, Any]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
) -> Optional[SearchResult]:
|
||||
candidate_name = _candidate_display_name(candidate)
|
||||
query_basename = get_filename_from_path(query or "").strip()
|
||||
filename = (
|
||||
@@ -1881,7 +1890,7 @@ def _resolve_civarchive_model_link(
|
||||
exact_only: bool = False,
|
||||
base_model_context: Optional[str] = None,
|
||||
allow_model_title_match: bool = False,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
) -> Optional[SearchResult]:
|
||||
details = get_civarchive_model_details(
|
||||
model_id,
|
||||
version_id,
|
||||
@@ -1915,7 +1924,7 @@ def _resolve_hash_search_candidate_from_page(
|
||||
base_model_context: Optional[str] = None,
|
||||
allow_model_title_match: bool = False,
|
||||
expected_model_type: Optional[str] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
) -> Optional[SearchResult]:
|
||||
page_text = _request_page_text(f"/sha256/{sha256.lower()}")
|
||||
hash_page_files = _extract_hash_page_files(page_text or "")
|
||||
model_links = _prefer_model_links_for_expected_type(
|
||||
@@ -1969,7 +1978,7 @@ def _resolve_hash_search_candidate_from_page(
|
||||
match_type="exact" if confidence == 100.0 else "similar",
|
||||
)
|
||||
if result:
|
||||
result["confidence"] = confidence
|
||||
result = result.with_updates(confidence=confidence)
|
||||
|
||||
selected_file = None
|
||||
if not result:
|
||||
@@ -1998,14 +2007,16 @@ def _resolve_hash_search_candidate_from_page(
|
||||
)
|
||||
|
||||
if result:
|
||||
if exact_only and float(result.get("confidence") or 0) < 100.0:
|
||||
if exact_only and result.confidence < 100.0:
|
||||
continue
|
||||
result.setdefault("sha256", sha256)
|
||||
result.setdefault("hash", sha256)
|
||||
hashes = result.get("hashes") if isinstance(result.get("hashes"), dict) else {}
|
||||
if not result.sha256:
|
||||
result = result.with_updates(sha256=sha256)
|
||||
if not result.extra_value("hash"):
|
||||
result = result.with_extra(hash=sha256)
|
||||
hashes = dict(result.hashes or {})
|
||||
if sha256 and not hashes.get("SHA256") and not hashes.get("sha256"):
|
||||
hashes = {**hashes, "SHA256": sha256}
|
||||
result["hashes"] = hashes
|
||||
result = result.with_updates(hashes=hashes)
|
||||
return result
|
||||
|
||||
return _build_result_from_search_candidate(
|
||||
@@ -2023,25 +2034,25 @@ def _resolve_search_candidate(
|
||||
base_model_context: Optional[str] = None,
|
||||
allow_model_title_match: bool = False,
|
||||
model_type: Optional[str] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
) -> Optional[SearchResult]:
|
||||
parsed = parse_civarchive_url(candidate.get("url", ""))
|
||||
if not parsed:
|
||||
return None
|
||||
|
||||
if parsed.get("sha256"):
|
||||
if parsed.sha256:
|
||||
result = _resolve_hash_search_candidate_from_page(
|
||||
candidate,
|
||||
parsed["sha256"],
|
||||
parsed.sha256,
|
||||
query=query,
|
||||
exact_only=exact_only,
|
||||
base_model_context=base_model_context,
|
||||
allow_model_title_match=allow_model_title_match,
|
||||
expected_model_type=model_type,
|
||||
)
|
||||
elif parsed.get("model_id"):
|
||||
elif parsed.model_id:
|
||||
result = _resolve_civarchive_model_link(
|
||||
parsed["model_id"],
|
||||
parsed.get("version_id"),
|
||||
parsed.model_id,
|
||||
parsed.version_id,
|
||||
query=query,
|
||||
exact_only=exact_only,
|
||||
base_model_context=base_model_context,
|
||||
@@ -2050,27 +2061,29 @@ def _resolve_search_candidate(
|
||||
else:
|
||||
result = None
|
||||
|
||||
if result and not result.get("confidence"):
|
||||
result["confidence"] = calculate_archived_model_confidence(
|
||||
if result and not result.confidence:
|
||||
result = result.with_updates(
|
||||
confidence=calculate_archived_model_confidence(
|
||||
query,
|
||||
candidate.get("name", ""),
|
||||
"",
|
||||
result.get("filename", ""),
|
||||
result.filename,
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _is_hash_verified_exact_match(result: Optional[Dict[str, Any]], confidence: float) -> bool:
|
||||
def _is_hash_verified_exact_match(result: Optional[SearchResult], confidence: float) -> bool:
|
||||
if not result or confidence < 100.0:
|
||||
return False
|
||||
return bool(result.get("sha256") or result.get("hash"))
|
||||
return bool(result.sha256 or result.extra_value("hash"))
|
||||
|
||||
|
||||
def search_civarchive(
|
||||
query: str,
|
||||
model_type: Optional[str] = None,
|
||||
limit: int = DEFAULT_CIVARCHIVE_CANDIDATE_LIMIT,
|
||||
) -> List[Dict[str, Any]]:
|
||||
) -> List[SearchResult]:
|
||||
"""
|
||||
Search CivArchive by model name or filename.
|
||||
"""
|
||||
@@ -2104,7 +2117,7 @@ def search_civarchive(
|
||||
if not resolved:
|
||||
continue
|
||||
|
||||
if resolved.get("confidence", 0) < 40:
|
||||
if resolved.confidence < 40:
|
||||
continue
|
||||
results.append(resolved)
|
||||
if len(results) >= detail_limit:
|
||||
@@ -2112,8 +2125,8 @@ def search_civarchive(
|
||||
|
||||
results.sort(
|
||||
key=lambda item: (
|
||||
item.get("confidence", 0),
|
||||
1 if item.get("match_type") == "exact" else 0,
|
||||
item.confidence,
|
||||
1 if item.match_type == "exact" else 0,
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
@@ -2129,7 +2142,7 @@ def search_civarchive_for_file(
|
||||
limit: int = DEFAULT_CIVARCHIVE_CANDIDATE_LIMIT,
|
||||
progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
|
||||
sha256: Optional[str] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
) -> Optional[SearchResult]:
|
||||
"""
|
||||
Search CivArchive for the best downloadable match for a model filename.
|
||||
"""
|
||||
@@ -2264,30 +2277,37 @@ def search_civarchive_for_file(
|
||||
if not resolved:
|
||||
continue
|
||||
|
||||
if resolved.get("match_type") == "model_title":
|
||||
if resolved.match_type == "model_title":
|
||||
title_confidence = resolved.extra_value("title_confidence")
|
||||
confidence = float(
|
||||
resolved.get("title_confidence")
|
||||
or resolved.get("confidence")
|
||||
title_confidence
|
||||
or resolved.confidence
|
||||
or calculate_model_title_confidence(
|
||||
normalized_filename,
|
||||
resolved.get("name", ""),
|
||||
resolved.name,
|
||||
)
|
||||
)
|
||||
else:
|
||||
confidence = calculate_archived_model_confidence(
|
||||
normalized_filename,
|
||||
resolved.get("name", ""),
|
||||
resolved.get("version_name", ""),
|
||||
resolved.get("filename", ""),
|
||||
resolved.name,
|
||||
resolved.version_name,
|
||||
resolved.filename,
|
||||
)
|
||||
if exact_only and confidence < 100.0:
|
||||
continue
|
||||
|
||||
resolved["confidence"] = confidence
|
||||
resolved["match_type"] = "exact" if confidence == 100.0 else "similar"
|
||||
resolved = resolved.with_updates(
|
||||
confidence=confidence,
|
||||
match_type="exact" if confidence == 100.0 else "similar",
|
||||
)
|
||||
|
||||
should_update, rank, base_model_matches = should_update_best_match(
|
||||
confidence, resolved.get("base_model"), base_model_context, best_rank, exact_only
|
||||
confidence,
|
||||
resolved.base_model,
|
||||
base_model_context,
|
||||
best_rank,
|
||||
exact_only,
|
||||
)
|
||||
if exact_only and confidence < 100.0:
|
||||
continue
|
||||
@@ -2320,7 +2340,7 @@ def search_civarchive_for_file(
|
||||
|
||||
if best_match and (
|
||||
not base_model_context
|
||||
or _base_model_matches(best_match.get("base_model"), base_model_context)
|
||||
or _base_model_matches(best_match.base_model, base_model_context)
|
||||
):
|
||||
break
|
||||
except CivArchiveSearchError:
|
||||
@@ -2341,7 +2361,7 @@ def search_civarchive_for_file(
|
||||
if (
|
||||
best_match
|
||||
and base_model_context
|
||||
and not _base_model_matches(best_match.get("base_model"), base_model_context)
|
||||
and not _base_model_matches(best_match.base_model, base_model_context)
|
||||
and not _is_hash_verified_exact_match(best_match, best_confidence)
|
||||
):
|
||||
best_match = None
|
||||
@@ -2363,7 +2383,7 @@ def search_civarchive_for_file(
|
||||
def build_civarchive_custom_result(
|
||||
details: Dict[str, Any],
|
||||
expected_filename: str = "",
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
) -> Optional[SearchResult]:
|
||||
if not isinstance(details, dict):
|
||||
return None
|
||||
|
||||
|
||||
+53
-48
@@ -12,6 +12,7 @@ from urllib.parse import quote, urlencode
|
||||
|
||||
import requests
|
||||
|
||||
from ..contracts import ProviderUrlReference, SearchResult
|
||||
from ..log_system import create_module_logger
|
||||
from ..matcher import (
|
||||
base_model_matches as _base_model_matches,
|
||||
@@ -203,7 +204,7 @@ def _build_civitai_result_from_version(
|
||||
file_info: Dict[str, Any],
|
||||
tags: Optional[List[str]] = None,
|
||||
match_type: str = "exact",
|
||||
) -> Dict[str, Any]:
|
||||
) -> SearchResult:
|
||||
"""Normalize CivitAI model/version/file data into the search result format."""
|
||||
version_id = version.get("id")
|
||||
size = extract_file_size(file_info)
|
||||
@@ -238,7 +239,7 @@ def _find_model_title_match_in_model(
|
||||
title_query: str,
|
||||
api_key: Optional[str] = None,
|
||||
base_model_context: Optional[str] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
) -> Optional[SearchResult]:
|
||||
"""For extensionless workflow values, resolve by CivitAI model page title."""
|
||||
from ..matcher import match_model_by_title_generic
|
||||
|
||||
@@ -266,7 +267,7 @@ def _find_model_title_match_in_model(
|
||||
match_type="model_title",
|
||||
)
|
||||
if res:
|
||||
res["version_name"] = v.get("name", "")
|
||||
res = res.with_updates(version_name=v.get("name", ""))
|
||||
return res
|
||||
|
||||
return match_model_by_title_generic(
|
||||
@@ -733,7 +734,7 @@ def _find_civitai_file_in_model(
|
||||
exact_only: bool = False,
|
||||
preferred_version_id: Optional[int] = None,
|
||||
base_model_context: Optional[str] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
) -> Optional[SearchResult]:
|
||||
"""Load one CivitAI model and search all its versions for the requested file."""
|
||||
filename_lower = filename.lower()
|
||||
filename_base = os.path.splitext(filename_lower)[0]
|
||||
@@ -743,7 +744,7 @@ def _find_civitai_file_in_model(
|
||||
|
||||
def build_result_from_resolved_version(
|
||||
resolved: Dict[str, Any], version_id: int
|
||||
) -> Dict[str, Any]:
|
||||
) -> SearchResult:
|
||||
expected_filename = resolved.get("expected_filename", "")
|
||||
primary_file = select_primary_model_file(
|
||||
resolved.get("files") or [],
|
||||
@@ -751,8 +752,8 @@ def _find_civitai_file_in_model(
|
||||
fallback_to_first=True,
|
||||
) or {}
|
||||
|
||||
return {
|
||||
"source": "civitai",
|
||||
return build_model_result(
|
||||
"civitai",
|
||||
**build_civitai_result_payload(
|
||||
model_id=model_id,
|
||||
version_id=version_id,
|
||||
@@ -769,7 +770,8 @@ def _find_civitai_file_in_model(
|
||||
filename, expected_filename
|
||||
),
|
||||
),
|
||||
}
|
||||
normalize_hashes=True,
|
||||
)
|
||||
|
||||
def resolved_version_matches(resolved: Dict[str, Any]) -> bool:
|
||||
expected_filename = str(resolved.get("expected_filename", "")).lower()
|
||||
@@ -849,9 +851,10 @@ def _find_civitai_file_in_model(
|
||||
best_resolved_confidence = ranking_score
|
||||
best_resolved_result = build_result_from_resolved_version(
|
||||
resolved, version_id
|
||||
).with_updates(
|
||||
match_type="similar",
|
||||
confidence=confidence,
|
||||
)
|
||||
best_resolved_result["match_type"] = "similar"
|
||||
best_resolved_result["confidence"] = confidence
|
||||
|
||||
files = version.get("files", [])
|
||||
file_names = [f.get("name", "") for f in files if isinstance(f, dict)]
|
||||
@@ -861,7 +864,7 @@ def _find_civitai_file_in_model(
|
||||
|
||||
if best_resolved_result and best_resolved_confidence >= 50.0:
|
||||
log.info(
|
||||
f"CivitAI best probable match: model_id={model_id}, version_id={best_resolved_result.get('version_id')}, filename={best_resolved_result.get('filename')}, confidence={best_resolved_confidence}"
|
||||
f"CivitAI best probable match: model_id={model_id}, version_id={best_resolved_result.version_id}, filename={best_resolved_result.filename}, confidence={best_resolved_confidence}"
|
||||
)
|
||||
return best_resolved_result
|
||||
|
||||
@@ -878,15 +881,17 @@ def _find_civitai_file_in_model(
|
||||
tags=data.get("tags", []),
|
||||
match_type=match["match_type"],
|
||||
)
|
||||
result["confidence"] = match.get(
|
||||
"confidence",
|
||||
calculate_filename_confidence(filename, result.get("filename", "")),
|
||||
result = result.with_updates(
|
||||
confidence=match.get(
|
||||
"confidence",
|
||||
calculate_filename_confidence(filename, result.filename),
|
||||
)
|
||||
)
|
||||
if match["match_type"] == "similar":
|
||||
log.info(
|
||||
f"CivitAI version-list probable match: model_id={model_id}, version_id={result.get('version_id')}, filename={result.get('filename')}, confidence={result['confidence']}"
|
||||
f"CivitAI version-list probable match: model_id={model_id}, version_id={result.version_id}, filename={result.filename}, confidence={result.confidence}"
|
||||
)
|
||||
if _base_model_matches(result.get("base_model"), base_model_context):
|
||||
if _base_model_matches(result.base_model, base_model_context):
|
||||
return result
|
||||
if not base_model_context:
|
||||
return result
|
||||
@@ -895,7 +900,7 @@ def _find_civitai_file_in_model(
|
||||
|
||||
|
||||
|
||||
def parse_civitai_url(url: str) -> Optional[Dict[str, Any]]:
|
||||
def parse_civitai_url(url: str) -> Optional[ProviderUrlReference]:
|
||||
"""
|
||||
Parse a CivitAI URL to extract model/version info.
|
||||
"""
|
||||
@@ -924,7 +929,7 @@ def search_civitai_for_file(
|
||||
use_html_fallback: bool = True,
|
||||
progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
|
||||
sha256: Optional[str] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
) -> Optional[SearchResult]:
|
||||
"""
|
||||
Search CivitAI for a specific model file.
|
||||
Returns the first model that actually has this exact filename.
|
||||
@@ -936,7 +941,7 @@ def search_civitai_for_file(
|
||||
If False, also try partial matching (for local file resolution).
|
||||
|
||||
Returns:
|
||||
Dict with download info if found, None otherwise
|
||||
A typed SearchResult if a matching model is found, otherwise None.
|
||||
"""
|
||||
global _search_cache
|
||||
|
||||
@@ -1146,7 +1151,7 @@ def search_civitai_for_file(
|
||||
base_model_context=base_model_context,
|
||||
)
|
||||
if result:
|
||||
confidence = float(result.get("confidence") or 0.0)
|
||||
confidence = result.confidence
|
||||
if confidence >= 100.0:
|
||||
_search_cache[cache_key] = result
|
||||
_report_progress(
|
||||
@@ -1154,12 +1159,12 @@ def search_civitai_for_file(
|
||||
"found",
|
||||
"Found exact CivitAI match",
|
||||
92,
|
||||
model_id=result.get("model_id"),
|
||||
version_id=result.get("version_id"),
|
||||
model_id=result.model_id,
|
||||
version_id=result.version_id,
|
||||
confidence=confidence,
|
||||
)
|
||||
log.info(
|
||||
f"Found exact CivitAI match for {filename}: model_id={result.get('model_id')}, version_id={result.get('version_id')}, candidate_limit={candidate_limit}"
|
||||
f"Found exact CivitAI match for {filename}: model_id={result.model_id}, version_id={result.version_id}, candidate_limit={candidate_limit}"
|
||||
)
|
||||
return result
|
||||
if confidence > best_confidence:
|
||||
@@ -1173,12 +1178,12 @@ def search_civitai_for_file(
|
||||
"found",
|
||||
"Found CivitAI match",
|
||||
92,
|
||||
model_id=best_result.get("model_id"),
|
||||
version_id=best_result.get("version_id"),
|
||||
model_id=best_result.model_id,
|
||||
version_id=best_result.version_id,
|
||||
confidence=best_confidence,
|
||||
)
|
||||
log.info(
|
||||
f"Found best CivitAI match for {filename}: model_id={best_result.get('model_id')}, version_id={best_result.get('version_id')}, confidence={best_confidence}, candidate_limit={candidate_limit}"
|
||||
f"Found best CivitAI match for {filename}: model_id={best_result.model_id}, version_id={best_result.version_id}, confidence={best_confidence}, candidate_limit={candidate_limit}"
|
||||
)
|
||||
return best_result
|
||||
|
||||
@@ -1217,7 +1222,7 @@ def search_civitai(
|
||||
model_type: Optional[str] = None,
|
||||
limit: int = 10,
|
||||
api_key: Optional[str] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
) -> List[SearchResult]:
|
||||
"""
|
||||
Search CivitAI for models (general search).
|
||||
Returns models that might be relevant.
|
||||
@@ -1245,7 +1250,7 @@ def search_civitai(
|
||||
for model in data.get("items", []):
|
||||
model_id = model.get("id")
|
||||
model_name = model.get("name", "")
|
||||
model_type = model.get("type", "")
|
||||
civitai_model_type = model.get("type", "")
|
||||
|
||||
versions = model.get("modelVersions", [])
|
||||
if versions:
|
||||
@@ -1258,24 +1263,24 @@ def search_civitai(
|
||||
prefer_first_marked=True,
|
||||
)
|
||||
|
||||
result = {
|
||||
"source": "civitai",
|
||||
"model_id": model_id,
|
||||
"version_id": version_id,
|
||||
"name": model_name,
|
||||
"type": model_type,
|
||||
"url": f"https://civitai.com/models/{model_id}",
|
||||
"download_url": get_civitai_download_url(version_id, api_key),
|
||||
"downloads": model.get("stats", {}).get("downloadCount", 0),
|
||||
"base_model": latest.get("baseModel"),
|
||||
"tags": model.get("tags", []),
|
||||
}
|
||||
|
||||
if primary_file:
|
||||
result["filename"] = primary_file.get("name", "")
|
||||
result["size"] = extract_file_size(primary_file) or 0
|
||||
|
||||
results.append(result)
|
||||
results.append(
|
||||
build_model_result(
|
||||
"civitai",
|
||||
model_id=model_id,
|
||||
version_id=version_id,
|
||||
name=model_name,
|
||||
type=civitai_model_type,
|
||||
filename=(primary_file or {}).get("name") or model_name,
|
||||
url=f"https://civitai.com/models/{model_id}",
|
||||
download_url=get_civitai_download_url(version_id, api_key),
|
||||
size=(extract_file_size(primary_file) or 0)
|
||||
if primary_file
|
||||
else None,
|
||||
base_model=latest.get("baseModel"),
|
||||
tags=model.get("tags", []),
|
||||
downloads=model.get("stats", {}).get("downloadCount", 0),
|
||||
)
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
log.error(f"CivitAI search error: {e}")
|
||||
@@ -2337,7 +2342,7 @@ def build_civitai_custom_result(
|
||||
details: Dict[str, Any],
|
||||
expected_filename: str = "",
|
||||
api_key: Optional[str] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
) -> Optional[SearchResult]:
|
||||
if not isinstance(details, dict):
|
||||
return None
|
||||
|
||||
@@ -2394,7 +2399,7 @@ def resolve_civitai_version_custom_result(
|
||||
version_id: int,
|
||||
expected_filename: str = "",
|
||||
api_key: Optional[str] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
) -> Optional[SearchResult]:
|
||||
if not version_id:
|
||||
return None
|
||||
|
||||
|
||||
+56
-52
@@ -13,6 +13,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
from urllib.parse import quote, unquote, urlparse
|
||||
|
||||
from ..contracts import HuggingFaceFileReference, SearchResult
|
||||
from ..log_system import create_module_logger
|
||||
from ..progress import get_progress_reporter
|
||||
|
||||
@@ -443,7 +444,7 @@ def refresh_known_author_fallback_indexes(
|
||||
return status
|
||||
|
||||
|
||||
def parse_huggingface_url(url: str) -> Optional[Dict[str, str]]:
|
||||
def parse_huggingface_url(url: str) -> Optional[HuggingFaceFileReference]:
|
||||
"""
|
||||
Parse a HuggingFace URL to extract repo and filename.
|
||||
|
||||
@@ -453,12 +454,15 @@ def parse_huggingface_url(url: str) -> Optional[Dict[str, str]]:
|
||||
- hf://user/repo/file.safetensors
|
||||
|
||||
Returns:
|
||||
Dictionary with 'repo' and 'filename' keys, or None if not HF URL
|
||||
Typed repository-file identity, or None if not an HF URL
|
||||
"""
|
||||
if url.startswith("hf://"):
|
||||
parts = url[5:].split("/", 2)
|
||||
if len(parts) >= 3:
|
||||
return {"repo": f"{parts[0]}/{parts[1]}", "filename": parts[2]}
|
||||
return HuggingFaceFileReference(
|
||||
repo=f"{parts[0]}/{parts[1]}",
|
||||
filename=parts[2],
|
||||
)
|
||||
return None
|
||||
|
||||
parsed = urlparse(url)
|
||||
@@ -467,11 +471,11 @@ def parse_huggingface_url(url: str) -> Optional[Dict[str, str]]:
|
||||
|
||||
match = re.match(r"^/([^/]+/[^/]+)/(resolve|blob)/([^/]+)/(.+)$", parsed.path)
|
||||
if match:
|
||||
return {
|
||||
"repo": match.group(1),
|
||||
"branch": match.group(3),
|
||||
"filename": match.group(4),
|
||||
}
|
||||
return HuggingFaceFileReference(
|
||||
repo=match.group(1),
|
||||
branch=match.group(3),
|
||||
filename=match.group(4),
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
@@ -700,7 +704,7 @@ def _build_huggingface_result(
|
||||
file_info: Dict[str, Any],
|
||||
match_type: str,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
) -> SearchResult:
|
||||
sha256 = _extract_huggingface_file_sha256(file_info)
|
||||
download_url = get_huggingface_download_url(repo_id, file_path)
|
||||
size = resolve_file_size(
|
||||
@@ -765,10 +769,10 @@ def get_huggingface_file_sha256(
|
||||
if not parsed:
|
||||
return ""
|
||||
|
||||
repo_id = str(parsed.get("repo") or "").strip()
|
||||
branch = str(parsed.get("branch") or "main").strip() or "main"
|
||||
repo_id = parsed.repo
|
||||
branch = parsed.branch
|
||||
target_path = _normalize_huggingface_path(
|
||||
unquote(str(parsed.get("filename") or ""))
|
||||
unquote(parsed.filename)
|
||||
)
|
||||
if not repo_id or not target_path:
|
||||
return ""
|
||||
@@ -787,7 +791,7 @@ def _find_matching_file_in_repo(
|
||||
filename: str,
|
||||
exact_only: bool = False,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
) -> Optional[SearchResult]:
|
||||
filename_lower = filename.lower()
|
||||
filename_base = os.path.splitext(filename_lower)[0]
|
||||
partial_match = None
|
||||
@@ -845,7 +849,7 @@ def _find_matching_file_in_author_index(
|
||||
exact_only: bool = False,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
sha256: Optional[str] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
) -> Optional[SearchResult]:
|
||||
files = index.get("files") or []
|
||||
requested_sha256 = normalize_sha256(sha256)
|
||||
|
||||
@@ -988,9 +992,9 @@ def _search_brave_for_huggingface_candidates(
|
||||
|
||||
parsed = parse_huggingface_url(candidate_url)
|
||||
if parsed:
|
||||
repo_id = parsed.get("repo", "")
|
||||
file_path = parsed.get("filename", "")
|
||||
branch = parsed.get("branch", "main")
|
||||
repo_id = parsed.repo
|
||||
file_path = parsed.filename
|
||||
branch = parsed.branch
|
||||
else:
|
||||
parsed_url = urlparse(candidate_url)
|
||||
if "huggingface.co" not in parsed_url.netloc:
|
||||
@@ -1032,7 +1036,7 @@ def search_huggingface_for_file(
|
||||
force_refresh: bool = False,
|
||||
progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
|
||||
sha256: Optional[str] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
) -> Optional[SearchResult]:
|
||||
"""
|
||||
Search HuggingFace for a specific model file.
|
||||
Returns the first repo that contains a matching file.
|
||||
@@ -1044,7 +1048,7 @@ def search_huggingface_for_file(
|
||||
If False, also try partial matching (for local file resolution).
|
||||
|
||||
Returns:
|
||||
Dict with url, repo, filename if found, None otherwise
|
||||
A typed SearchResult if a matching file is found, otherwise None.
|
||||
"""
|
||||
global _search_cache
|
||||
|
||||
@@ -1069,14 +1073,14 @@ def search_huggingface_for_file(
|
||||
return _search_cache[cache_key]
|
||||
|
||||
try:
|
||||
def result_matches_requested_hash(result: Optional[Dict[str, Any]]) -> bool:
|
||||
def result_matches_requested_hash(result: Optional[SearchResult]) -> bool:
|
||||
if not requested_sha256:
|
||||
return True
|
||||
return normalize_sha256(
|
||||
result.get("sha256")
|
||||
or result.get("hash")
|
||||
or (result.get("hashes") or {}).get("SHA256")
|
||||
or (result.get("hashes") or {}).get("sha256")
|
||||
result.sha256
|
||||
or result.extra_value("hash")
|
||||
or (result.hashes or {}).get("SHA256")
|
||||
or (result.hashes or {}).get("sha256")
|
||||
) == requested_sha256
|
||||
|
||||
headers = {}
|
||||
@@ -1087,7 +1091,7 @@ def search_huggingface_for_file(
|
||||
|
||||
def find_author_index_result(
|
||||
author: str,
|
||||
) -> tuple[Optional[Dict[str, Any]], Optional[Dict[str, Any]]]:
|
||||
) -> tuple[Optional[Dict[str, Any]], Optional[SearchResult]]:
|
||||
index = _get_author_index(
|
||||
author,
|
||||
headers={},
|
||||
@@ -1155,13 +1159,13 @@ def search_huggingface_for_file(
|
||||
"found",
|
||||
"Found HuggingFace SHA-256 match",
|
||||
92,
|
||||
repo=result.get("repo_id"),
|
||||
match_type=result.get("match_type"),
|
||||
repo=result.extra_value("repo_id"),
|
||||
match_type=result.match_type,
|
||||
)
|
||||
log.info(
|
||||
f"HuggingFace found source=author_index "
|
||||
f"match=hash file={result['filename']} "
|
||||
f"repo={result['repo_id']} path={result['path']}"
|
||||
f"match=hash file={result.filename} "
|
||||
f"repo={result.extra_value('repo_id')} path={result.extra_value('path')}"
|
||||
)
|
||||
return result
|
||||
|
||||
@@ -1250,10 +1254,10 @@ def search_huggingface_for_file(
|
||||
"Found HuggingFace match",
|
||||
92,
|
||||
repo=repo_id,
|
||||
match_type=result.get("match_type"),
|
||||
match_type=result.match_type,
|
||||
)
|
||||
log.info(
|
||||
f"HuggingFace found match={result['match_type']} file={filename} repo={repo_id} path={result['path']}"
|
||||
f"HuggingFace found match={result.match_type} file={filename} repo={repo_id} path={result.extra_value('path')}"
|
||||
)
|
||||
return result
|
||||
|
||||
@@ -1295,11 +1299,11 @@ def search_huggingface_for_file(
|
||||
"found",
|
||||
"Found HuggingFace fallback match",
|
||||
92,
|
||||
repo=result.get("repo_id"),
|
||||
match_type=result.get("match_type"),
|
||||
repo=result.extra_value("repo_id"),
|
||||
match_type=result.match_type,
|
||||
)
|
||||
log.info(
|
||||
f"HuggingFace found source=author_index match={result['match_type']} file={filename} repo={result['repo_id']} path={result['path']}"
|
||||
f"HuggingFace found source=author_index match={result.match_type} file={filename} repo={result.extra_value('repo_id')} path={result.extra_value('path')}"
|
||||
)
|
||||
return result
|
||||
|
||||
@@ -1356,10 +1360,10 @@ def search_huggingface_for_file(
|
||||
"Found HuggingFace fallback match",
|
||||
92,
|
||||
repo=repo_id,
|
||||
match_type=result.get("match_type"),
|
||||
match_type=result.match_type,
|
||||
)
|
||||
log.info(
|
||||
f"HuggingFace found source=author_fallback match={result['match_type']} file={filename} repo={repo_id} path={result['path']}"
|
||||
f"HuggingFace found source=author_fallback match={result.match_type} file={filename} repo={repo_id} path={result.extra_value('path')}"
|
||||
)
|
||||
return result
|
||||
|
||||
@@ -1411,7 +1415,7 @@ def search_huggingface_for_file(
|
||||
)
|
||||
if (
|
||||
result
|
||||
and result.get("filename", "").lower() == filename.lower()
|
||||
and result.filename.lower() == filename.lower()
|
||||
and result_matches_requested_hash(result)
|
||||
):
|
||||
_search_cache[cache_key] = result
|
||||
@@ -1421,10 +1425,10 @@ def search_huggingface_for_file(
|
||||
"Found HuggingFace Brave match",
|
||||
92,
|
||||
repo=repo_id,
|
||||
match_type=result.get("match_type"),
|
||||
match_type=result.match_type,
|
||||
)
|
||||
log.info(
|
||||
f"HuggingFace found source=brave_fallback match=exact file={filename} repo={repo_id} path={result['path']}"
|
||||
f"HuggingFace found source=brave_fallback match=exact file={filename} repo={repo_id} path={result.extra_value('path')}"
|
||||
)
|
||||
return result
|
||||
|
||||
@@ -1461,7 +1465,7 @@ def search_huggingface(
|
||||
model_type: Optional[str] = None,
|
||||
limit: int = 10,
|
||||
token: Optional[str] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
) -> List[SearchResult]:
|
||||
"""
|
||||
Search HuggingFace Hub for models (general search).
|
||||
Returns repos that might be relevant, not guaranteed to have exact file.
|
||||
@@ -1486,14 +1490,14 @@ def search_huggingface(
|
||||
repo_id = model.get("id", "")
|
||||
|
||||
results.append(
|
||||
{
|
||||
"source": "huggingface",
|
||||
"repo": repo_id,
|
||||
"name": model.get("modelId", repo_id),
|
||||
"downloads": model.get("downloads", 0),
|
||||
"likes": model.get("likes", 0),
|
||||
"url": f"https://huggingface.co/{repo_id}",
|
||||
}
|
||||
build_model_result(
|
||||
"huggingface",
|
||||
name=model.get("modelId", repo_id),
|
||||
url=f"https://huggingface.co/{repo_id}",
|
||||
repo=repo_id,
|
||||
downloads=model.get("downloads", 0),
|
||||
likes=model.get("likes", 0),
|
||||
)
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
@@ -1506,16 +1510,16 @@ def build_huggingface_custom_result(
|
||||
url: str,
|
||||
expected_filename: str = "",
|
||||
token: Optional[str] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
) -> Optional[SearchResult]:
|
||||
from urllib.parse import quote, unquote
|
||||
|
||||
parsed = parse_huggingface_url(url)
|
||||
if not parsed:
|
||||
return None
|
||||
|
||||
repo_id = parsed.get("repo") or ""
|
||||
branch = parsed.get("branch") or "main"
|
||||
file_path = unquote(parsed.get("filename") or "")
|
||||
repo_id = parsed.repo
|
||||
branch = parsed.branch
|
||||
file_path = unquote(parsed.filename)
|
||||
if not repo_id or not file_path:
|
||||
return None
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ import sqlite3
|
||||
import time
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
from ..contracts import SearchResult
|
||||
from ..log_system import create_module_logger
|
||||
from ..matcher import (
|
||||
base_model_matches as _base_model_matches,
|
||||
@@ -386,7 +387,7 @@ def _load_full_rows_for_versions(
|
||||
|
||||
def _build_result_from_row(
|
||||
conn: sqlite3.Connection, row: sqlite3.Row, query: str
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
) -> Optional[SearchResult]:
|
||||
try:
|
||||
model_data = json.loads(row["model_data"] or "{}")
|
||||
except Exception:
|
||||
@@ -469,7 +470,7 @@ def search_lora_manager_archive(
|
||||
base_model_context: Optional[str] = None,
|
||||
limit: int = 10,
|
||||
progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
) -> List[SearchResult]:
|
||||
"""
|
||||
Search the local comfyui-lora-manager CivitAI archive by model name.
|
||||
|
||||
@@ -613,19 +614,19 @@ def search_lora_manager_archive(
|
||||
if not full_row:
|
||||
continue
|
||||
result = _build_result_from_row(conn, full_row, normalized_query)
|
||||
if not result or result["confidence"] < 40:
|
||||
if not result or result.confidence < 40:
|
||||
continue
|
||||
if base_model_context and not _base_model_matches(
|
||||
result.get("base_model"), base_model_context
|
||||
result.base_model, base_model_context
|
||||
):
|
||||
continue
|
||||
results.append(result)
|
||||
|
||||
results.sort(
|
||||
key=lambda item: (
|
||||
_base_model_score(item.get("base_model"), base_model_context),
|
||||
item.get("confidence", 0),
|
||||
1 if item.get("match_type") == "exact" else 0,
|
||||
_base_model_score(item.base_model, base_model_context),
|
||||
item.confidence,
|
||||
1 if item.match_type == "exact" else 0,
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
@@ -666,7 +667,7 @@ def search_lora_manager_archive_for_file(
|
||||
limit: int = 10,
|
||||
progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
|
||||
sha256: Optional[str] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
) -> Optional[SearchResult]:
|
||||
"""
|
||||
Search the archive for the best model/version match for a filename.
|
||||
"""
|
||||
@@ -696,16 +697,16 @@ def search_lora_manager_archive_for_file(
|
||||
matching_size = extract_file_size(matching_file)
|
||||
if matching_size is None:
|
||||
matching_size = matching_file.get("size")
|
||||
result.update(
|
||||
{
|
||||
"filename": matching_file.get("name") or result.get("filename", ""),
|
||||
"download_url": matching_file.get("downloadUrl") or result.get("download_url"),
|
||||
"size": matching_size if matching_size is not None else result.get("size"),
|
||||
"sha256": requested_sha256,
|
||||
"hashes": matching_file.get("hashes") or {"SHA256": requested_sha256},
|
||||
"match_type": "hash",
|
||||
"confidence": 100.0,
|
||||
}
|
||||
result = result.with_updates(
|
||||
filename=matching_file.get("name") or result.filename,
|
||||
download_url=matching_file.get("downloadUrl")
|
||||
or result.download_url,
|
||||
size=matching_size if matching_size is not None else result.size,
|
||||
sha256=requested_sha256,
|
||||
hashes=matching_file.get("hashes")
|
||||
or {"SHA256": requested_sha256},
|
||||
match_type="hash",
|
||||
confidence=100.0,
|
||||
)
|
||||
_report_progress(
|
||||
progress_callback,
|
||||
@@ -774,19 +775,29 @@ def search_lora_manager_archive_for_file(
|
||||
candidate_index=candidate_index,
|
||||
candidate_count=total_candidates,
|
||||
)
|
||||
candidate_filename = candidate.get("filename", "")
|
||||
confidence = calculate_archived_model_confidence(filename, candidate.get("name", ""), candidate.get("version_name", ""), candidate_filename)
|
||||
candidate_filename = candidate.filename
|
||||
confidence = calculate_archived_model_confidence(
|
||||
filename,
|
||||
candidate.name,
|
||||
candidate.version_name,
|
||||
candidate_filename,
|
||||
)
|
||||
should_update, rank, base_model_matches = should_update_best_match(
|
||||
confidence, candidate.get("base_model"), base_model_context, best_rank, exact_only
|
||||
confidence,
|
||||
candidate.base_model,
|
||||
base_model_context,
|
||||
best_rank,
|
||||
exact_only,
|
||||
)
|
||||
if base_model_context and not base_model_matches:
|
||||
continue
|
||||
if should_update:
|
||||
best_rank = rank
|
||||
best_confidence = confidence
|
||||
best_match = dict(candidate)
|
||||
best_match["confidence"] = confidence
|
||||
best_match["match_type"] = "exact" if confidence == 100.0 else "similar"
|
||||
best_match = candidate.with_updates(
|
||||
confidence=confidence,
|
||||
match_type="exact" if confidence == 100.0 else "similar",
|
||||
)
|
||||
if confidence == 100.0 and base_model_matches:
|
||||
_report_progress(
|
||||
progress_callback,
|
||||
|
||||
+41
-32
@@ -8,8 +8,10 @@ import hashlib
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from ..contracts import SearchResult
|
||||
from ..log_system import create_module_logger
|
||||
from ..matcher import calculate_similarity, normalize_filename
|
||||
from ..type_utils import build_model_result
|
||||
|
||||
log = create_module_logger(__name__)
|
||||
|
||||
@@ -177,21 +179,28 @@ def update_model_list_from_remote() -> Dict[str, Any]:
|
||||
|
||||
|
||||
|
||||
def _build_model_list_result(model: Dict[str, Any]) -> Dict[str, Any]:
|
||||
return {
|
||||
"source": "model_list",
|
||||
"filename": model.get("filename", ""),
|
||||
"url": model.get("url", ""),
|
||||
"name": model.get("name", ""),
|
||||
"type": model.get("type", ""),
|
||||
"directory": model.get("save_path", "checkpoints"),
|
||||
"size": model.get("size", ""),
|
||||
}
|
||||
def _build_model_list_result(
|
||||
model: Dict[str, Any],
|
||||
*,
|
||||
match_type: str = "similar",
|
||||
confidence: float = 0.0,
|
||||
) -> SearchResult:
|
||||
return build_model_result(
|
||||
"model_list",
|
||||
filename=model.get("filename", ""),
|
||||
name=model.get("name", ""),
|
||||
url=model.get("url", ""),
|
||||
type=model.get("type", ""),
|
||||
directory=model.get("save_path", "checkpoints"),
|
||||
size=model.get("size"),
|
||||
match_type=match_type,
|
||||
confidence=confidence,
|
||||
)
|
||||
|
||||
|
||||
def search_model_list(
|
||||
filename: str, exact_only: bool = False
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
) -> Optional[SearchResult]:
|
||||
"""
|
||||
Search model-list.json for a model by filename.
|
||||
Uses exact match first, then fuzzy matching (unless exact_only=True).
|
||||
@@ -202,7 +211,7 @@ def search_model_list(
|
||||
If False, also try fuzzy matching (for local file resolution).
|
||||
|
||||
Returns:
|
||||
Dict with url, filename, type, etc. if found, None otherwise
|
||||
Typed search result with URL, filename, type, etc. if found.
|
||||
"""
|
||||
models = _load_model_list()
|
||||
if not models:
|
||||
@@ -218,10 +227,7 @@ def search_model_list(
|
||||
if model_filename.lower() == filename_lower:
|
||||
url = model.get("url", "")
|
||||
if url:
|
||||
return {
|
||||
**_build_model_list_result(model),
|
||||
"match_type": "exact",
|
||||
}
|
||||
return _build_model_list_result(model, match_type="exact")
|
||||
|
||||
# If exact_only is True, don't try fuzzy matching - prevents confusing
|
||||
# users with wrong model suggestions for downloads
|
||||
@@ -242,11 +248,11 @@ def search_model_list(
|
||||
url = model.get("url", "")
|
||||
if url:
|
||||
score = calculate_similarity(filename_norm, normalize_filename(model_filename))
|
||||
return {
|
||||
**_build_model_list_result(model),
|
||||
"match_type": "fuzzy",
|
||||
"confidence": round(score * 100, 1),
|
||||
}
|
||||
return _build_model_list_result(
|
||||
model,
|
||||
match_type="fuzzy",
|
||||
confidence=round(score * 100, 1),
|
||||
)
|
||||
|
||||
# 3. Try normalized similarity matching on all models (fallback)
|
||||
best_match = None
|
||||
@@ -264,16 +270,19 @@ def search_model_list(
|
||||
url = model.get("url", "")
|
||||
if url:
|
||||
best_score = score
|
||||
best_match = {
|
||||
**_build_model_list_result(model),
|
||||
"match_type": "similar",
|
||||
"confidence": round(score * 100, 1),
|
||||
}
|
||||
best_match = _build_model_list_result(
|
||||
model,
|
||||
match_type="similar",
|
||||
confidence=round(score * 100, 1),
|
||||
)
|
||||
|
||||
return best_match
|
||||
|
||||
|
||||
def search_model_list_multiple(filename: str, limit: int = 5) -> List[Dict[str, Any]]:
|
||||
def search_model_list_multiple(
|
||||
filename: str,
|
||||
limit: int = 5,
|
||||
) -> List[SearchResult]:
|
||||
"""
|
||||
Search model-list.json and return multiple fuzzy matches.
|
||||
|
||||
@@ -303,14 +312,14 @@ def search_model_list_multiple(filename: str, limit: int = 5) -> List[Dict[str,
|
||||
url = model.get("url", "")
|
||||
if url:
|
||||
results.append(
|
||||
{
|
||||
**_build_model_list_result(model),
|
||||
"confidence": round(score * 100, 1),
|
||||
}
|
||||
_build_model_list_result(
|
||||
model,
|
||||
confidence=round(score * 100, 1),
|
||||
)
|
||||
)
|
||||
|
||||
# Sort by confidence descending
|
||||
results.sort(key=lambda x: x["confidence"], reverse=True)
|
||||
results.sort(key=lambda x: x.confidence, reverse=True)
|
||||
|
||||
return results[:limit]
|
||||
|
||||
|
||||
+52
-13
@@ -8,6 +8,7 @@ import os
|
||||
import re
|
||||
from typing import Any, Dict, List, Optional, Set
|
||||
|
||||
from ..contracts import ModelCatalogEntry, SearchResult
|
||||
from ..log_system import create_module_logger
|
||||
|
||||
log = create_module_logger(__name__)
|
||||
@@ -16,7 +17,7 @@ log = create_module_logger(__name__)
|
||||
from ..catalog_manager import CatalogManager
|
||||
from ..network_utils import request_source_json
|
||||
from ..path_utils import METADATA_DIR, read_json_safe
|
||||
from ..type_utils import normalize_alphanumeric_key
|
||||
from ..type_utils import build_model_result, normalize_alphanumeric_key
|
||||
|
||||
POPULAR_MODELS_FILE = os.path.join(METADATA_DIR, "popular-models.json")
|
||||
MODEL_ALIASES_FILE = os.path.join(METADATA_DIR, "model-aliases.json")
|
||||
@@ -129,7 +130,7 @@ def load_base_model_aliases() -> Dict[str, List[str]]:
|
||||
return _base_models_aliases_cache
|
||||
|
||||
|
||||
def get_popular_model_url(filename: str) -> Optional[Dict[str, Any]]:
|
||||
def get_popular_model_url(filename: str) -> Optional[ModelCatalogEntry]:
|
||||
"""
|
||||
Look up a model filename in the popular models database.
|
||||
|
||||
@@ -137,19 +138,30 @@ def get_popular_model_url(filename: str) -> Optional[Dict[str, Any]]:
|
||||
filename: Model filename to look up
|
||||
|
||||
Returns:
|
||||
Dictionary with url, type, directory if found, None otherwise
|
||||
Typed catalog entry with URL, type and directory if found
|
||||
"""
|
||||
models = _load_popular_models()
|
||||
|
||||
# Direct lookup
|
||||
if filename in models:
|
||||
return models[filename].copy()
|
||||
try:
|
||||
return ModelCatalogEntry.from_mapping(
|
||||
models[filename],
|
||||
filename=filename,
|
||||
)
|
||||
except (TypeError, ValueError) as exc:
|
||||
log.warning(f"Invalid popular model entry for {filename}: {exc}")
|
||||
return None
|
||||
|
||||
# Try lowercase
|
||||
filename_lower = filename.lower()
|
||||
for name, info in models.items():
|
||||
if name.lower() == filename_lower:
|
||||
return info.copy()
|
||||
try:
|
||||
return ModelCatalogEntry.from_mapping(info, filename=name)
|
||||
except (TypeError, ValueError) as exc:
|
||||
log.warning(f"Invalid popular model entry for {name}: {exc}")
|
||||
return None
|
||||
|
||||
# Try aliases
|
||||
aliases = _load_model_aliases()
|
||||
@@ -158,14 +170,21 @@ def get_popular_model_url(filename: str) -> Optional[Dict[str, Any]]:
|
||||
filename in alias_list
|
||||
or filename_lower in [a.lower() for a in alias_list]
|
||||
) and canonical in models:
|
||||
result = models[canonical].copy()
|
||||
result["canonical_name"] = canonical
|
||||
return result
|
||||
try:
|
||||
return ModelCatalogEntry.from_mapping(
|
||||
models[canonical],
|
||||
filename=canonical,
|
||||
).with_extra(canonical_name=canonical)
|
||||
except (TypeError, ValueError) as exc:
|
||||
log.warning(
|
||||
f"Invalid popular model entry for {canonical}: {exc}"
|
||||
)
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def search_popular_models(query: str, limit: int = 10) -> List[Dict[str, Any]]:
|
||||
def search_popular_models(query: str, limit: int = 10) -> List[SearchResult]:
|
||||
"""
|
||||
Search popular models database by filename pattern.
|
||||
|
||||
@@ -174,7 +193,7 @@ def search_popular_models(query: str, limit: int = 10) -> List[Dict[str, Any]]:
|
||||
limit: Maximum results to return
|
||||
|
||||
Returns:
|
||||
List of matching models with url info
|
||||
Typed search results with URL information
|
||||
"""
|
||||
models = _load_popular_models()
|
||||
query_lower = query.lower()
|
||||
@@ -182,9 +201,29 @@ def search_popular_models(query: str, limit: int = 10) -> List[Dict[str, Any]]:
|
||||
results = []
|
||||
for name, info in models.items():
|
||||
if query_lower in name.lower():
|
||||
result = info.copy()
|
||||
result["filename"] = name
|
||||
results.append(result)
|
||||
try:
|
||||
catalog_entry = ModelCatalogEntry.from_mapping(
|
||||
info,
|
||||
filename=name,
|
||||
)
|
||||
except (TypeError, ValueError) as exc:
|
||||
log.warning(f"Invalid popular model entry for {name}: {exc}")
|
||||
continue
|
||||
results.append(
|
||||
build_model_result(
|
||||
"popular",
|
||||
filename=catalog_entry.filename or name,
|
||||
name=catalog_entry.name or name,
|
||||
url=catalog_entry.url or catalog_entry.download_url or "",
|
||||
download_url=catalog_entry.download_url,
|
||||
type=catalog_entry.model_type,
|
||||
size=catalog_entry.size,
|
||||
match_type=catalog_entry.match_type or "similar",
|
||||
confidence=catalog_entry.confidence,
|
||||
directory=catalog_entry.directory,
|
||||
**dict(catalog_entry.extra),
|
||||
)
|
||||
)
|
||||
|
||||
if len(results) >= limit:
|
||||
break
|
||||
|
||||
+57
-77
@@ -8,6 +8,8 @@ import math
|
||||
import re
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from .contracts import ProviderUrlReference, SearchResult
|
||||
|
||||
|
||||
def as_dict(value: Any) -> Dict[str, Any]:
|
||||
"""
|
||||
@@ -706,7 +708,10 @@ def extract_response_file_size(response: Any) -> Optional[int]:
|
||||
return None
|
||||
|
||||
|
||||
def parse_civitai_model_path(path: str, query_string: str) -> Optional[Dict[str, int]]:
|
||||
def parse_civitai_model_path(
|
||||
path: str,
|
||||
query_string: str,
|
||||
) -> Optional[ProviderUrlReference]:
|
||||
"""
|
||||
Common helper to parse `/models/{model_id}?modelVersionId={version_id}` URLs
|
||||
for CivitAI and CivArchive.
|
||||
@@ -716,15 +721,17 @@ def parse_civitai_model_path(path: str, query_string: str) -> Optional[Dict[str,
|
||||
|
||||
model_match = re.search(r"/models/(\d+)", path)
|
||||
if model_match:
|
||||
result = {"model_id": int(model_match.group(1))}
|
||||
model_id = int(model_match.group(1))
|
||||
query = parse_qs(query_string)
|
||||
version_id_list = query.get("modelVersionId")
|
||||
if version_id_list:
|
||||
try:
|
||||
result["version_id"] = int(version_id_list[0])
|
||||
version_id = int(version_id_list[0])
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
return result
|
||||
version_id = None
|
||||
else:
|
||||
version_id = None
|
||||
return ProviderUrlReference(model_id=model_id, version_id=version_id)
|
||||
return None
|
||||
|
||||
|
||||
@@ -887,8 +894,10 @@ def normalize_sha256(value: Any) -> str:
|
||||
"""Return a normalized SHA256 hex string or an empty string."""
|
||||
if value is None:
|
||||
return ""
|
||||
if not isinstance(value, str):
|
||||
return ""
|
||||
|
||||
text = str(value).strip()
|
||||
text = value.strip()
|
||||
for prefix in ("sha256:", "sha256="):
|
||||
if text.lower().startswith(prefix):
|
||||
text = text[len(prefix):].strip()
|
||||
@@ -899,19 +908,19 @@ def normalize_sha256(value: Any) -> str:
|
||||
return text.lower() if SHA256_PATTERN.match(text) else ""
|
||||
|
||||
|
||||
def extract_file_sha256(file_info: Any) -> Any:
|
||||
def extract_file_sha256(file_info: Any) -> str:
|
||||
"""Extract a provider file's SHA256 value using the common field precedence."""
|
||||
if not isinstance(file_info, dict):
|
||||
return ""
|
||||
|
||||
hashes = file_info.get("hashes") if isinstance(file_info.get("hashes"), dict) else {}
|
||||
return (
|
||||
file_info.get("sha256")
|
||||
or file_info.get("hash")
|
||||
or hashes.get("SHA256")
|
||||
or hashes.get("sha256")
|
||||
or ""
|
||||
value = first_non_empty(
|
||||
file_info.get("sha256"),
|
||||
file_info.get("hash"),
|
||||
hashes.get("SHA256"),
|
||||
hashes.get("sha256"),
|
||||
)
|
||||
return value if isinstance(value, str) else ""
|
||||
|
||||
|
||||
def normalize_hashes_dict(hashes: Optional[Dict[str, Any]]) -> Dict[str, str]:
|
||||
@@ -1310,7 +1319,7 @@ def normalize_model_file_info(
|
||||
) -> Dict[str, Any]:
|
||||
"""Normalize a single model file dictionary to a unified schema."""
|
||||
hashes = raw.get("hashes") if isinstance(raw.get("hashes"), dict) else {}
|
||||
sha256 = raw.get("sha256") or hashes.get("SHA256") or hashes.get("sha256") or ""
|
||||
sha256 = extract_file_sha256(raw)
|
||||
|
||||
size = raw.get("size")
|
||||
if size is None and raw.get("sizeKB") is not None:
|
||||
@@ -1331,16 +1340,20 @@ def normalize_model_file_info(
|
||||
"sha256": sha256,
|
||||
"hashes": hashes,
|
||||
"metadata": raw.get("metadata") if isinstance(raw.get("metadata"), dict) else {},
|
||||
"model_id": model_id or raw.get("modelId"),
|
||||
"version_id": version_id or raw.get("modelVersionId"),
|
||||
"model_id": model_id if model_id is not None else raw.get("modelId"),
|
||||
"version_id": (
|
||||
version_id
|
||||
if version_id is not None
|
||||
else raw.get("modelVersionId")
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def build_model_result(
|
||||
source: str,
|
||||
*,
|
||||
model_id: Any = None,
|
||||
version_id: Any = None,
|
||||
model_id: int | str | None = None,
|
||||
version_id: int | str | None = None,
|
||||
name: str = "",
|
||||
version_name: str = "",
|
||||
type: str = "",
|
||||
@@ -1362,79 +1375,46 @@ def build_model_result(
|
||||
custom_url: bool = False,
|
||||
result_mode: str = "search",
|
||||
**extra: Any,
|
||||
) -> Dict[str, Any]:
|
||||
) -> SearchResult:
|
||||
"""Build the shared model-result contract for search and custom URL results."""
|
||||
if normalize_hashes:
|
||||
hashes = normalize_hashes_dict(hashes)
|
||||
if not sha256 and "sha256" in hashes:
|
||||
sha256 = hashes["sha256"]
|
||||
|
||||
if result_mode not in {"search", "custom_url", "compact_custom_url"}:
|
||||
raise ValueError(f"Unsupported model result mode: {result_mode}")
|
||||
|
||||
if result_mode == "compact_custom_url":
|
||||
result = {
|
||||
result = SearchResult.from_mapping(
|
||||
{
|
||||
"source": source,
|
||||
"details_source": details_source or source,
|
||||
"model_id": model_id,
|
||||
"version_id": version_id,
|
||||
"name": name,
|
||||
"version_name": version_name,
|
||||
"type": type,
|
||||
"filename": filename,
|
||||
"url": url,
|
||||
"version_url": version_url or url,
|
||||
"download_url": download_url,
|
||||
"size": size,
|
||||
"base_model": base_model,
|
||||
"tags": tags,
|
||||
"match_type": match_type,
|
||||
"confidence": confidence,
|
||||
"sha256": sha256,
|
||||
"hashes": hashes,
|
||||
"trained_words": trained_words,
|
||||
"images": images,
|
||||
"details_source": details_source,
|
||||
"version_url": version_url,
|
||||
"custom_url": custom_url,
|
||||
"result_mode": result_mode,
|
||||
}
|
||||
if model_id is not None:
|
||||
result["model_id"] = model_id
|
||||
if version_id is not None:
|
||||
result["version_id"] = version_id
|
||||
result.update(extra)
|
||||
return result
|
||||
|
||||
result = {
|
||||
"source": source,
|
||||
"model_id": model_id,
|
||||
"version_id": version_id,
|
||||
"name": name,
|
||||
"version_name": version_name,
|
||||
"type": type,
|
||||
"filename": filename,
|
||||
"url": url,
|
||||
"download_url": download_url,
|
||||
"size": size,
|
||||
"base_model": base_model,
|
||||
"tags": tags or [],
|
||||
"match_type": match_type,
|
||||
"confidence": confidence,
|
||||
"sha256": sha256,
|
||||
"hashes": hashes or {},
|
||||
"trained_words": trained_words or [],
|
||||
"images": images or [],
|
||||
}
|
||||
|
||||
if result_mode == "custom_url":
|
||||
result.pop("confidence", None)
|
||||
result.update(
|
||||
{
|
||||
"details_source": details_source or source,
|
||||
"version_url": version_url or url,
|
||||
"match_type": "custom_url",
|
||||
"custom_url": custom_url,
|
||||
}
|
||||
)
|
||||
else:
|
||||
if details_source is not None:
|
||||
result["details_source"] = details_source
|
||||
if version_url is not None:
|
||||
result["version_url"] = version_url
|
||||
if custom_url:
|
||||
result["custom_url"] = True
|
||||
|
||||
result.update(extra)
|
||||
return result
|
||||
)
|
||||
return result.with_extra(**extra)
|
||||
|
||||
|
||||
def parse_provider_model_url(url: str, allowed_domains: list[str]) -> Optional[Dict[str, Any]]:
|
||||
def parse_provider_model_url(
|
||||
url: str,
|
||||
allowed_domains: list[str],
|
||||
) -> Optional[ProviderUrlReference]:
|
||||
"""Parse a provider model URL (e.g. CivitAI or mirror site) to extract model/version info."""
|
||||
import re
|
||||
from urllib.parse import urlparse
|
||||
@@ -1457,13 +1437,13 @@ def parse_provider_model_url(url: str, allowed_domains: list[str]) -> Optional[D
|
||||
# Standard CivArchive sha256 mirrors pattern
|
||||
sha_match = re.search(r"/sha256/([a-fA-F0-9]{64})", parsed.path)
|
||||
if sha_match:
|
||||
return {"sha256": sha_match.group(1).lower()}
|
||||
return ProviderUrlReference(sha256=sha_match.group(1).lower())
|
||||
|
||||
# Standard CivitAI download pattern
|
||||
if "/api/download/models/" in parsed.path:
|
||||
match = re.search(r"/api/download/models/(\d+)", parsed.path)
|
||||
if match:
|
||||
return {"version_id": int(match.group(1))}
|
||||
return ProviderUrlReference(version_id=int(match.group(1)))
|
||||
|
||||
return parse_civitai_model_path(parsed.path, parsed.query)
|
||||
|
||||
|
||||
+92
-66
@@ -2,8 +2,9 @@
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
from typing import Any, Callable, Dict, List, Optional, Sequence
|
||||
|
||||
from ..contracts import MissingModel, ModelReference, ResolvedModel
|
||||
from ..log_system import create_module_logger
|
||||
from . import references
|
||||
from . import subgraphs as subgraph_utils
|
||||
@@ -203,7 +204,7 @@ def _get_promoted_widget_context_cache_key(
|
||||
|
||||
def analyze_workflow_models(
|
||||
workflow_json: Dict[str, Any],
|
||||
available_models: Optional[List[Dict[str, Any]]] = None,
|
||||
available_models: Optional[List[ResolvedModel]] = None,
|
||||
progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
|
||||
*,
|
||||
previous_node_cache: Optional[
|
||||
@@ -214,7 +215,7 @@ def analyze_workflow_models(
|
||||
] = None,
|
||||
analysis_stats: Optional[Dict[str, int]] = None,
|
||||
analysis_context: Optional[str] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
) -> List[ModelReference]:
|
||||
"""
|
||||
Extract all model references from a workflow, including nested subgraphs.
|
||||
|
||||
@@ -224,10 +225,10 @@ def analyze_workflow_models(
|
||||
progress_callback: Optional callback called while workflow nodes are scanned
|
||||
|
||||
Returns:
|
||||
List of model reference dictionaries (same format as get_node_model_info)
|
||||
Each dict includes 'subgraph_id' if the model is in a subgraph
|
||||
Typed model references. Each reference includes ``subgraph_id`` when
|
||||
the model is in a subgraph.
|
||||
"""
|
||||
all_model_refs = []
|
||||
all_model_refs: List[ModelReference] = []
|
||||
reused_nodes = 0
|
||||
analyzed_nodes = 0
|
||||
log_context = f" ({analysis_context})" if analysis_context else ""
|
||||
@@ -315,7 +316,12 @@ def analyze_workflow_models(
|
||||
and cached_node.get("fingerprint") == node_fingerprint
|
||||
)
|
||||
if reused:
|
||||
base_model_refs = cached_node.get("refs", [])
|
||||
base_model_refs = [
|
||||
ref
|
||||
if isinstance(ref, ModelReference)
|
||||
else ModelReference.from_mapping(ref)
|
||||
for ref in cached_node.get("refs", [])
|
||||
]
|
||||
reused_nodes += 1
|
||||
else:
|
||||
base_model_refs = references.get_node_model_info(
|
||||
@@ -323,10 +329,9 @@ def analyze_workflow_models(
|
||||
available_models=available_models,
|
||||
)
|
||||
analyzed_nodes += 1
|
||||
# Keep cached node references as raw extraction results. Subgraph
|
||||
# instance context is workflow-level state and must be reapplied
|
||||
# when a cached node is reused.
|
||||
model_refs = [dict(ref) for ref in base_model_refs]
|
||||
# Subgraph instance context is workflow-level state and must be
|
||||
# reapplied when a cached node is reused.
|
||||
model_refs = list(base_model_refs)
|
||||
node_type = node.get("type", "")
|
||||
|
||||
# Check if node type is a subgraph UUID
|
||||
@@ -340,26 +345,32 @@ def analyze_workflow_models(
|
||||
# For top-level subgraph instance nodes, subgraph_path is None
|
||||
# This distinguishes them from nodes within subgraph definitions
|
||||
if subgraph_id:
|
||||
for ref in model_refs:
|
||||
for index, ref in enumerate(model_refs):
|
||||
context = promoted_widget_contexts.get(
|
||||
"instance_widgets", {}
|
||||
).get((str(node.get("id")), ref.get("widget_index")))
|
||||
).get((str(node.get("id")), ref.widget_index))
|
||||
if context:
|
||||
promoted_instance_slots.add(
|
||||
(str(node.get("id")), ref.get("widget_index"))
|
||||
(str(node.get("id")), ref.widget_index)
|
||||
)
|
||||
subgraph_utils._apply_instance_promoted_widget_context(
|
||||
ref, context, available_models
|
||||
model_refs[index] = (
|
||||
subgraph_utils._apply_instance_promoted_widget_context(
|
||||
ref,
|
||||
context,
|
||||
available_models,
|
||||
)
|
||||
)
|
||||
for ref in model_refs:
|
||||
ref["subgraph_id"] = subgraph_id
|
||||
ref["subgraph_name"] = subgraph_name
|
||||
ref["subgraph_path"] = None
|
||||
ref["is_top_level"] = True
|
||||
for index, ref in enumerate(model_refs):
|
||||
model_refs[index] = ref.with_updates(
|
||||
subgraph_id=subgraph_id,
|
||||
subgraph_name=subgraph_name,
|
||||
subgraph_path=None,
|
||||
is_top_level=True,
|
||||
)
|
||||
if node_cache_out is not None:
|
||||
node_cache_out[node_cache_key] = {
|
||||
"fingerprint": node_fingerprint,
|
||||
"refs": base_model_refs,
|
||||
"refs": list(base_model_refs),
|
||||
}
|
||||
report_node_progress(node, reused=reused)
|
||||
all_model_refs.extend(model_refs)
|
||||
@@ -404,7 +415,12 @@ def analyze_workflow_models(
|
||||
and cached_node.get("fingerprint") == node_fingerprint
|
||||
)
|
||||
if reused:
|
||||
base_model_refs = cached_node.get("refs", [])
|
||||
base_model_refs = [
|
||||
ref
|
||||
if isinstance(ref, ModelReference)
|
||||
else ModelReference.from_mapping(ref)
|
||||
for ref in cached_node.get("refs", [])
|
||||
]
|
||||
reused_nodes += 1
|
||||
reused_subgraph_nodes += 1
|
||||
else:
|
||||
@@ -414,18 +430,19 @@ def analyze_workflow_models(
|
||||
)
|
||||
analyzed_nodes += 1
|
||||
analyzed_subgraph_nodes += 1
|
||||
model_refs = []
|
||||
model_refs: List[ModelReference] = []
|
||||
for base_ref in base_model_refs:
|
||||
scoped_ref = dict(base_ref)
|
||||
scoped_ref["subgraph_id"] = subgraph_id
|
||||
scoped_ref["subgraph_name"] = subgraph_name
|
||||
scoped_ref["subgraph_path"] = [
|
||||
"definitions",
|
||||
"subgraphs",
|
||||
subgraph_id,
|
||||
"nodes",
|
||||
]
|
||||
scoped_ref["is_top_level"] = False
|
||||
scoped_ref = base_ref.with_updates(
|
||||
subgraph_id=subgraph_id,
|
||||
subgraph_name=subgraph_name,
|
||||
subgraph_path=[
|
||||
"definitions",
|
||||
"subgraphs",
|
||||
subgraph_id,
|
||||
"nodes",
|
||||
],
|
||||
is_top_level=False,
|
||||
)
|
||||
promoted_refs = subgraph_utils._promote_model_reference_to_instances(
|
||||
scoped_ref,
|
||||
promoted_widget_contexts,
|
||||
@@ -437,15 +454,14 @@ def analyze_workflow_models(
|
||||
continue
|
||||
|
||||
ref = scoped_ref
|
||||
subgraph_utils._apply_promoted_widget_locator(
|
||||
ref,
|
||||
promoted_widget_contexts,
|
||||
ref = subgraph_utils._apply_promoted_widget_locator(
|
||||
ref, promoted_widget_contexts
|
||||
)
|
||||
model_refs.append(ref)
|
||||
if node_cache_out is not None:
|
||||
node_cache_out[node_cache_key] = {
|
||||
"fingerprint": node_fingerprint,
|
||||
"refs": base_model_refs,
|
||||
"refs": list(base_model_refs),
|
||||
}
|
||||
report_node_progress(
|
||||
node,
|
||||
@@ -484,8 +500,9 @@ def analyze_workflow_models(
|
||||
return all_model_refs
|
||||
|
||||
def identify_missing_models(
|
||||
workflow_models: List[Dict[str, Any]], available_models: List[Dict[str, str]] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
workflow_models: Sequence[ModelReference],
|
||||
available_models: Optional[Sequence[ResolvedModel]] = None,
|
||||
) -> List[MissingModel]:
|
||||
"""
|
||||
Identify which models from the workflow are missing.
|
||||
Deduplicates by category and filename - same model file only appears once even if
|
||||
@@ -496,48 +513,57 @@ def identify_missing_models(
|
||||
available_models: Optional list of available models (if None, checks via folder_paths)
|
||||
|
||||
Returns:
|
||||
List of missing model references (deduplicated by category and filename).
|
||||
Each entry has 'all_node_refs' containing all node references for that model.
|
||||
Typed missing model references (deduplicated by category and filename).
|
||||
Each entry retains 'all_node_refs' for HTTP compatibility.
|
||||
"""
|
||||
# Group missing models by category and filename to deduplicate workflow
|
||||
# references without merging unrelated model folders that happen to share a name.
|
||||
missing_by_model: Dict[tuple[str, str], Dict[str, Any]] = {}
|
||||
missing_by_model: Dict[tuple[str, str], MissingModel] = {}
|
||||
|
||||
for model_ref in workflow_models:
|
||||
for raw_model_ref in workflow_models:
|
||||
model_ref = (
|
||||
raw_model_ref
|
||||
if isinstance(raw_model_ref, ModelReference)
|
||||
else ModelReference.from_mapping(raw_model_ref)
|
||||
)
|
||||
# If exists is False, it's missing
|
||||
if not model_ref.get("exists", False):
|
||||
filename = model_ref.get("original_path", "")
|
||||
category = model_ref.get("category", "")
|
||||
if not model_ref.exists:
|
||||
filename = model_ref.original_path
|
||||
category = model_ref.category
|
||||
group_key = (str(category or ""), str(filename or ""))
|
||||
|
||||
if group_key not in missing_by_model:
|
||||
# First occurrence - use this as the primary entry
|
||||
missing_by_model[group_key] = {
|
||||
**model_ref,
|
||||
"reference_count": 1,
|
||||
"all_node_refs": [
|
||||
model_ref.copy()
|
||||
], # Track all nodes needing this model
|
||||
}
|
||||
missing_by_model[group_key] = MissingModel(
|
||||
reference=model_ref,
|
||||
all_node_refs=(model_ref,),
|
||||
extra=dict(model_ref.extra),
|
||||
)
|
||||
else:
|
||||
# Duplicate - just add to the node refs list
|
||||
existing = missing_by_model[group_key]
|
||||
existing["all_node_refs"].append(model_ref.copy())
|
||||
existing["reference_count"] = len(existing["all_node_refs"])
|
||||
if model_ref.get("auto_download_capable"):
|
||||
existing["auto_download_capable"] = True
|
||||
if model_ref.get("auto_download_candidate"):
|
||||
existing["auto_download_candidate"] = True
|
||||
if model_ref.get("input_choice_matches_value"):
|
||||
existing["input_choice_matches_value"] = True
|
||||
existing_source = str(existing.get("input_choice_source") or "").lower()
|
||||
model_source = str(model_ref.get("input_choice_source") or "").lower()
|
||||
updates = {}
|
||||
for key in (
|
||||
"auto_download_capable",
|
||||
"auto_download_candidate",
|
||||
"input_choice_matches_value",
|
||||
):
|
||||
if model_ref.extra_value(key):
|
||||
updates[key] = True
|
||||
existing_source = str(
|
||||
existing.extra_value("input_choice_source") or ""
|
||||
).lower()
|
||||
model_source = str(
|
||||
model_ref.extra_value("input_choice_source") or ""
|
||||
).lower()
|
||||
if existing_source != "hybrid" and model_source in {
|
||||
"static",
|
||||
"hybrid",
|
||||
"workflow_schema",
|
||||
}:
|
||||
existing["input_choice_source"] = model_source
|
||||
updates["input_choice_source"] = model_source
|
||||
missing_by_model[group_key] = existing.with_references(
|
||||
(*existing.all_node_refs, model_ref)
|
||||
).with_extra(**updates)
|
||||
|
||||
# Return deduplicated list
|
||||
return list(missing_by_model.values())
|
||||
|
||||
+30
-17
@@ -5,6 +5,7 @@ import time
|
||||
from collections import OrderedDict
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
from ..contracts import ModelReference, WorkflowModelInventory
|
||||
from ..log_system import create_module_logger
|
||||
from . import analysis
|
||||
|
||||
@@ -17,18 +18,18 @@ _WORKFLOW_MODEL_INVENTORY_CACHE: OrderedDict[str, Dict[str, Any]] = OrderedDict(
|
||||
|
||||
def _build_workflow_node_cache(
|
||||
workflow_json: Dict[str, Any],
|
||||
model_refs: List[Dict[str, Any]],
|
||||
model_refs: List[ModelReference],
|
||||
) -> Dict[tuple[str, str, str], Dict[str, Any]]:
|
||||
fingerprints = analysis._get_workflow_node_fingerprints(workflow_json)
|
||||
refs_by_node: Dict[tuple[str, str, str], List[Dict[str, Any]]] = {
|
||||
refs_by_node: Dict[tuple[str, str, str], List[ModelReference]] = {
|
||||
key: [] for key in fingerprints
|
||||
}
|
||||
|
||||
for ref in model_refs:
|
||||
is_top_level = ref.get("is_top_level") is not False
|
||||
is_top_level = ref.is_top_level is not False
|
||||
key = analysis._get_workflow_scope_key(
|
||||
ref.get("node_id"),
|
||||
subgraph_id=ref.get("subgraph_id"),
|
||||
ref.node_id,
|
||||
subgraph_id=ref.subgraph_id,
|
||||
is_top_level=is_top_level,
|
||||
)
|
||||
if key in refs_by_node:
|
||||
@@ -71,16 +72,26 @@ def get_workflow_model_inventory(
|
||||
force_rescan: bool = False,
|
||||
progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
|
||||
analysis_id: Optional[str] = None,
|
||||
) -> Dict[str, List[Dict[str, Any]]]:
|
||||
) -> WorkflowModelInventory:
|
||||
"""
|
||||
Return the shared base model inventory for a workflow.
|
||||
|
||||
Missing Models and Loaded Models use this snapshot so the same unchanged
|
||||
workflow is not parsed separately by both endpoints. Callers must treat the
|
||||
returned lists as read-only.
|
||||
returned tuples as read-only.
|
||||
"""
|
||||
from ..scanner import get_model_files
|
||||
|
||||
def normalize_model_refs(values: List[Any]) -> List[ModelReference]:
|
||||
if not isinstance(values, (list, tuple)):
|
||||
raise TypeError("workflow model references must be an array")
|
||||
return [
|
||||
value
|
||||
if isinstance(value, ModelReference)
|
||||
else ModelReference.from_mapping(value)
|
||||
for value in values
|
||||
]
|
||||
|
||||
cache_key = _get_workflow_model_inventory_cache_key(workflow_json)
|
||||
analysis_context = _get_analysis_log_context(cache_key, analysis_id)
|
||||
promoted_context_key = analysis._get_promoted_widget_context_cache_key(workflow_json)
|
||||
@@ -124,7 +135,7 @@ def get_workflow_model_inventory(
|
||||
previous_inventory = entry
|
||||
|
||||
if cached is not None:
|
||||
model_refs = cached["model_refs"]
|
||||
model_refs = normalize_model_refs(cached["model_refs"])
|
||||
if progress_callback:
|
||||
progress_callback(
|
||||
{
|
||||
@@ -138,15 +149,15 @@ def get_workflow_model_inventory(
|
||||
log.debug(
|
||||
f"Reusing shared workflow model analysis ({analysis_context})"
|
||||
)
|
||||
return {
|
||||
"available_models": cached["available_models"],
|
||||
"model_refs": model_refs,
|
||||
}
|
||||
return WorkflowModelInventory(
|
||||
available_models=tuple(cached["available_models"]),
|
||||
model_refs=tuple(model_refs),
|
||||
)
|
||||
|
||||
node_cache = {}
|
||||
analysis_stats = {}
|
||||
if previous_inventory is not None:
|
||||
available_models = previous_inventory["available_models"]
|
||||
available_models = list(previous_inventory["available_models"])
|
||||
model_refs = analysis.analyze_workflow_models(
|
||||
workflow_json,
|
||||
available_models=available_models,
|
||||
@@ -156,6 +167,7 @@ def get_workflow_model_inventory(
|
||||
analysis_stats=analysis_stats,
|
||||
analysis_context=analysis_context,
|
||||
)
|
||||
model_refs = normalize_model_refs(model_refs)
|
||||
else:
|
||||
available_models = get_model_files(force_rescan=force_rescan)
|
||||
model_refs = analysis.analyze_workflow_models(
|
||||
@@ -164,6 +176,7 @@ def get_workflow_model_inventory(
|
||||
progress_callback=progress_callback,
|
||||
analysis_context=analysis_context,
|
||||
)
|
||||
model_refs = normalize_model_refs(model_refs)
|
||||
node_cache = _build_workflow_node_cache(workflow_json, model_refs)
|
||||
analysis_stats = {
|
||||
"total_nodes": len(node_cache),
|
||||
@@ -196,7 +209,7 @@ def get_workflow_model_inventory(
|
||||
):
|
||||
_WORKFLOW_MODEL_INVENTORY_CACHE.popitem(last=False)
|
||||
|
||||
return {
|
||||
"available_models": available_models,
|
||||
"model_refs": model_refs,
|
||||
}
|
||||
return WorkflowModelInventory(
|
||||
available_models=tuple(available_models),
|
||||
model_refs=tuple(model_refs),
|
||||
)
|
||||
|
||||
+30
-25
@@ -4,11 +4,14 @@ import os
|
||||
import re
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from ..contracts import ModelReference, ResolvedModel
|
||||
from ..custom_nodes import (
|
||||
analyze_custom_node_references,
|
||||
get_custom_node_model_adapter,
|
||||
)
|
||||
from ..log_system import create_module_logger
|
||||
from ..metadata_model_utils import normalize_models
|
||||
from ..path_utils import normalize_string_values
|
||||
from ..type_utils import MODEL_EXTENSIONS, URN_TYPE_MAP, normalize_download_category
|
||||
from . import dynamic_widgets
|
||||
from .widgets import (
|
||||
@@ -209,7 +212,7 @@ def _normalize_model_path_for_lookup(value: str) -> str:
|
||||
def _resolve_from_available_models(
|
||||
filename: str,
|
||||
categories: Optional[List[str]],
|
||||
available_models: Optional[List[Dict[str, Any]]],
|
||||
available_models: Optional[List[ResolvedModel]],
|
||||
) -> Optional[tuple[str, str]]:
|
||||
"""Resolve using scanner data with case-sensitive relative path matching."""
|
||||
if not available_models:
|
||||
@@ -223,11 +226,13 @@ def _resolve_from_available_models(
|
||||
if requested_is_absolute:
|
||||
requested_key = _normalize_model_path_for_lookup(os.path.abspath(filename))
|
||||
|
||||
normalized_models = normalize_models(available_models)
|
||||
|
||||
if categories is None:
|
||||
category_order = []
|
||||
seen_categories = set()
|
||||
for model in available_models:
|
||||
category = model.get("category")
|
||||
for model in normalized_models:
|
||||
category = model.category
|
||||
if category and category not in seen_categories:
|
||||
seen_categories.add(category)
|
||||
category_order.append(category)
|
||||
@@ -239,8 +244,8 @@ def _resolve_from_available_models(
|
||||
|
||||
for category in category_order:
|
||||
normalized_category = normalize_download_category(category)
|
||||
for model in available_models:
|
||||
model_category = model.get("category")
|
||||
for model in normalized_models:
|
||||
model_category = model.category
|
||||
if (
|
||||
model_category != category
|
||||
and normalize_download_category(model_category) != normalized_category
|
||||
@@ -248,14 +253,14 @@ def _resolve_from_available_models(
|
||||
continue
|
||||
|
||||
if requested_is_absolute:
|
||||
model_path = model.get("path") or ""
|
||||
model_path = model.path or ""
|
||||
else:
|
||||
model_path = model.get("relative_path") or model.get("filename") or ""
|
||||
model_path = model.relative_path or model.filename or ""
|
||||
|
||||
if _normalize_model_path_for_lookup(model_path) != requested_key:
|
||||
continue
|
||||
|
||||
full_path = model.get("path")
|
||||
full_path = model.path
|
||||
if full_path and os.path.exists(full_path):
|
||||
return (category, full_path)
|
||||
|
||||
@@ -272,7 +277,9 @@ def _category_has_exact_filename(category: str, filename: str) -> Optional[bool]
|
||||
return False
|
||||
|
||||
try:
|
||||
available_filenames = folder_paths.get_filename_list(category) or []
|
||||
available_filenames = normalize_string_values(
|
||||
folder_paths.get_filename_list(category)
|
||||
)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
@@ -286,7 +293,7 @@ def _category_has_exact_filename(category: str, filename: str) -> Optional[bool]
|
||||
def try_resolve_model_path(
|
||||
value: str,
|
||||
categories: List[str] = None,
|
||||
available_models: Optional[List[Dict[str, Any]]] = None,
|
||||
available_models: Optional[List[ResolvedModel]] = None,
|
||||
) -> Optional[tuple[str, str]]:
|
||||
"""Resolve a workflow model value using scanner data or ComfyUI paths."""
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
@@ -336,8 +343,8 @@ def try_resolve_model_path(
|
||||
return None
|
||||
|
||||
def get_node_model_info(
|
||||
node: Dict[str, Any], available_models: Optional[List[Dict[str, Any]]] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
node: Dict[str, Any], available_models: Optional[List[ResolvedModel]] = None
|
||||
) -> List[ModelReference]:
|
||||
"""
|
||||
Extract model references from a single node.
|
||||
|
||||
@@ -348,16 +355,8 @@ def get_node_model_info(
|
||||
node: Node dictionary from workflow JSON
|
||||
|
||||
Returns:
|
||||
List of model reference dictionaries:
|
||||
{
|
||||
'node_id': node id,
|
||||
'node_type': node type,
|
||||
'widget_index': index in widgets_values,
|
||||
'original_path': original path from workflow,
|
||||
'category': model category (if found),
|
||||
'exists': True if model exists,
|
||||
'connected': True if node has any connected inputs/outputs
|
||||
}
|
||||
Typed model references. Provider/custom-node-specific fields are kept
|
||||
in ``ModelReference.extra``.
|
||||
"""
|
||||
model_refs = []
|
||||
node_id = node.get("id")
|
||||
@@ -380,7 +379,7 @@ def get_node_model_info(
|
||||
is_active = is_connected and not is_bypassed
|
||||
|
||||
if not widgets_values:
|
||||
return model_refs
|
||||
return []
|
||||
|
||||
custom_model_refs = analyze_custom_node_references(
|
||||
node,
|
||||
@@ -389,7 +388,10 @@ def get_node_model_info(
|
||||
get_widget_name_hint=get_widget_name_hint,
|
||||
)
|
||||
if custom_model_refs is not None:
|
||||
return custom_model_refs
|
||||
return [
|
||||
ref if isinstance(ref, ModelReference) else ModelReference.from_mapping(ref)
|
||||
for ref in custom_model_refs
|
||||
]
|
||||
|
||||
# For each widget value, check if it looks like a model file or URN
|
||||
for idx, value in enumerate(widgets_values):
|
||||
@@ -646,4 +648,7 @@ def get_node_model_info(
|
||||
}
|
||||
)
|
||||
|
||||
return model_refs
|
||||
return [
|
||||
ref if isinstance(ref, ModelReference) else ModelReference.from_mapping(ref)
|
||||
for ref in model_refs
|
||||
]
|
||||
|
||||
+83
-78
@@ -3,6 +3,7 @@
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from ..contracts import ModelReference, ResolvedModel
|
||||
from . import references
|
||||
from .widgets import (
|
||||
_has_widget_input,
|
||||
@@ -330,46 +331,50 @@ def _build_promoted_widget_contexts(
|
||||
|
||||
|
||||
def _apply_instance_promoted_widget_context(
|
||||
ref: Dict[str, Any],
|
||||
ref: ModelReference,
|
||||
context: Optional[Dict[str, Any]],
|
||||
available_models: Optional[List[Dict[str, Any]]],
|
||||
) -> None:
|
||||
available_models: Optional[List[ResolvedModel]],
|
||||
) -> ModelReference:
|
||||
if not context:
|
||||
return
|
||||
return ref
|
||||
|
||||
ref["promoted_widget_name"] = context.get("proxy_widget_name", "")
|
||||
ref["promoted_inner_node_id"] = context.get("node_id")
|
||||
ref["promoted_inner_node_type"] = context.get("node_type", "")
|
||||
ref["promoted_inner_node_title"] = context.get("node_title", "")
|
||||
ref["promoted_inner_widget_index"] = context.get("widget_index")
|
||||
ref["promoted_inner_widget_name"] = context.get("widget_name", "")
|
||||
updates = {
|
||||
"promoted_widget_name": context.get("proxy_widget_name", ""),
|
||||
"promoted_inner_node_id": context.get("node_id"),
|
||||
"promoted_inner_node_type": context.get("node_type", ""),
|
||||
"promoted_inner_node_title": context.get("node_title", ""),
|
||||
"promoted_inner_widget_index": context.get("widget_index"),
|
||||
"promoted_inner_widget_name": context.get("widget_name", ""),
|
||||
}
|
||||
|
||||
promoted_value = context.get("promoted_value")
|
||||
nested_key = ref.get("nested_key")
|
||||
nested_key = ref.extra_value("nested_key")
|
||||
if nested_key and isinstance(promoted_value, dict):
|
||||
promoted_value = promoted_value.get(nested_key)
|
||||
if _has_promoted_value(promoted_value):
|
||||
ref["original_path"] = str(promoted_value).strip()
|
||||
ref["is_urn"] = bool(references.URN_REGEX.match(ref["original_path"]))
|
||||
original_path = str(promoted_value).strip()
|
||||
updates["original_path"] = original_path
|
||||
updates["is_urn"] = bool(references.URN_REGEX.match(original_path))
|
||||
|
||||
category = context.get("category")
|
||||
original_path = ref.get("original_path", "")
|
||||
original_path = updates.get("original_path", ref.original_path)
|
||||
if not category or not original_path:
|
||||
return
|
||||
return ref.with_updates(**updates)
|
||||
|
||||
resolved = references.try_resolve_model_path(
|
||||
original_path,
|
||||
[category],
|
||||
available_models=available_models,
|
||||
)
|
||||
ref["category"] = category
|
||||
updates["category"] = category
|
||||
if resolved:
|
||||
_, full_path = resolved
|
||||
ref["full_path"] = full_path
|
||||
ref["exists"] = os.path.exists(full_path)
|
||||
updates["full_path"] = full_path
|
||||
updates["exists"] = os.path.exists(full_path)
|
||||
else:
|
||||
ref["full_path"] = None
|
||||
ref["exists"] = False
|
||||
updates["full_path"] = None
|
||||
updates["exists"] = False
|
||||
return ref.with_updates(**updates)
|
||||
|
||||
|
||||
def _select_promoted_locator(
|
||||
@@ -385,37 +390,37 @@ def _select_promoted_locator(
|
||||
|
||||
|
||||
def _get_promoted_widget_locators(
|
||||
ref: Dict[str, Any], contexts: Dict[str, Dict[Any, Any]]
|
||||
ref: ModelReference, contexts: Dict[str, Dict[Any, Any]]
|
||||
) -> List[Dict[str, Any]]:
|
||||
subgraph_id = str(ref.get("subgraph_id") or "")
|
||||
subgraph_id = str(ref.subgraph_id or "")
|
||||
if not subgraph_id:
|
||||
return []
|
||||
|
||||
exact_key = (subgraph_id, str(ref.get("node_id")), ref.get("widget_index"))
|
||||
exact_key = (subgraph_id, str(ref.node_id), ref.widget_index)
|
||||
locators = contexts.get("inner_widgets", {}).get(exact_key, [])
|
||||
if not locators and ref.get("widget_name"):
|
||||
if not locators and ref.extra_value("widget_name"):
|
||||
name_key = (
|
||||
subgraph_id,
|
||||
str(ref.get("node_id")),
|
||||
normalize_widget_name(ref.get("widget_name")),
|
||||
str(ref.node_id),
|
||||
normalize_widget_name(ref.extra_value("widget_name")),
|
||||
)
|
||||
locators = contexts.get("inner_widget_names", {}).get(name_key, [])
|
||||
return locators
|
||||
|
||||
|
||||
def _promote_model_reference_to_instances(
|
||||
ref: Dict[str, Any],
|
||||
ref: ModelReference,
|
||||
contexts: Dict[str, Dict[Any, Any]],
|
||||
available_models: Optional[List[Dict[str, Any]]] = None,
|
||||
available_models: Optional[List[ResolvedModel]] = None,
|
||||
existing_instance_slots: Optional[set] = None,
|
||||
) -> Optional[List[Dict[str, Any]]]:
|
||||
) -> Optional[List[ModelReference]]:
|
||||
locators = _get_promoted_widget_locators(ref, contexts)
|
||||
if not locators:
|
||||
return None
|
||||
|
||||
if existing_instance_slots is None:
|
||||
existing_instance_slots = set()
|
||||
promoted_refs = []
|
||||
promoted_refs: List[ModelReference] = []
|
||||
for locator in locators:
|
||||
instance_slot = (
|
||||
str(locator.get("node_id")),
|
||||
@@ -427,46 +432,43 @@ def _promote_model_reference_to_instances(
|
||||
continue
|
||||
|
||||
value = locator.get("promoted_value")
|
||||
nested_key = ref.get("nested_key")
|
||||
nested_key = ref.extra_value("nested_key")
|
||||
if nested_key and isinstance(value, dict):
|
||||
value = value.get(nested_key)
|
||||
if not _has_promoted_value(value):
|
||||
continue
|
||||
|
||||
value_str = str(value).strip()
|
||||
promoted = dict(ref)
|
||||
promoted.update(
|
||||
{
|
||||
"node_id": locator.get("node_id"),
|
||||
"node_type": locator.get("node_type", ""),
|
||||
"node_title": locator.get("node_title", ""),
|
||||
"widget_index": locator.get("proxy_widget_index"),
|
||||
"widget_name": locator.get("proxy_widget_name", ""),
|
||||
"original_path": value_str,
|
||||
"subgraph_id": locator.get("subgraph_id", ""),
|
||||
"subgraph_name": locator.get("subgraph_name", ""),
|
||||
"subgraph_path": None,
|
||||
"is_top_level": True,
|
||||
"promoted_widget_name": locator.get("proxy_widget_name", ""),
|
||||
"promoted_inner_node_id": ref.get("node_id"),
|
||||
"promoted_inner_node_type": ref.get("node_type", ""),
|
||||
"promoted_inner_node_title": ref.get("node_title", ""),
|
||||
"promoted_inner_widget_index": ref.get("widget_index"),
|
||||
"promoted_inner_widget_name": ref.get("widget_name", ""),
|
||||
"locate_node_id": locator.get("node_id"),
|
||||
"locate_node_type": locator.get("node_type", ""),
|
||||
"locate_node_title": locator.get("node_title", ""),
|
||||
"locate_subgraph_id": "",
|
||||
"locate_subgraph_name": locator.get("subgraph_name", ""),
|
||||
"locate_is_top_level": True,
|
||||
"locate_via_promoted_widget": True,
|
||||
"is_urn": bool(references.URN_REGEX.match(value_str)),
|
||||
}
|
||||
promoted = ref.with_updates(
|
||||
node_id=locator.get("node_id"),
|
||||
node_type=locator.get("node_type", ""),
|
||||
widget_index=locator.get("proxy_widget_index"),
|
||||
original_path=value_str,
|
||||
subgraph_id=locator.get("subgraph_id", ""),
|
||||
is_top_level=True,
|
||||
node_title=locator.get("node_title", ""),
|
||||
widget_name=locator.get("proxy_widget_name", ""),
|
||||
subgraph_name=locator.get("subgraph_name", ""),
|
||||
subgraph_path=None,
|
||||
promoted_widget_name=locator.get("proxy_widget_name", ""),
|
||||
promoted_inner_node_id=ref.node_id,
|
||||
promoted_inner_node_type=ref.node_type,
|
||||
promoted_inner_node_title=ref.extra_value("node_title", ""),
|
||||
promoted_inner_widget_index=ref.widget_index,
|
||||
promoted_inner_widget_name=ref.extra_value("widget_name", ""),
|
||||
locate_node_id=locator.get("node_id"),
|
||||
locate_node_type=locator.get("node_type", ""),
|
||||
locate_node_title=locator.get("node_title", ""),
|
||||
locate_subgraph_id="",
|
||||
locate_subgraph_name=locator.get("subgraph_name", ""),
|
||||
locate_is_top_level=True,
|
||||
locate_via_promoted_widget=True,
|
||||
is_urn=bool(references.URN_REGEX.match(value_str)),
|
||||
)
|
||||
|
||||
category_hints = promoted.get("category_hints") or []
|
||||
if not category_hints and promoted.get("category"):
|
||||
category_hints = [promoted["category"]]
|
||||
category_hints = promoted.extra_value("category_hints") or []
|
||||
if not category_hints and promoted.category:
|
||||
category_hints = [promoted.category]
|
||||
resolved = references.try_resolve_model_path(
|
||||
value_str,
|
||||
category_hints or None,
|
||||
@@ -474,12 +476,13 @@ def _promote_model_reference_to_instances(
|
||||
)
|
||||
if resolved:
|
||||
category, full_path = resolved
|
||||
promoted["category"] = category
|
||||
promoted["full_path"] = full_path
|
||||
promoted["exists"] = os.path.exists(full_path)
|
||||
promoted = promoted.with_updates(
|
||||
category=category,
|
||||
full_path=full_path,
|
||||
exists=os.path.exists(full_path),
|
||||
)
|
||||
else:
|
||||
promoted["full_path"] = None
|
||||
promoted["exists"] = False
|
||||
promoted = promoted.with_updates(full_path=None, exists=False)
|
||||
|
||||
promoted_refs.append(promoted)
|
||||
existing_instance_slots.add(instance_slot)
|
||||
@@ -488,17 +491,19 @@ def _promote_model_reference_to_instances(
|
||||
|
||||
|
||||
def _apply_promoted_widget_locator(
|
||||
ref: Dict[str, Any], contexts: Dict[str, Dict[Any, Any]]
|
||||
) -> None:
|
||||
ref: ModelReference, contexts: Dict[str, Dict[Any, Any]]
|
||||
) -> ModelReference:
|
||||
locators = _get_promoted_widget_locators(ref, contexts)
|
||||
locator = _select_promoted_locator(locators, ref.get("original_path", ""))
|
||||
locator = _select_promoted_locator(locators, ref.original_path)
|
||||
if not locator:
|
||||
return
|
||||
return ref
|
||||
|
||||
ref["locate_node_id"] = locator.get("node_id")
|
||||
ref["locate_node_type"] = locator.get("node_type", "")
|
||||
ref["locate_node_title"] = locator.get("node_title", "")
|
||||
ref["locate_subgraph_id"] = ""
|
||||
ref["locate_subgraph_name"] = locator.get("subgraph_name", "")
|
||||
ref["locate_is_top_level"] = True
|
||||
ref["locate_via_promoted_widget"] = True
|
||||
return ref.with_updates(
|
||||
locate_node_id=locator.get("node_id"),
|
||||
locate_node_type=locator.get("node_type", ""),
|
||||
locate_node_title=locator.get("node_title", ""),
|
||||
locate_subgraph_id="",
|
||||
locate_subgraph_name=locator.get("subgraph_name", ""),
|
||||
locate_is_top_level=True,
|
||||
locate_via_promoted_widget=True,
|
||||
)
|
||||
|
||||
+61
-82
@@ -7,11 +7,17 @@ Updates workflow JSON by replacing model paths in nodes.
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .contracts import Resolution, ResolvedModel
|
||||
from .custom_nodes import update_custom_node_model_path
|
||||
from .log_system import create_module_logger
|
||||
|
||||
log = create_module_logger(__name__)
|
||||
from .path_utils import get_filename_from_path, is_path_within
|
||||
from .path_utils import (
|
||||
get_filename_from_path,
|
||||
is_path_within,
|
||||
normalize_folder_path_values,
|
||||
normalize_string_values,
|
||||
)
|
||||
|
||||
|
||||
def convert_to_relative_path(
|
||||
@@ -45,7 +51,9 @@ def convert_to_relative_path(
|
||||
|
||||
# Get all available filenames for this category
|
||||
# This returns paths with OS-native separators (backslashes on Windows)
|
||||
available_filenames = folder_paths.get_filename_list(category)
|
||||
available_filenames = normalize_string_values(
|
||||
folder_paths.get_filename_list(category)
|
||||
)
|
||||
|
||||
# Try to find a matching entry in ComfyUI's list
|
||||
# Compare by finding the file that resolves to our absolute path
|
||||
@@ -83,31 +91,32 @@ def convert_to_relative_path(
|
||||
|
||||
|
||||
def get_base_directory_for_model(
|
||||
model_dict: Dict[str, str], category: str
|
||||
model: ResolvedModel, category: str
|
||||
) -> Optional[str]:
|
||||
"""
|
||||
Get the base directory for a model based on its metadata.
|
||||
|
||||
Args:
|
||||
model_dict: Model dictionary with 'base_directory' or 'path' key
|
||||
model: Typed local model with optional base directory and path
|
||||
category: Model category
|
||||
|
||||
Returns:
|
||||
Base directory path if found, None otherwise
|
||||
"""
|
||||
# Try to get base_directory from model dict
|
||||
if "base_directory" in model_dict:
|
||||
return model_dict["base_directory"]
|
||||
if model.base_directory:
|
||||
return model.base_directory
|
||||
|
||||
# If we have the full path, try to find the category base directory
|
||||
if "path" in model_dict:
|
||||
full_path = model_dict["path"]
|
||||
if model.path:
|
||||
full_path = model.path
|
||||
# Import here to avoid circular dependency
|
||||
import folder_paths
|
||||
|
||||
# Try to get category directories
|
||||
if category in folder_paths.folder_names_and_paths:
|
||||
category_paths = folder_paths.get_folder_paths(category)
|
||||
category_paths = normalize_folder_path_values(
|
||||
folder_paths.get_folder_paths(category)
|
||||
)
|
||||
# Find which base directory this path belongs to
|
||||
for base_dir in category_paths:
|
||||
if is_path_within(full_path, base_dir):
|
||||
@@ -116,37 +125,35 @@ def get_base_directory_for_model(
|
||||
return None
|
||||
|
||||
|
||||
def update_model_path(
|
||||
workflow: Dict[str, Any],
|
||||
node_id: int,
|
||||
widget_index: int,
|
||||
resolved_path: str,
|
||||
category: str = None,
|
||||
base_directory: str = None,
|
||||
resolved_model: Dict[str, Any] = None,
|
||||
subgraph_id: str = None,
|
||||
is_top_level: bool = None,
|
||||
mapping: Dict[str, Any] = None,
|
||||
) -> bool:
|
||||
def update_model_path(workflow: Dict[str, Any], resolution: Resolution) -> bool:
|
||||
"""
|
||||
Update a single model path in a workflow node, supporting both top-level and subgraph nodes.
|
||||
|
||||
Args:
|
||||
workflow: Workflow JSON dictionary
|
||||
node_id: ID of the node to update
|
||||
widget_index: Widget index to update
|
||||
resolved_path: Absolute path to resolved model
|
||||
category: Model category
|
||||
base_directory: Base directory for category
|
||||
resolved_model: Model dict from scanner
|
||||
subgraph_id: Subgraph ID (if node is in subgraph)
|
||||
is_top_level: Whether node is in top-level (not subgraph definition)
|
||||
mapping: Full mapping dictionary for standard or custom node updates
|
||||
resolution: Validated typed workflow resolution
|
||||
|
||||
Returns:
|
||||
True if update was successful, False otherwise
|
||||
"""
|
||||
try:
|
||||
resolution.validate()
|
||||
except ValueError as exc:
|
||||
log.warning(f"Invalid workflow resolution: {exc}")
|
||||
return False
|
||||
|
||||
node = None
|
||||
node_id = resolution.node_id
|
||||
widget_index = resolution.widget_index
|
||||
resolved_path = resolution.resolved_path or (
|
||||
resolution.resolved_model.path if resolution.resolved_model else ""
|
||||
)
|
||||
category = resolution.category
|
||||
base_directory = resolution.base_directory
|
||||
subgraph_id = resolution.subgraph_id
|
||||
is_top_level = resolution.is_top_level
|
||||
resolved_model_contract = resolution.resolved_model
|
||||
metadata = resolution.custom_node_metadata
|
||||
|
||||
# Determine if this is a top-level node or inside a subgraph definition
|
||||
# - If is_top_level is True, it's a top-level node (even if it's a subgraph instance)
|
||||
@@ -211,8 +218,7 @@ def update_model_path(
|
||||
if widget_index >= len(widgets_values):
|
||||
can_extend_promoted_input = bool(
|
||||
is_top_level is True
|
||||
and mapping
|
||||
and mapping.get("promoted_widget_name")
|
||||
and resolution.promoted_widget_name
|
||||
)
|
||||
if can_extend_promoted_input:
|
||||
widgets_values.extend([None] * (widget_index + 1 - len(widgets_values)))
|
||||
@@ -225,14 +231,14 @@ def update_model_path(
|
||||
return False
|
||||
|
||||
# Get category from resolved_model if not provided
|
||||
if not category and resolved_model:
|
||||
category = resolved_model.get("category")
|
||||
if not category and resolved_model_contract:
|
||||
category = resolved_model_contract.category
|
||||
|
||||
custom_update_result = update_custom_node_model_path(
|
||||
node,
|
||||
widget_index,
|
||||
resolved_model,
|
||||
mapping,
|
||||
resolved_model_contract,
|
||||
metadata,
|
||||
)
|
||||
if custom_update_result is not None:
|
||||
return custom_update_result
|
||||
@@ -243,8 +249,8 @@ def update_model_path(
|
||||
if os.path.isabs(resolved_path):
|
||||
# Use category from resolved_model for path conversion
|
||||
effective_category = category
|
||||
if resolved_model:
|
||||
effective_category = resolved_model.get("category", category)
|
||||
if resolved_model_contract and resolved_model_contract.category:
|
||||
effective_category = resolved_model_contract.category
|
||||
|
||||
relative_path = convert_to_relative_path(
|
||||
resolved_path, effective_category, base_directory
|
||||
@@ -254,7 +260,7 @@ def update_model_path(
|
||||
|
||||
# Update the widget value
|
||||
# Handle model references stored inside dictionary widget values.
|
||||
nested_key = mapping.get("nested_key") if mapping else None
|
||||
nested_key = resolution.nested_key
|
||||
if nested_key and isinstance(widgets_values[widget_index], dict):
|
||||
widgets_values[widget_index][nested_key] = relative_path
|
||||
log.debug(
|
||||
@@ -267,63 +273,36 @@ def update_model_path(
|
||||
|
||||
|
||||
def update_workflow_nodes(
|
||||
workflow: Dict[str, Any], mappings: List[Dict[str, Any]]
|
||||
workflow: Dict[str, Any], resolutions: List[Resolution]
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Apply multiple model path changes to a workflow.
|
||||
|
||||
Args:
|
||||
workflow: Workflow JSON dictionary (will be modified in place)
|
||||
mappings: List of mapping dictionaries:
|
||||
{
|
||||
'node_id': node ID,
|
||||
'widget_index': widget index,
|
||||
'resolved_path': absolute path to resolved model,
|
||||
'category': model category (optional),
|
||||
'base_directory': base directory for category (optional),
|
||||
'resolved_model': model dict from scanner (optional, for base_directory)
|
||||
}
|
||||
resolutions: Validated typed workflow resolutions.
|
||||
|
||||
Returns:
|
||||
Updated workflow dictionary (same reference, modified in place)
|
||||
"""
|
||||
updated_count = 0
|
||||
|
||||
for mapping in mappings:
|
||||
node_id = mapping.get("node_id")
|
||||
widget_index = mapping.get("widget_index")
|
||||
resolved_path = mapping.get("resolved_path")
|
||||
|
||||
if not all([node_id is not None, widget_index is not None, resolved_path]):
|
||||
log.warning(f"Invalid mapping: {mapping}")
|
||||
for resolution in resolutions:
|
||||
try:
|
||||
resolution.validate()
|
||||
except ValueError as exc:
|
||||
log.warning(f"Skipping invalid workflow resolution: {exc}")
|
||||
continue
|
||||
|
||||
# Try to get base_directory from resolved_model if provided
|
||||
base_directory = mapping.get("base_directory")
|
||||
if not base_directory and "resolved_model" in mapping:
|
||||
resolved_model = mapping["resolved_model"]
|
||||
category = mapping.get("category", "")
|
||||
base_directory = get_base_directory_for_model(resolved_model, category)
|
||||
# Try to get base_directory from the typed local model if not explicit.
|
||||
if not resolution.base_directory and resolution.resolved_model:
|
||||
base_directory = get_base_directory_for_model(
|
||||
resolution.resolved_model,
|
||||
resolution.category or "",
|
||||
)
|
||||
resolution = resolution.with_base_directory(base_directory)
|
||||
|
||||
category = mapping.get("category")
|
||||
resolved_model = mapping.get("resolved_model")
|
||||
subgraph_id = mapping.get("subgraph_id")
|
||||
is_top_level = mapping.get(
|
||||
"is_top_level"
|
||||
) # True for top-level nodes, False for nodes in subgraph definitions
|
||||
|
||||
success = update_model_path(
|
||||
workflow,
|
||||
node_id,
|
||||
widget_index,
|
||||
resolved_path,
|
||||
category,
|
||||
base_directory,
|
||||
resolved_model,
|
||||
subgraph_id,
|
||||
is_top_level,
|
||||
mapping,
|
||||
)
|
||||
success = update_model_path(workflow, resolution)
|
||||
|
||||
if success:
|
||||
updated_count += 1
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,7 +1,7 @@
|
||||
import asyncio
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from aiohttp import web
|
||||
@@ -203,6 +203,29 @@ async def test_download_route_rejects_missing_unsafe_and_unsupported_inputs():
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"payload",
|
||||
[
|
||||
{"url": 123},
|
||||
{"url": "https://example.com/model.safetensors", "filename": 123},
|
||||
{
|
||||
"url": "https://example.com/model.safetensors",
|
||||
"civitai_session_token": {"token": "invalid"},
|
||||
},
|
||||
],
|
||||
)
|
||||
async def test_download_route_rejects_malformed_text_fields(payload):
|
||||
handlers, values = _build_download_routes()
|
||||
handler = handlers[("POST", "/model_resolver/download")]
|
||||
|
||||
response = await handler(_request(payload))
|
||||
|
||||
assert response.status == 400
|
||||
assert json.loads(response.text)["error"].startswith("Download request")
|
||||
values["start_background_download"].assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_route_supports_huggingface_headers_and_optional_inputs():
|
||||
handlers, values = _build_download_routes()
|
||||
@@ -212,8 +235,7 @@ async def test_download_route_supports_huggingface_headers_and_optional_inputs()
|
||||
_request(
|
||||
{
|
||||
"url": "https://huggingface.co/org/repo/resolve/main/model.safetensors",
|
||||
"path_metadata": ["invalid"],
|
||||
"metadata": ["invalid"],
|
||||
"metadata": {},
|
||||
"base_directory": r"C:\custom",
|
||||
"hf_token": "hf-token",
|
||||
}
|
||||
@@ -230,6 +252,106 @@ async def test_download_route_supports_huggingface_headers_and_optional_inputs()
|
||||
values["get_default_root_for_category"].assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_route_rejects_invalid_download_metadata():
|
||||
handlers, values = _build_download_routes()
|
||||
handler = handlers[("POST", "/model_resolver/download")]
|
||||
|
||||
response = await handler(
|
||||
_request(
|
||||
{
|
||||
"url": "https://example.com/model.safetensors",
|
||||
"metadata": [],
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
assert response.status == 400
|
||||
assert json.loads(response.text) == {
|
||||
"error": "Download metadata must be an object"
|
||||
}
|
||||
values["start_background_download"].assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_route_rejects_invalid_path_metadata_and_identifiers():
|
||||
handlers, values = _build_download_routes()
|
||||
handler = handlers[("POST", "/model_resolver/download")]
|
||||
|
||||
response = await handler(
|
||||
_request(
|
||||
{
|
||||
"url": "https://example.com/model.safetensors",
|
||||
"path_metadata": ["invalid"],
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
assert response.status == 400
|
||||
assert json.loads(response.text) == {
|
||||
"error": "Download path metadata must be an object"
|
||||
}
|
||||
|
||||
response = await handler(
|
||||
_request(
|
||||
{
|
||||
"url": "https://example.com/model.safetensors",
|
||||
"download_metadata": {"model_id": {"value": 123}},
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
assert response.status == 400
|
||||
assert json.loads(response.text) == {
|
||||
"error": "Download metadata model_id must be an integer or string"
|
||||
}
|
||||
values["start_background_download"].assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_route_rejects_invalid_expected_sha256():
|
||||
handlers, values = _build_download_routes()
|
||||
handler = handlers[("POST", "/model_resolver/download")]
|
||||
|
||||
response = await handler(
|
||||
_request(
|
||||
{
|
||||
"url": "https://example.com/model.safetensors",
|
||||
"metadata": {"sha256": 123},
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
assert response.status == 400
|
||||
assert json.loads(response.text) == {
|
||||
"error": "DownloadSpec expected_sha256 must be a string"
|
||||
}
|
||||
values["start_background_download"].assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_route_handles_download_spec_type_errors():
|
||||
handlers, _values = _build_download_routes()
|
||||
handler = handlers[("POST", "/model_resolver/download")]
|
||||
|
||||
with patch(
|
||||
"core.services.download_service.DownloadSpec",
|
||||
side_effect=TypeError("invalid download specification"),
|
||||
):
|
||||
response = await handler(
|
||||
_request(
|
||||
{
|
||||
"url": "https://example.com/model.safetensors",
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
assert response.status == 400
|
||||
assert json.loads(response.text) == {
|
||||
"error": "invalid download specification"
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_route_derives_encoded_filename_from_url_path():
|
||||
handlers, values = _build_download_routes()
|
||||
@@ -331,6 +453,20 @@ async def test_aria2_install_handles_invalid_and_non_mapping_payloads():
|
||||
assert values["install_aria2_engine"].call_args_list[-1].args == (False,)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aria2_install_rejects_malformed_force_flag():
|
||||
handlers, values = _build_download_routes()
|
||||
handler = handlers[("POST", "/model_resolver/aria2/install")]
|
||||
|
||||
response = await handler(_request({"force": {"enabled": True}}))
|
||||
|
||||
assert response.status == 400
|
||||
assert json.loads(response.text) == {
|
||||
"error": "Aria2 install request force must be a boolean"
|
||||
}
|
||||
values["install_aria2_engine"].assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_control_routes_preserve_progress_and_status_codes():
|
||||
handlers, values = _build_download_routes()
|
||||
|
||||
+39
-11
@@ -8,6 +8,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
|
||||
import pytest
|
||||
from aiohttp import web
|
||||
|
||||
from core.contracts import ModelMatch, ModelReference, WorkflowModelInventory
|
||||
from core.path_utils import HashCalculationCancelled, normalize_absolute_path
|
||||
from core.routes.context import RouteContext
|
||||
from core.routes.hashes import register_hash_routes
|
||||
@@ -83,7 +84,7 @@ def _build_hash_routes(overrides=None):
|
||||
f"{path}.modelresolver.json"
|
||||
),
|
||||
"get_workflow_model_inventory": MagicMock(
|
||||
return_value={"model_refs": []}
|
||||
return_value=WorkflowModelInventory()
|
||||
),
|
||||
"is_path_in_configured_model_roots": MagicMock(return_value=True),
|
||||
"json_api_endpoint": lambda _name: lambda handler: handler,
|
||||
@@ -165,6 +166,9 @@ async def test_hash_routes_validate_local_metadata_and_preview_paths():
|
||||
response = await preview_handler(SimpleNamespace(query={}))
|
||||
assert response.status == 400
|
||||
|
||||
response = await preview_handler(SimpleNamespace(query={"path": 123}))
|
||||
assert response.status == 400
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
model_path = os.path.join(temp_dir, "model.safetensors")
|
||||
preview_path = os.path.join(temp_dir, "model.png")
|
||||
@@ -267,7 +271,13 @@ async def test_workflow_model_hashes_skips_invalid_refs_and_deduplicates_models(
|
||||
handlers, values = _build_hash_routes(
|
||||
{
|
||||
"get_workflow_model_inventory": MagicMock(
|
||||
return_value={"model_refs": refs}
|
||||
return_value=WorkflowModelInventory(
|
||||
model_refs=tuple(
|
||||
ModelReference.from_mapping(ref)
|
||||
for ref in refs
|
||||
if ref.get("full_path")
|
||||
)
|
||||
)
|
||||
),
|
||||
"get_local_model_hash_metadata": get_metadata,
|
||||
}
|
||||
@@ -287,7 +297,11 @@ async def test_workflow_model_hashes_skips_invalid_refs_and_deduplicates_models(
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_local_matches_by_hash_normalizes_and_enriches_results():
|
||||
matches = [{"path": r"C:\models\local.safetensors"}]
|
||||
matches = [
|
||||
ModelMatch.from_mapping(
|
||||
{"model": {"path": r"C:\models\local.safetensors"}}
|
||||
)
|
||||
]
|
||||
search_matches = MagicMock(return_value=matches)
|
||||
handlers, values = _build_hash_routes(
|
||||
{"search_local_matches_by_hash": search_matches}
|
||||
@@ -310,14 +324,12 @@ async def test_local_matches_by_hash_normalizes_and_enriches_results():
|
||||
|
||||
body = json.loads(response.text)
|
||||
assert body["sha256"] == sha256
|
||||
assert body["local_hash_matches"] == [
|
||||
{
|
||||
"path": r"C:\models\local.safetensors",
|
||||
"hash_lookup_source": "civit_ai",
|
||||
"hash_lookup_filename": "remote.safetensors",
|
||||
"hash_lookup_sha256": sha256,
|
||||
}
|
||||
]
|
||||
assert len(body["local_hash_matches"]) == 1
|
||||
match = body["local_hash_matches"][0]
|
||||
assert match["model"]["path"] == r"C:\models\local.safetensors"
|
||||
assert match["hash_lookup_source"] == "civit_ai"
|
||||
assert match["hash_lookup_filename"] == "remote.safetensors"
|
||||
assert match["hash_lookup_sha256"] == sha256
|
||||
values["search_local_matches_by_hash"].assert_called_once_with(
|
||||
sha256,
|
||||
category="checkpoints",
|
||||
@@ -326,6 +338,22 @@ async def test_local_matches_by_hash_normalizes_and_enriches_results():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_local_matches_by_hash_rejects_malformed_limit():
|
||||
handlers, values = _build_hash_routes()
|
||||
handler = handlers[("POST", "/model_resolver/local-matches-by-hash")]
|
||||
|
||||
response = await handler(
|
||||
_request({"sha256": "a" * 64, "max_matches": {"value": 20}})
|
||||
)
|
||||
|
||||
assert response.status == 400
|
||||
assert json.loads(response.text) == {
|
||||
"error": "Local hash matches request max_matches must be an integer"
|
||||
}
|
||||
values["search_local_matches_by_hash"].assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_open_containing_folder_maps_file_manager_errors():
|
||||
handlers, values = _build_hash_routes()
|
||||
|
||||
+67
-29
@@ -1,12 +1,13 @@
|
||||
import json
|
||||
import re
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from core.contracts import SearchResult
|
||||
from core.matcher import build_filename_search_queries
|
||||
from core.sources.common import probe_remote_file_size
|
||||
from core.sources.huggingface import (
|
||||
HF_AUTHOR_FALLBACKS,
|
||||
_build_author_index_from_models,
|
||||
@@ -23,14 +24,39 @@ from core.sources.huggingface import (
|
||||
get_huggingface_model_details,
|
||||
get_known_author_fallback_indexes_status,
|
||||
parse_huggingface_url,
|
||||
search_huggingface,
|
||||
search_huggingface_for_file,
|
||||
)
|
||||
from core.sources.common import probe_remote_file_size
|
||||
from core.type_utils import extract_file_size
|
||||
|
||||
|
||||
def _result_dict(result):
|
||||
assert isinstance(result, SearchResult)
|
||||
return result.to_dict()
|
||||
|
||||
|
||||
class HuggingFaceSourceTests(unittest.TestCase):
|
||||
|
||||
@patch("core.sources.huggingface.execute_provider_json_request")
|
||||
def test_general_search_returns_typed_results(self, request_json):
|
||||
request_json.return_value = [
|
||||
{
|
||||
"id": "Comfy-Org/example",
|
||||
"modelId": "Comfy-Org/example",
|
||||
"downloads": 12,
|
||||
"likes": 3,
|
||||
}
|
||||
]
|
||||
|
||||
results = search_huggingface("example", limit=1)
|
||||
|
||||
self.assertEqual(1, len(results))
|
||||
self.assertIsInstance(results[0], SearchResult)
|
||||
self.assertEqual("Comfy-Org/example", results[0].name)
|
||||
self.assertEqual("Comfy-Org/example", results[0].extra_value("repo"))
|
||||
self.assertEqual(12, results[0].extra_value("downloads"))
|
||||
self.assertEqual("https://huggingface.co/Comfy-Org/example", results[0].url)
|
||||
|
||||
def test_known_author_fallbacks_include_comfy_org_and_kijai(self):
|
||||
self.assertEqual(["Comfy-Org", "Kijai"], HF_AUTHOR_FALLBACKS)
|
||||
|
||||
@@ -142,6 +168,7 @@ class HuggingFaceSourceTests(unittest.TestCase):
|
||||
)
|
||||
|
||||
self.assertIsNotNone(result)
|
||||
result = _result_dict(result)
|
||||
self.assertEqual("hash", result["match_type"])
|
||||
self.assertEqual(sha256, result["sha256"])
|
||||
self.assertEqual("models/example.safetensors", result["path"])
|
||||
@@ -180,6 +207,7 @@ class HuggingFaceSourceTests(unittest.TestCase):
|
||||
)
|
||||
|
||||
self.assertIsNotNone(result)
|
||||
result = _result_dict(result)
|
||||
self.assertEqual("hash", result["match_type"])
|
||||
self.assertEqual(sha256, result["sha256"])
|
||||
get_index.assert_called_once()
|
||||
@@ -434,6 +462,7 @@ class HuggingFaceSourceTests(unittest.TestCase):
|
||||
)
|
||||
|
||||
self.assertIsNotNone(result)
|
||||
result = _result_dict(result)
|
||||
self.assertEqual("hash", result["match_type"])
|
||||
fetch_index.assert_called_once_with("Comfy-Org", headers={})
|
||||
|
||||
@@ -482,6 +511,7 @@ class HuggingFaceSourceTests(unittest.TestCase):
|
||||
)
|
||||
|
||||
self.assertIsNotNone(result)
|
||||
result = _result_dict(result)
|
||||
self.assertEqual("hash", result["match_type"])
|
||||
fetch_index.assert_not_called()
|
||||
|
||||
@@ -521,16 +551,16 @@ class HuggingFaceSourceTests(unittest.TestCase):
|
||||
url = "https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.safetensors"
|
||||
result = parse_huggingface_url(url)
|
||||
self.assertIsNotNone(result)
|
||||
self.assertEqual(result["repo"], "runwayml/stable-diffusion-v1-5")
|
||||
self.assertEqual(result["branch"], "main")
|
||||
self.assertEqual(result["filename"], "v1-5-pruned-emaonly.safetensors")
|
||||
self.assertEqual(result.repo, "runwayml/stable-diffusion-v1-5")
|
||||
self.assertEqual(result.branch, "main")
|
||||
self.assertEqual(result.filename, "v1-5-pruned-emaonly.safetensors")
|
||||
|
||||
def test_parse_huggingface_url_valid_hf_protocol(self):
|
||||
url = "hf://stabilityai/stable-diffusion-xl-base-1.0/sd_xl_base_1.0.safetensors"
|
||||
result = parse_huggingface_url(url)
|
||||
self.assertIsNotNone(result)
|
||||
self.assertEqual(result["repo"], "stabilityai/stable-diffusion-xl-base-1.0")
|
||||
self.assertEqual(result["filename"], "sd_xl_base_1.0.safetensors")
|
||||
self.assertEqual(result.repo, "stabilityai/stable-diffusion-xl-base-1.0")
|
||||
self.assertEqual(result.filename, "sd_xl_base_1.0.safetensors")
|
||||
|
||||
def test_parse_huggingface_url_invalid(self):
|
||||
self.assertIsNone(parse_huggingface_url("https://example.com/not-hf"))
|
||||
@@ -633,6 +663,7 @@ class HuggingFaceSourceTests(unittest.TestCase):
|
||||
result = build_huggingface_custom_result(url)
|
||||
|
||||
self.assertIsNotNone(result)
|
||||
result = _result_dict(result)
|
||||
self.assertEqual(sha256, result["sha256"])
|
||||
self.assertEqual({"SHA256": sha256}, result["hashes"])
|
||||
self.assertEqual(123, result["size"])
|
||||
@@ -641,14 +672,17 @@ class HuggingFaceSourceTests(unittest.TestCase):
|
||||
|
||||
def test_custom_result_leaves_hash_empty_when_huggingface_does_not_provide_it(self):
|
||||
url = "https://huggingface.co/user/repo/blob/main/model.safetensors"
|
||||
with patch("core.sources.huggingface._get_repo_tree", return_value=None):
|
||||
with patch(
|
||||
with (
|
||||
patch("core.sources.huggingface._get_repo_tree", return_value=None),
|
||||
patch(
|
||||
"core.sources.common.fetch_remote_file_size_cached",
|
||||
return_value=456,
|
||||
):
|
||||
result = build_huggingface_custom_result(url)
|
||||
),
|
||||
):
|
||||
result = build_huggingface_custom_result(url)
|
||||
|
||||
self.assertIsNotNone(result)
|
||||
result = _result_dict(result)
|
||||
self.assertEqual("", result["sha256"])
|
||||
self.assertEqual({}, result["hashes"])
|
||||
self.assertEqual(456, result["size"])
|
||||
@@ -676,8 +710,9 @@ class HuggingFaceSourceTests(unittest.TestCase):
|
||||
},
|
||||
{"path": "diffusion_models/README.md", "size": 10},
|
||||
]
|
||||
with patch("core.sources.huggingface._get_repo_tree", return_value=tree):
|
||||
with patch(
|
||||
with (
|
||||
patch("core.sources.huggingface._get_repo_tree", return_value=tree),
|
||||
patch(
|
||||
"core.sources.huggingface.execute_provider_json_request",
|
||||
return_value={
|
||||
"modelId": "Comfy-Org/example",
|
||||
@@ -686,11 +721,12 @@ class HuggingFaceSourceTests(unittest.TestCase):
|
||||
"likes": 4,
|
||||
"tags": ["comfyui"],
|
||||
},
|
||||
):
|
||||
details = get_huggingface_model_details(
|
||||
"Comfy-Org/example",
|
||||
"diffusion_models/model_fp8_scaled.safetensors",
|
||||
)
|
||||
),
|
||||
):
|
||||
details = get_huggingface_model_details(
|
||||
"Comfy-Org/example",
|
||||
"diffusion_models/model_fp8_scaled.safetensors",
|
||||
)
|
||||
|
||||
self.assertIsNotNone(details)
|
||||
self.assertEqual("huggingface", details["source"])
|
||||
@@ -710,19 +746,21 @@ class HuggingFaceSourceTests(unittest.TestCase):
|
||||
|
||||
def test_model_details_passes_huggingface_token_to_tree_and_metadata(self):
|
||||
tree = [{"path": "model.safetensors", "size": 100}]
|
||||
with patch(
|
||||
"core.sources.huggingface._get_repo_tree",
|
||||
return_value=tree,
|
||||
) as mock_tree:
|
||||
with patch(
|
||||
with (
|
||||
patch(
|
||||
"core.sources.huggingface._get_repo_tree",
|
||||
return_value=tree,
|
||||
) as mock_tree,
|
||||
patch(
|
||||
"core.sources.huggingface.execute_provider_json_request",
|
||||
return_value={},
|
||||
) as mock_request:
|
||||
details = get_huggingface_model_details(
|
||||
"private/repo",
|
||||
"model.safetensors",
|
||||
token="hf_secret",
|
||||
)
|
||||
) as mock_request,
|
||||
):
|
||||
details = get_huggingface_model_details(
|
||||
"private/repo",
|
||||
"model.safetensors",
|
||||
token="hf_secret",
|
||||
)
|
||||
|
||||
self.assertIsNotNone(details)
|
||||
expected_headers = {"Authorization": "Bearer hf_secret"}
|
||||
|
||||
@@ -7,6 +7,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
|
||||
import pytest
|
||||
from aiohttp import web
|
||||
|
||||
from core.contracts import WorkflowModelInventory
|
||||
from core.path_utils import get_filename_from_path
|
||||
from core.routes.context import RouteContext
|
||||
from core.routes.loaded_models import register_loaded_model_routes
|
||||
@@ -35,7 +36,17 @@ def _json_api_endpoint(_name, **_kwargs):
|
||||
return lambda handler: handler
|
||||
|
||||
|
||||
def _as_inventory(inventory=None):
|
||||
if not isinstance(inventory, WorkflowModelInventory):
|
||||
inventory = WorkflowModelInventory(
|
||||
available_models=(inventory or {}).get("available_models", []),
|
||||
model_refs=(inventory or {}).get("model_refs", []),
|
||||
)
|
||||
return inventory
|
||||
|
||||
|
||||
def _build_routes(inventory=None):
|
||||
inventory = _as_inventory(inventory)
|
||||
routes = _Routes()
|
||||
extension = SimpleNamespace(
|
||||
loaded_progress=MagicMock(),
|
||||
@@ -54,7 +65,6 @@ def _build_routes(inventory=None):
|
||||
),
|
||||
"get_workflow_model_inventory": MagicMock(
|
||||
return_value=inventory
|
||||
or {"available_models": [], "model_refs": []}
|
||||
),
|
||||
"json_api_endpoint": _json_api_endpoint,
|
||||
"routes": routes,
|
||||
@@ -247,7 +257,7 @@ async def test_loaded_models_progress_counts_top_level_and_subgraph_nodes():
|
||||
"total": 5,
|
||||
}
|
||||
)
|
||||
return {"available_models": [], "model_refs": []}
|
||||
return WorkflowModelInventory()
|
||||
|
||||
values["get_workflow_model_inventory"].side_effect = inventory_with_progress
|
||||
workflow = {
|
||||
@@ -415,7 +425,7 @@ async def test_loaded_models_route_uses_adapter_name_and_original_path_fallback(
|
||||
|
||||
def inventory_with_progress(_workflow, progress_callback):
|
||||
progress_callback(progress_payload)
|
||||
return inventory
|
||||
return _as_inventory(inventory)
|
||||
|
||||
values["get_workflow_model_inventory"].side_effect = inventory_with_progress
|
||||
|
||||
|
||||
@@ -1,11 +1,21 @@
|
||||
from unittest.mock import Mock
|
||||
|
||||
from core.contracts import ModelMatch
|
||||
from core.local_hash_matches import collect_local_hash_matches_for_result
|
||||
|
||||
|
||||
def test_collect_local_hash_matches_forwards_lookup_options_and_enriches_results():
|
||||
sha256 = "a" * 64
|
||||
lookup = Mock(return_value=[{"path": r"C:\models\local.safetensors"}])
|
||||
lookup = Mock(
|
||||
return_value=[
|
||||
ModelMatch.from_mapping(
|
||||
{
|
||||
"model": {"path": r"C:\models\local.safetensors"},
|
||||
"filename": "local.safetensors",
|
||||
}
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
matches = collect_local_hash_matches_for_result(
|
||||
sha256,
|
||||
@@ -23,14 +33,11 @@ def test_collect_local_hash_matches_forwards_lookup_options_and_enriches_results
|
||||
max_matches=7,
|
||||
force_rescan=True,
|
||||
)
|
||||
assert matches == [
|
||||
{
|
||||
"path": r"C:\models\local.safetensors",
|
||||
"hash_lookup_source": "civitai",
|
||||
"hash_lookup_filename": "remote.safetensors",
|
||||
"hash_lookup_sha256": sha256,
|
||||
}
|
||||
]
|
||||
assert len(matches) == 1
|
||||
assert matches[0].model.path == r"C:\models\local.safetensors"
|
||||
assert matches[0].extra_value("hash_lookup_source") == "civitai"
|
||||
assert matches[0].extra_value("hash_lookup_filename") == "remote.safetensors"
|
||||
assert matches[0].extra_value("hash_lookup_sha256") == sha256
|
||||
|
||||
|
||||
def test_collect_local_hash_matches_skips_empty_hash_without_lookup():
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
import io
|
||||
import logging
|
||||
|
||||
from core.log_system.logger import DirectConsoleHandler
|
||||
|
||||
|
||||
def test_console_handler_replaces_characters_unsupported_by_windows_encoding():
|
||||
buffer = io.BytesIO()
|
||||
stream = io.TextIOWrapper(buffer, encoding="cp1250", errors="strict")
|
||||
handler = DirectConsoleHandler(stream)
|
||||
handler.setFormatter(logging.Formatter("%(message)s"))
|
||||
|
||||
handler.emit(
|
||||
logging.LogRecord(
|
||||
"test",
|
||||
logging.INFO,
|
||||
__file__,
|
||||
1,
|
||||
"Download complete ✓",
|
||||
(),
|
||||
None,
|
||||
)
|
||||
)
|
||||
stream.flush()
|
||||
|
||||
assert "Download complete ?" in buffer.getvalue().decode("cp1250")
|
||||
@@ -1,17 +1,20 @@
|
||||
import unittest
|
||||
import sqlite3
|
||||
import json
|
||||
import sqlite3
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from core.contracts import SearchResult
|
||||
from core.sources.lora_manager_archive import (
|
||||
_normalize_model_type,
|
||||
_extract_search_tokens,
|
||||
_query_candidate_rows,
|
||||
_load_full_rows_for_versions,
|
||||
_build_result_from_row,
|
||||
_extract_search_tokens,
|
||||
_find_hash_match_row,
|
||||
_load_full_rows_for_versions,
|
||||
_normalize_model_type,
|
||||
_query_candidate_rows,
|
||||
search_lora_manager_archive_for_file,
|
||||
)
|
||||
|
||||
|
||||
class LoraManagerArchiveTests(unittest.TestCase):
|
||||
|
||||
def test_normalize_model_type(self):
|
||||
@@ -146,12 +149,12 @@ class LoraManagerArchiveTests(unittest.TestCase):
|
||||
# Test building result from row
|
||||
result = _build_result_from_row(conn, row, "ghost_checkpoint")
|
||||
self.assertIsNotNone(result)
|
||||
self.assertEqual(result["filename"], "ghost_v1.safetensors")
|
||||
self.assertEqual(result["name"], "ghost_checkpoint")
|
||||
self.assertEqual(result["trained_words"], ["ghost_word"])
|
||||
self.assertEqual(result["size"], 3 * 1024)
|
||||
self.assertEqual(result["sha256"], archive_hash)
|
||||
self.assertEqual(result["hashes"], {"SHA256": archive_hash})
|
||||
self.assertEqual(result.filename, "ghost_v1.safetensors")
|
||||
self.assertEqual(result.name, "ghost_checkpoint")
|
||||
self.assertEqual(list(result.trained_words), ["ghost_word"])
|
||||
self.assertEqual(result.size, 3 * 1024)
|
||||
self.assertEqual(result.sha256, archive_hash)
|
||||
self.assertEqual(dict(result.hashes), {"SHA256": archive_hash})
|
||||
|
||||
hash_match = _find_hash_match_row(conn, archive_hash)
|
||||
self.assertIsNotNone(hash_match)
|
||||
@@ -168,6 +171,27 @@ class LoraManagerArchiveTests(unittest.TestCase):
|
||||
)
|
||||
|
||||
self.assertIsNotNone(hash_result)
|
||||
self.assertEqual(hash_result["match_type"], "hash")
|
||||
self.assertEqual(hash_result["confidence"], 100.0)
|
||||
self.assertEqual(hash_result["sha256"], archive_hash)
|
||||
self.assertEqual(hash_result.match_type, "hash")
|
||||
self.assertEqual(hash_result.confidence, 100.0)
|
||||
self.assertEqual(hash_result.sha256, archive_hash)
|
||||
|
||||
def test_filename_search_keeps_typed_candidates(self):
|
||||
filename = "typed-lora-archive-model.safetensors"
|
||||
candidate = SearchResult(
|
||||
source="lora_manager_archive",
|
||||
name="typed-lora-archive-model",
|
||||
version_name="v1",
|
||||
filename=filename,
|
||||
download_url="https://example.com/typed-lora-archive-model.safetensors",
|
||||
)
|
||||
|
||||
with patch(
|
||||
"core.sources.lora_manager_archive.search_lora_manager_archive",
|
||||
return_value=[candidate],
|
||||
):
|
||||
result = search_lora_manager_archive_for_file(filename)
|
||||
|
||||
self.assertIsInstance(result, SearchResult)
|
||||
self.assertEqual(filename, result.filename)
|
||||
self.assertEqual(100.0, result.confidence)
|
||||
self.assertEqual("exact", result.match_type)
|
||||
|
||||
+53
-21
@@ -1,6 +1,7 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from core.contracts import SearchResult
|
||||
from core.matcher import (
|
||||
build_filename_search_queries,
|
||||
calculate_archived_model_confidence,
|
||||
@@ -12,6 +13,7 @@ from core.matcher import (
|
||||
)
|
||||
from core.progress import report_progress
|
||||
from core.sources.model_list import search_model_list, search_model_list_multiple
|
||||
from core.sources.popular import search_popular_models
|
||||
from core.type_utils import (
|
||||
MODEL_CONTAINER_SUFFIXES,
|
||||
MODEL_VARIANT_SUFFIXES,
|
||||
@@ -47,8 +49,8 @@ class MatcherTests(unittest.TestCase):
|
||||
]
|
||||
matches = find_matches("sd_xl_base_1.0.safetensors", candidates)
|
||||
self.assertEqual(2, len(matches))
|
||||
self.assertEqual("sd_xl_base_1.0.safetensors", matches[0]["filename"])
|
||||
self.assertEqual(1.0, matches[0]["similarity"])
|
||||
self.assertEqual("sd_xl_base_1.0.safetensors", matches[0].filename)
|
||||
self.assertEqual(1.0, matches[0].similarity)
|
||||
|
||||
# With higher threshold, only the best match should be returned
|
||||
matches_filtered = find_matches("sd_xl_base_1.0.safetensors", candidates, threshold=0.7)
|
||||
@@ -165,9 +167,21 @@ class MatcherTests(unittest.TestCase):
|
||||
"qwen3vl-4b-heretic_int8.safetensors",
|
||||
"qwen_image_fp8_e4m3fn.safetensors",
|
||||
],
|
||||
[match["filename"] for match in matches],
|
||||
[match.filename for match in matches],
|
||||
)
|
||||
|
||||
def test_find_matches_does_not_mutate_candidate_mappings(self):
|
||||
candidate = {"filename": "model.safetensors", "category": "checkpoints"}
|
||||
|
||||
matches = find_matches("model.safetensors", [candidate])
|
||||
|
||||
self.assertEqual(1, len(matches))
|
||||
self.assertEqual(
|
||||
{"filename": "model.safetensors", "category": "checkpoints"},
|
||||
candidate,
|
||||
)
|
||||
self.assertEqual("checkpoints", matches[0].model.category)
|
||||
|
||||
def test_search_model_list_unification(self):
|
||||
try:
|
||||
search_model_list("nonexistent_model_name_xyz.safetensors", exact_only=True)
|
||||
@@ -200,25 +214,43 @@ class MatcherTests(unittest.TestCase):
|
||||
fuzzy = search_model_list("target-model-extra.safetensors")
|
||||
multiple = search_model_list_multiple("exact", limit=1)
|
||||
|
||||
self.assertEqual(
|
||||
exact,
|
||||
{
|
||||
"source": "model_list",
|
||||
"filename": "exact.safetensors",
|
||||
"url": "https://example.test/exact",
|
||||
"name": "Exact model",
|
||||
"type": "checkpoints",
|
||||
"directory": "checkpoints",
|
||||
"size": "12 MB",
|
||||
"match_type": "exact",
|
||||
self.assertIsInstance(exact, SearchResult)
|
||||
self.assertEqual(exact.source, "model_list")
|
||||
self.assertEqual(exact.filename, "exact.safetensors")
|
||||
self.assertEqual(exact.url, "https://example.test/exact")
|
||||
self.assertEqual(exact.name, "Exact model")
|
||||
self.assertEqual(exact.model_type, "checkpoints")
|
||||
self.assertEqual(exact.extra_value("directory"), "checkpoints")
|
||||
self.assertEqual(exact.size, 12 * 1024**2)
|
||||
self.assertEqual(exact.match_type, "exact")
|
||||
self.assertEqual(fuzzy.filename, "target-model.safetensors")
|
||||
self.assertEqual(fuzzy.match_type, "fuzzy")
|
||||
self.assertGreater(fuzzy.confidence, 0)
|
||||
self.assertEqual(multiple[0].filename, "exact.safetensors")
|
||||
self.assertGreater(multiple[0].confidence, 0)
|
||||
self.assertEqual(multiple[0].match_type, "similar")
|
||||
|
||||
def test_popular_search_returns_typed_search_results(self):
|
||||
with patch(
|
||||
"core.sources.popular._load_popular_models",
|
||||
return_value={
|
||||
"popular-model.safetensors": {
|
||||
"url": "https://example.test/popular",
|
||||
"type": "checkpoints",
|
||||
"directory": "checkpoints",
|
||||
"canonical_name": "popular-model",
|
||||
}
|
||||
},
|
||||
)
|
||||
self.assertEqual(fuzzy["filename"], "target-model.safetensors")
|
||||
self.assertEqual(fuzzy["match_type"], "fuzzy")
|
||||
self.assertIn("confidence", fuzzy)
|
||||
self.assertEqual(multiple[0]["filename"], "exact.safetensors")
|
||||
self.assertIn("confidence", multiple[0])
|
||||
self.assertNotIn("match_type", multiple[0])
|
||||
):
|
||||
results = search_popular_models("popular-model", limit=1)
|
||||
|
||||
self.assertEqual(1, len(results))
|
||||
self.assertIsInstance(results[0], SearchResult)
|
||||
self.assertEqual("popular-model.safetensors", results[0].filename)
|
||||
self.assertEqual("https://example.test/popular", results[0].url)
|
||||
self.assertEqual("checkpoints", results[0].model_type)
|
||||
self.assertEqual("checkpoints", results[0].extra_value("directory"))
|
||||
self.assertEqual("popular-model", results[0].extra_value("canonical_name"))
|
||||
|
||||
def test_build_filename_search_queries(self):
|
||||
# Verify that common precisions are extracted correctly
|
||||
|
||||
@@ -497,6 +497,10 @@ class MetadataBuilderTests(unittest.TestCase):
|
||||
normalize_metadata_build_mode("unsupported"),
|
||||
)
|
||||
|
||||
def test_invalid_metadata_mode_type_is_rejected(self):
|
||||
with self.assertRaises(TypeError):
|
||||
normalize_metadata_build_mode(123)
|
||||
|
||||
def test_exact_sidecar_remains_canonical_after_sibling_is_removed(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
bin_path = self._write_model(tmpdir, "adapter.bin", b"bin")
|
||||
|
||||
@@ -3,8 +3,13 @@ import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from core.contracts import ResolvedModel
|
||||
from core.metadata_audit import _model_key
|
||||
from core.metadata_model_utils import dedupe_models, is_model_file_path
|
||||
from core.metadata_model_utils import (
|
||||
dedupe_models,
|
||||
is_model_file_path,
|
||||
normalize_models,
|
||||
)
|
||||
from core.path_utils import get_model_path_identity
|
||||
from core.resolver import _build_local_hash_match_cache
|
||||
from core.scanner import scan_directory
|
||||
@@ -51,7 +56,16 @@ class MetadataModelHelperTests(unittest.TestCase):
|
||||
get_model_path_identity(duplicate["path"]),
|
||||
)
|
||||
self.assertEqual("", get_model_path_identity(empty["path"]))
|
||||
self.assertEqual([first], dedupe_models(models))
|
||||
deduped = dedupe_models(models)
|
||||
self.assertEqual(1, len(deduped))
|
||||
self.assertIsInstance(deduped[0], ResolvedModel)
|
||||
self.assertEqual(model_path, deduped[0].path)
|
||||
|
||||
def test_normalize_models_rejects_invalid_collection_payloads(self):
|
||||
with self.assertRaises(TypeError):
|
||||
normalize_models(0)
|
||||
with self.assertRaises(TypeError):
|
||||
normalize_models({"path": "model.safetensors"})
|
||||
|
||||
def test_scanner_and_metadata_identity_share_path_fallback(self):
|
||||
model_path = os.path.join("models", "model.safetensors")
|
||||
@@ -82,7 +96,7 @@ class MetadataModelHelperTests(unittest.TestCase):
|
||||
):
|
||||
models = scan_directory("models", {".safetensors"}, "checkpoints")
|
||||
|
||||
self.assertEqual(["model.safetensors"], [model["filename"] for model in models])
|
||||
self.assertEqual(["model.safetensors"], [model.filename for model in models])
|
||||
|
||||
with patch("core.resolver.os.path.isdir", return_value=False), patch(
|
||||
"core.resolver.find_metadata_sidecar_path",
|
||||
@@ -99,7 +113,7 @@ class MetadataModelHelperTests(unittest.TestCase):
|
||||
)
|
||||
|
||||
self.assertEqual(1, len(index["a" * 64]))
|
||||
self.assertEqual(model_path, index["a" * 64][0]["model"]["path"])
|
||||
self.assertEqual(model_path, index["a" * 64][0].model.path)
|
||||
|
||||
def test_model_path_identity_rejects_empty_and_whitespace_paths(self):
|
||||
self.assertEqual("", get_model_path_identity(None))
|
||||
|
||||
@@ -33,10 +33,13 @@ from core.network_utils import (
|
||||
validate_public_http_url,
|
||||
)
|
||||
from core.path_utils import (
|
||||
dedupe_local_base_directories,
|
||||
get_safe_metadata_sidecar_path,
|
||||
get_safe_model_resolver_sidecar_path,
|
||||
is_path_in_configured_model_roots,
|
||||
is_path_within,
|
||||
normalize_folder_path_values,
|
||||
normalize_string_values,
|
||||
)
|
||||
|
||||
|
||||
@@ -52,6 +55,33 @@ class DummyFolderPaths:
|
||||
|
||||
|
||||
class PathSecurityTests(unittest.TestCase):
|
||||
def test_external_string_collections_do_not_iterate_scalar_text(self):
|
||||
self.assertEqual(
|
||||
[r"C:\\models\\checkpoints"],
|
||||
normalize_folder_path_values(r"C:\\models\\checkpoints"),
|
||||
)
|
||||
self.assertEqual(
|
||||
[r"C:\\models\\checkpoints"],
|
||||
normalize_folder_path_values(
|
||||
(r"C:\\models\\checkpoints", {".safetensors"})
|
||||
),
|
||||
)
|
||||
self.assertEqual(
|
||||
["model.safetensors"],
|
||||
normalize_string_values("model.safetensors"),
|
||||
)
|
||||
self.assertEqual(
|
||||
["model.safetensors"],
|
||||
normalize_string_values([None, "model.safetensors", 123]),
|
||||
)
|
||||
|
||||
def test_dedupe_local_base_directories_rejects_scalar_payloads(self):
|
||||
self.assertEqual([], dedupe_local_base_directories(None))
|
||||
with self.assertRaises(TypeError):
|
||||
dedupe_local_base_directories(0)
|
||||
with self.assertRaises(TypeError):
|
||||
dedupe_local_base_directories("models")
|
||||
|
||||
def test_is_path_within_rejects_sibling_directory_with_shared_prefix(self):
|
||||
with tempfile.TemporaryDirectory() as temp_root:
|
||||
model_root = os.path.join(temp_root, "models")
|
||||
|
||||
+108
-88
@@ -1,9 +1,9 @@
|
||||
import unittest
|
||||
|
||||
from core.contracts import ResolvedModel
|
||||
from core.path_templates import infer_download_path_templates
|
||||
from core.settings import calculate_template_subfolder
|
||||
|
||||
|
||||
BASE_MODELS = {
|
||||
"base_models": [
|
||||
{
|
||||
@@ -42,26 +42,36 @@ BASE_MODELS = {
|
||||
}
|
||||
|
||||
|
||||
def _typed_models(items):
|
||||
return [ResolvedModel.from_mapping(item) for item in items]
|
||||
|
||||
|
||||
class PathTemplateInferenceTests(unittest.TestCase):
|
||||
def test_rejects_non_model_collections(self):
|
||||
with self.assertRaises(TypeError):
|
||||
infer_download_path_templates(0, BASE_MODELS)
|
||||
|
||||
def test_detects_base_model_and_first_tag_layout(self):
|
||||
result = infer_download_path_templates(
|
||||
[
|
||||
{
|
||||
"category": "loras",
|
||||
"relative_path": r"ponyxl\style\model-a.safetensors",
|
||||
"path": r"C:\models\loras\ponyxl\style\model-a.safetensors",
|
||||
},
|
||||
{
|
||||
"category": "lora",
|
||||
"relative_path": r"ponyxl\character\model-b.safetensors",
|
||||
"path": r"C:\models\loras\ponyxl\character\model-b.safetensors",
|
||||
},
|
||||
{
|
||||
"category": "loras",
|
||||
"relative_path": r"sdxl\style\model-c.safetensors",
|
||||
"path": r"C:\models\loras\sdxl\style\model-c.safetensors",
|
||||
},
|
||||
],
|
||||
_typed_models(
|
||||
[
|
||||
{
|
||||
"category": "loras",
|
||||
"relative_path": r"ponyxl\style\model-a.safetensors",
|
||||
"path": r"C:\models\loras\ponyxl\style\model-a.safetensors",
|
||||
},
|
||||
{
|
||||
"category": "lora",
|
||||
"relative_path": r"ponyxl\character\model-b.safetensors",
|
||||
"path": r"C:\models\loras\ponyxl\character\model-b.safetensors",
|
||||
},
|
||||
{
|
||||
"category": "loras",
|
||||
"relative_path": r"sdxl\style\model-c.safetensors",
|
||||
"path": r"C:\models\loras\sdxl\style\model-c.safetensors",
|
||||
},
|
||||
]
|
||||
),
|
||||
BASE_MODELS,
|
||||
)
|
||||
|
||||
@@ -72,18 +82,20 @@ class PathTemplateInferenceTests(unittest.TestCase):
|
||||
|
||||
def test_detects_flat_layout(self):
|
||||
result = infer_download_path_templates(
|
||||
[
|
||||
{
|
||||
"category": "vae",
|
||||
"relative_path": "vae-a.safetensors",
|
||||
"path": r"C:\models\vae\vae-a.safetensors",
|
||||
},
|
||||
{
|
||||
"category": "vae",
|
||||
"relative_path": "vae-b.safetensors",
|
||||
"path": r"C:\models\vae\vae-b.safetensors",
|
||||
},
|
||||
],
|
||||
_typed_models(
|
||||
[
|
||||
{
|
||||
"category": "vae",
|
||||
"relative_path": "vae-a.safetensors",
|
||||
"path": r"C:\models\vae\vae-a.safetensors",
|
||||
},
|
||||
{
|
||||
"category": "vae",
|
||||
"relative_path": "vae-b.safetensors",
|
||||
"path": r"C:\models\vae\vae-b.safetensors",
|
||||
},
|
||||
]
|
||||
),
|
||||
BASE_MODELS,
|
||||
)
|
||||
|
||||
@@ -92,28 +104,30 @@ class PathTemplateInferenceTests(unittest.TestCase):
|
||||
|
||||
def test_detects_nested_checkpoint_base_model_mappings(self):
|
||||
result = infer_download_path_templates(
|
||||
[
|
||||
{
|
||||
"category": "checkpoints",
|
||||
"relative_path": r"SDXL\Pony\pony-realism.safetensors",
|
||||
"path": r"C:\models\checkpoints\SDXL\Pony\pony-realism.safetensors",
|
||||
},
|
||||
{
|
||||
"category": "checkpoints",
|
||||
"relative_path": r"SDXL\Illustrious\wai.safetensors",
|
||||
"path": r"C:\models\checkpoints\SDXL\Illustrious\wai.safetensors",
|
||||
},
|
||||
{
|
||||
"category": "checkpoints",
|
||||
"relative_path": r"SDXL\Illustrious\realistic\real.safetensors",
|
||||
"path": r"C:\models\checkpoints\SDXL\Illustrious\realistic\real.safetensors",
|
||||
},
|
||||
{
|
||||
"category": "checkpoints",
|
||||
"relative_path": r"SDXL\SDXL\juggernaut.safetensors",
|
||||
"path": r"C:\models\checkpoints\SDXL\SDXL\juggernaut.safetensors",
|
||||
},
|
||||
],
|
||||
_typed_models(
|
||||
[
|
||||
{
|
||||
"category": "checkpoints",
|
||||
"relative_path": r"SDXL\Pony\pony-realism.safetensors",
|
||||
"path": r"C:\models\checkpoints\SDXL\Pony\pony-realism.safetensors",
|
||||
},
|
||||
{
|
||||
"category": "checkpoints",
|
||||
"relative_path": r"SDXL\Illustrious\wai.safetensors",
|
||||
"path": r"C:\models\checkpoints\SDXL\Illustrious\wai.safetensors",
|
||||
},
|
||||
{
|
||||
"category": "checkpoints",
|
||||
"relative_path": r"SDXL\Illustrious\realistic\real.safetensors",
|
||||
"path": r"C:\models\checkpoints\SDXL\Illustrious\realistic\real.safetensors",
|
||||
},
|
||||
{
|
||||
"category": "checkpoints",
|
||||
"relative_path": r"SDXL\SDXL\juggernaut.safetensors",
|
||||
"path": r"C:\models\checkpoints\SDXL\SDXL\juggernaut.safetensors",
|
||||
},
|
||||
]
|
||||
),
|
||||
BASE_MODELS,
|
||||
)
|
||||
|
||||
@@ -142,18 +156,20 @@ class PathTemplateInferenceTests(unittest.TestCase):
|
||||
|
||||
def test_detect_does_not_map_krea2_to_flux_krea_folder(self):
|
||||
result = infer_download_path_templates(
|
||||
[
|
||||
{
|
||||
"category": "loras",
|
||||
"relative_path": r"FLUX\KREA\concept\snofs_krea_v1.safetensors",
|
||||
"path": r"C:\models\loras\FLUX\KREA\concept\snofs_krea_v1.safetensors",
|
||||
},
|
||||
{
|
||||
"category": "loras",
|
||||
"relative_path": r"FLUX\KREA\style\another_flux_krea.safetensors",
|
||||
"path": r"C:\models\loras\FLUX\KREA\style\another_flux_krea.safetensors",
|
||||
},
|
||||
],
|
||||
_typed_models(
|
||||
[
|
||||
{
|
||||
"category": "loras",
|
||||
"relative_path": r"FLUX\KREA\concept\snofs_krea_v1.safetensors",
|
||||
"path": r"C:\models\loras\FLUX\KREA\concept\snofs_krea_v1.safetensors",
|
||||
},
|
||||
{
|
||||
"category": "loras",
|
||||
"relative_path": r"FLUX\KREA\style\another_flux_krea.safetensors",
|
||||
"path": r"C:\models\loras\FLUX\KREA\style\another_flux_krea.safetensors",
|
||||
},
|
||||
]
|
||||
),
|
||||
BASE_MODELS,
|
||||
)
|
||||
|
||||
@@ -165,18 +181,20 @@ class PathTemplateInferenceTests(unittest.TestCase):
|
||||
|
||||
def test_detect_does_not_map_anima_to_wan_animate_folder(self):
|
||||
result = infer_download_path_templates(
|
||||
[
|
||||
{
|
||||
"category": "checkpoints",
|
||||
"relative_path": r"WAN\WAN2.2ANIMATE\model-a.safetensors",
|
||||
"path": r"C:\models\checkpoints\WAN\WAN2.2ANIMATE\model-a.safetensors",
|
||||
},
|
||||
{
|
||||
"category": "checkpoints",
|
||||
"relative_path": r"WAN\WAN2.2ANIMATE\model-b.safetensors",
|
||||
"path": r"C:\models\checkpoints\WAN\WAN2.2ANIMATE\model-b.safetensors",
|
||||
},
|
||||
],
|
||||
_typed_models(
|
||||
[
|
||||
{
|
||||
"category": "checkpoints",
|
||||
"relative_path": r"WAN\WAN2.2ANIMATE\model-a.safetensors",
|
||||
"path": r"C:\models\checkpoints\WAN\WAN2.2ANIMATE\model-a.safetensors",
|
||||
},
|
||||
{
|
||||
"category": "checkpoints",
|
||||
"relative_path": r"WAN\WAN2.2ANIMATE\model-b.safetensors",
|
||||
"path": r"C:\models\checkpoints\WAN\WAN2.2ANIMATE\model-b.safetensors",
|
||||
},
|
||||
]
|
||||
),
|
||||
BASE_MODELS,
|
||||
)
|
||||
|
||||
@@ -184,18 +202,20 @@ class PathTemplateInferenceTests(unittest.TestCase):
|
||||
|
||||
def test_detect_accepts_version_and_plural_alias_suffixes(self):
|
||||
result = infer_download_path_templates(
|
||||
[
|
||||
{
|
||||
"category": "diffusion_models",
|
||||
"relative_path": r"IDEOGRAM4\model-a.safetensors",
|
||||
"path": r"C:\models\diffusion_models\IDEOGRAM4\model-a.safetensors",
|
||||
},
|
||||
{
|
||||
"category": "upscale_models",
|
||||
"relative_path": r"ESRGAN\Upscalers\photo\model-a.pth",
|
||||
"path": r"C:\models\upscale_models\ESRGAN\Upscalers\photo\model-a.pth",
|
||||
},
|
||||
],
|
||||
_typed_models(
|
||||
[
|
||||
{
|
||||
"category": "diffusion_models",
|
||||
"relative_path": r"IDEOGRAM4\model-a.safetensors",
|
||||
"path": r"C:\models\diffusion_models\IDEOGRAM4\model-a.safetensors",
|
||||
},
|
||||
{
|
||||
"category": "upscale_models",
|
||||
"relative_path": r"ESRGAN\Upscalers\photo\model-a.pth",
|
||||
"path": r"C:\models\upscale_models\ESRGAN\Upscalers\photo\model-a.pth",
|
||||
},
|
||||
]
|
||||
),
|
||||
BASE_MODELS,
|
||||
)
|
||||
|
||||
|
||||
@@ -7,6 +7,8 @@ import unittest
|
||||
from typing import ClassVar
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from core.contracts import MissingModel, WorkflowAnalysisResult, WorkflowModelInventory
|
||||
|
||||
# Make sure parent package directory is in sys.path
|
||||
parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
if parent_dir not in sys.path:
|
||||
@@ -58,6 +60,9 @@ mock_routes.post = post_decorator
|
||||
|
||||
# Import the module so that routes register
|
||||
node_mod = importlib.import_module("comfyui-model-resolver")
|
||||
TypedSearchResult = importlib.import_module(
|
||||
"comfyui-model-resolver.core.contracts"
|
||||
).SearchResult
|
||||
|
||||
# Force setup_routes to run to populate routes_registered mock registry
|
||||
node_mod.extension.routes_setup = False
|
||||
@@ -506,12 +511,12 @@ class TestRefactoringTargets(unittest.IsolatedAsyncioTestCase):
|
||||
if function_name == "build_huggingface_custom_result":
|
||||
self.assertEqual(args, (url, "model.safetensors", None))
|
||||
self.assertEqual(kwargs, {})
|
||||
return {
|
||||
"source": "huggingface",
|
||||
"filename": "model.safetensors",
|
||||
"url": url,
|
||||
"download_url": url,
|
||||
}
|
||||
return TypedSearchResult(
|
||||
source="huggingface",
|
||||
filename="model.safetensors",
|
||||
url=url,
|
||||
download_url=url,
|
||||
)
|
||||
self.fail(f"Unexpected threaded function: {function_name}")
|
||||
|
||||
with (
|
||||
@@ -551,14 +556,27 @@ class TestRefactoringTargets(unittest.IsolatedAsyncioTestCase):
|
||||
self.assertEqual(getattr(func, "__name__", ""), "analyze_and_find_matches")
|
||||
self.assertEqual(args[:3], (workflow, 0.0, 10))
|
||||
self.assertEqual(kwargs["force_rescan"], False)
|
||||
return {
|
||||
"missing_models": [
|
||||
{
|
||||
"name": "already-matched.safetensors",
|
||||
"matches": [{"confidence": 100}],
|
||||
}
|
||||
]
|
||||
}
|
||||
return WorkflowAnalysisResult(
|
||||
missing_models=(
|
||||
MissingModel.from_mapping(
|
||||
{
|
||||
"name": "already-matched.safetensors",
|
||||
"matches": [
|
||||
{
|
||||
"model": {
|
||||
"path": "models/already-matched.safetensors",
|
||||
"filename": "already-matched.safetensors",
|
||||
"category": "checkpoints",
|
||||
},
|
||||
"confidence": 100,
|
||||
}
|
||||
],
|
||||
}
|
||||
),
|
||||
),
|
||||
total_missing=1,
|
||||
total_models_analyzed=1,
|
||||
)
|
||||
|
||||
with (
|
||||
patch("asyncio.to_thread", side_effect=fake_to_thread),
|
||||
@@ -602,7 +620,14 @@ class TestRefactoringTargets(unittest.IsolatedAsyncioTestCase):
|
||||
request = AsyncMock()
|
||||
request.json.return_value = {
|
||||
"workflow": workflow,
|
||||
"resolutions": [{"node_id": 1, "widget_index": 0}],
|
||||
"resolutions": [
|
||||
{
|
||||
"node_id": 1,
|
||||
"widget_index": 0,
|
||||
"resolved_path": "resolved.safetensors",
|
||||
"category": "checkpoints",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
with patch("aiohttp.web.json_response") as mock_json_response:
|
||||
@@ -650,6 +675,22 @@ class TestRefactoringTargets(unittest.IsolatedAsyncioTestCase):
|
||||
{"loaded_models": [], "total": 0}
|
||||
)
|
||||
|
||||
async def test_loaded_models_route_rejects_non_text_progress_id(self):
|
||||
post_handler = routes_registered[("POST", "/model_resolver/loaded")]
|
||||
request = AsyncMock()
|
||||
request.json.return_value = {
|
||||
"workflow": {"nodes": []},
|
||||
"loaded_id": 123,
|
||||
}
|
||||
|
||||
with patch("aiohttp.web.json_response") as mock_json_response:
|
||||
await post_handler(request)
|
||||
|
||||
mock_json_response.assert_called_once_with(
|
||||
{"error": "Loaded models request loaded_id must be a string"},
|
||||
status=400,
|
||||
)
|
||||
|
||||
async def test_loaded_models_route_rejects_non_object_workflow(self):
|
||||
post_handler = routes_registered[("POST", "/model_resolver/loaded")]
|
||||
request = AsyncMock()
|
||||
@@ -796,6 +837,38 @@ class TestRefactoringTargets(unittest.IsolatedAsyncioTestCase):
|
||||
status=400,
|
||||
)
|
||||
|
||||
async def test_model_details_route_rejects_non_numeric_provider_ids(self):
|
||||
post_handler = routes_registered[("POST", "/model_resolver/model-details")]
|
||||
request = AsyncMock()
|
||||
request.json.return_value = {
|
||||
"source": "civitai",
|
||||
"model_id": "not-a-number",
|
||||
}
|
||||
|
||||
with patch("aiohttp.web.json_response") as mock_json_response:
|
||||
await post_handler(request)
|
||||
|
||||
mock_json_response.assert_called_once_with(
|
||||
{"error": "Model details request model_id must be an integer"},
|
||||
status=400,
|
||||
)
|
||||
|
||||
async def test_model_details_route_keeps_huggingface_ids_textual(self):
|
||||
post_handler = routes_registered[("POST", "/model_resolver/model-details")]
|
||||
request = AsyncMock()
|
||||
request.json.return_value = {
|
||||
"source": "huggingface",
|
||||
"model_id": 123,
|
||||
}
|
||||
|
||||
with patch("aiohttp.web.json_response") as mock_json_response:
|
||||
await post_handler(request)
|
||||
|
||||
mock_json_response.assert_called_once_with(
|
||||
{"error": "Model details model_id must be a string for Hugging Face"},
|
||||
status=400,
|
||||
)
|
||||
|
||||
async def test_civitai_search_route_rejects_missing_filename(self):
|
||||
post_handler = routes_registered[("POST", "/model_resolver/civitai-search")]
|
||||
request = AsyncMock()
|
||||
@@ -912,7 +985,7 @@ class TestRefactoringTargets(unittest.IsolatedAsyncioTestCase):
|
||||
if function_name == "get_workflow_model_inventory":
|
||||
self.assertEqual((workflow,), args)
|
||||
self.assertEqual({}, kwargs)
|
||||
return {"available_models": [], "model_refs": []}
|
||||
return WorkflowModelInventory()
|
||||
self.fail(f"Unexpected threaded function: {function_name}")
|
||||
|
||||
with (
|
||||
@@ -1238,7 +1311,6 @@ class TestRefactoringTargets(unittest.IsolatedAsyncioTestCase):
|
||||
"request_public_url",
|
||||
"resolve_civarchive_by_hash",
|
||||
"search_huggingface_for_file",
|
||||
"to_bool",
|
||||
"web",
|
||||
"write_model_resolver_metadata",
|
||||
},
|
||||
@@ -1356,7 +1428,9 @@ class TestRefactoringTargets(unittest.IsolatedAsyncioTestCase):
|
||||
|
||||
response = MagicMock(status_code=200)
|
||||
request_public_url = MagicMock(return_value=(response, mirror_url, {}))
|
||||
resolve_civarchive_by_hash = MagicMock(return_value=result)
|
||||
resolve_civarchive_by_hash = MagicMock(
|
||||
return_value=TypedSearchResult.from_mapping(result, source="civarchive")
|
||||
)
|
||||
service = self._build_model_service_for_unit_test(
|
||||
service_module="civitai_search_service",
|
||||
service_class="CivitAISearchService",
|
||||
@@ -1389,8 +1463,12 @@ class TestRefactoringTargets(unittest.IsolatedAsyncioTestCase):
|
||||
}
|
||||
|
||||
try:
|
||||
with patch(
|
||||
"core.sources.civitai.get_model_info_by_hash",
|
||||
civitai_module = importlib.import_module(
|
||||
"comfyui-model-resolver.core.sources.civitai"
|
||||
)
|
||||
with patch.object(
|
||||
civitai_module,
|
||||
"get_model_info_by_hash",
|
||||
return_value=None,
|
||||
):
|
||||
api_response = await service.civitai_search(request)
|
||||
@@ -1404,6 +1482,100 @@ class TestRefactoringTargets(unittest.IsolatedAsyncioTestCase):
|
||||
payload = json.loads(api_response.text)
|
||||
self.assertEqual(payload["download_url"], mirror_url)
|
||||
|
||||
async def test_civitai_service_accepts_typed_huggingface_result(self):
|
||||
expected_hash = "c" * 64
|
||||
hf_result = TypedSearchResult(
|
||||
source="huggingface",
|
||||
name="Typed HuggingFace model",
|
||||
filename="model.safetensors",
|
||||
url=(
|
||||
"https://huggingface.co/example/repo/blob/main/model.safetensors"
|
||||
),
|
||||
download_url=(
|
||||
"https://huggingface.co/example/repo/resolve/main/model.safetensors"
|
||||
),
|
||||
sha256=expected_hash,
|
||||
extra={
|
||||
"repo_id": "example/repo",
|
||||
"path": "model.safetensors",
|
||||
"from_metadata": True,
|
||||
},
|
||||
)
|
||||
|
||||
def to_bool(value, default=False):
|
||||
return default if value is None else bool(value)
|
||||
|
||||
search_huggingface_for_file = MagicMock(return_value=hf_result)
|
||||
service = self._build_model_service_for_unit_test(
|
||||
service_module="civitai_search_service",
|
||||
service_class="CivitAISearchService",
|
||||
download_available=True,
|
||||
extract_sha256_from_metadata=lambda value: value.get("sha256", ""),
|
||||
get_filename_from_path=lambda value: str(value or "").replace(
|
||||
"\\", "/"
|
||||
).rsplit("/", 1)[-1],
|
||||
normalize_category_to_model_type=lambda value: "Checkpoint",
|
||||
normalize_sha256=lambda value: str(value or ""),
|
||||
resolve_civarchive_by_hash=MagicMock(return_value=None),
|
||||
search_huggingface_for_file=search_huggingface_for_file,
|
||||
to_bool=to_bool,
|
||||
)
|
||||
request = AsyncMock()
|
||||
with tempfile.NamedTemporaryFile(
|
||||
suffix=".safetensors", delete=False
|
||||
) as handle:
|
||||
model_path = handle.name
|
||||
request.json.return_value = {
|
||||
"filename": "model.safetensors",
|
||||
"category": "checkpoints",
|
||||
"resolved_path": model_path,
|
||||
"sha256": expected_hash,
|
||||
}
|
||||
|
||||
civitai_module = importlib.import_module(
|
||||
"comfyui-model-resolver.core.sources.civitai"
|
||||
)
|
||||
try:
|
||||
with patch.object(
|
||||
civitai_module,
|
||||
"get_model_info_by_hash",
|
||||
return_value=None,
|
||||
):
|
||||
response = await service.civitai_search(request)
|
||||
finally:
|
||||
os.unlink(model_path)
|
||||
|
||||
self.assertEqual(response.status, 200)
|
||||
payload = json.loads(response.text)
|
||||
self.assertEqual("huggingface", payload["source"])
|
||||
self.assertEqual(expected_hash, payload["sha256"])
|
||||
self.assertEqual(
|
||||
hf_result.download_url,
|
||||
payload["download_url"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"https://huggingface.co/example/repo/blob/main/model.safetensors",
|
||||
payload["url"],
|
||||
)
|
||||
search_huggingface_for_file.assert_called_once()
|
||||
|
||||
async def test_civitai_service_rejects_malformed_text_fields(self):
|
||||
service = self._build_model_service_for_unit_test(
|
||||
service_module="civitai_search_service",
|
||||
service_class="CivitAISearchService",
|
||||
download_available=True,
|
||||
)
|
||||
request = AsyncMock()
|
||||
request.json.return_value = {"filename": 123}
|
||||
|
||||
response = await service.civitai_search(request)
|
||||
|
||||
self.assertEqual(response.status, 400)
|
||||
self.assertEqual(
|
||||
json.loads(response.text)["error"],
|
||||
"CivitAI search request filename must be a string",
|
||||
)
|
||||
|
||||
async def test_civitai_service_preserves_remote_sidecar_payload_fields(self):
|
||||
expected_hash = "b" * 64
|
||||
remote_result = {
|
||||
@@ -1667,6 +1839,22 @@ class TestRefactoringTargets(unittest.IsolatedAsyncioTestCase):
|
||||
self.assertEqual(response.status, 400)
|
||||
self.assertEqual(json.loads(response.text)["error"], "Unsafe URL")
|
||||
|
||||
async def test_custom_url_service_rejects_malformed_text_fields(self):
|
||||
service = self._build_model_service_for_unit_test(
|
||||
service_module="custom_url_service",
|
||||
service_class="CustomUrlService",
|
||||
)
|
||||
request = AsyncMock()
|
||||
request.json.return_value = {"url": 123}
|
||||
|
||||
response = await service.custom_url(request)
|
||||
|
||||
self.assertEqual(response.status, 400)
|
||||
self.assertEqual(
|
||||
json.loads(response.text)["error"],
|
||||
"Custom URL request url must be a string",
|
||||
)
|
||||
|
||||
async def test_model_details_service_returns_unavailable_when_providers_are_missing(self):
|
||||
service = self._build_model_service_for_unit_test(
|
||||
service_module="model_details_service",
|
||||
@@ -1703,3 +1891,23 @@ class TestRefactoringTargets(unittest.IsolatedAsyncioTestCase):
|
||||
json.loads(response.text)["error"],
|
||||
"Model details not found",
|
||||
)
|
||||
|
||||
async def test_model_details_service_rejects_malformed_identifiers(self):
|
||||
service = self._build_model_service_for_unit_test(
|
||||
service_module="model_details_service",
|
||||
service_class="ModelDetailsService",
|
||||
download_available=True,
|
||||
)
|
||||
request = AsyncMock()
|
||||
request.json.return_value = {
|
||||
"source": "civitai",
|
||||
"model_id": True,
|
||||
}
|
||||
|
||||
response = await service.model_details(request)
|
||||
|
||||
self.assertEqual(response.status, 400)
|
||||
self.assertEqual(
|
||||
json.loads(response.text)["error"],
|
||||
"Model details request model_id must be an integer or string",
|
||||
)
|
||||
|
||||
@@ -9,6 +9,7 @@ parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
if parent_dir not in sys.path:
|
||||
sys.path.insert(0, parent_dir)
|
||||
|
||||
from core.contracts import SearchResult
|
||||
from core.matcher import MODEL_TITLE_MATCH_THRESHOLD
|
||||
from core.type_utils import (
|
||||
build_model_result,
|
||||
@@ -30,13 +31,17 @@ class TestRefactoringUnification(unittest.TestCase):
|
||||
from core import resolver
|
||||
|
||||
calls = MagicMock()
|
||||
with patch.object(resolver, "invalidate_model_files_cache") as invalidate_inventory:
|
||||
with patch.object(
|
||||
with (
|
||||
patch.object(
|
||||
resolver, "invalidate_model_files_cache"
|
||||
) as invalidate_inventory,
|
||||
patch.object(
|
||||
resolver, "invalidate_local_hash_match_cache"
|
||||
) as invalidate_hash_matches:
|
||||
calls.attach_mock(invalidate_inventory, "inventory")
|
||||
calls.attach_mock(invalidate_hash_matches, "hash_matches")
|
||||
resolver.invalidate_model_caches()
|
||||
) as invalidate_hash_matches,
|
||||
):
|
||||
calls.attach_mock(invalidate_inventory, "inventory")
|
||||
calls.attach_mock(invalidate_hash_matches, "hash_matches")
|
||||
resolver.invalidate_model_caches()
|
||||
|
||||
calls.assert_has_calls([call.inventory(), call.hash_matches()])
|
||||
invalidate_inventory.assert_called_once_with()
|
||||
@@ -74,12 +79,14 @@ class TestRefactoringUnification(unittest.TestCase):
|
||||
filename="test.safetensors",
|
||||
extra_field="hello_world",
|
||||
)
|
||||
self.assertEqual(res["source"], "civitai")
|
||||
self.assertEqual(res["model_id"], 100)
|
||||
self.assertEqual(res["version_id"], 200)
|
||||
self.assertEqual(res["name"], "Test Model")
|
||||
self.assertEqual(res["filename"], "test.safetensors")
|
||||
self.assertEqual(res["extra_field"], "hello_world")
|
||||
self.assertIsInstance(res, SearchResult)
|
||||
serialized = res.to_dict()
|
||||
self.assertEqual(serialized["source"], "civitai")
|
||||
self.assertEqual(serialized["model_id"], 100)
|
||||
self.assertEqual(serialized["version_id"], 200)
|
||||
self.assertEqual(serialized["name"], "Test Model")
|
||||
self.assertEqual(serialized["filename"], "test.safetensors")
|
||||
self.assertEqual(serialized["extra_field"], "hello_world")
|
||||
|
||||
def test_build_model_result_custom_url_contract(self):
|
||||
res = build_model_result(
|
||||
@@ -97,10 +104,11 @@ class TestRefactoringUnification(unittest.TestCase):
|
||||
result_mode="custom_url",
|
||||
)
|
||||
|
||||
self.assertEqual(res["details_source"], "civitai")
|
||||
self.assertEqual(res["version_url"], res["url"])
|
||||
self.assertTrue(res["custom_url"])
|
||||
self.assertNotIn("confidence", res)
|
||||
serialized = res.to_dict()
|
||||
self.assertEqual(serialized["details_source"], "civitai")
|
||||
self.assertEqual(serialized["version_url"], serialized["url"])
|
||||
self.assertTrue(serialized["custom_url"])
|
||||
self.assertNotIn("confidence", serialized)
|
||||
|
||||
def test_build_model_result_compact_custom_url_contract(self):
|
||||
res = build_model_result(
|
||||
@@ -126,7 +134,7 @@ class TestRefactoringUnification(unittest.TestCase):
|
||||
"download_url": "https://civitai.com/api/download/models/200",
|
||||
"match_type": "custom_url",
|
||||
"custom_url": True,
|
||||
}, res)
|
||||
}, res.to_dict())
|
||||
|
||||
@patch("requests.get")
|
||||
def test_request_page_text_success(self, mock_get):
|
||||
@@ -409,11 +417,12 @@ class TestRefactoringUnification(unittest.TestCase):
|
||||
hashes={"sha256": valid_sha.lower()},
|
||||
)
|
||||
self.assertEqual(expected, res)
|
||||
self.assertEqual(res["source"], "civitai")
|
||||
self.assertEqual(res["model_id"], 123)
|
||||
self.assertEqual(res["version_id"], 456)
|
||||
self.assertEqual(res["sha256"], valid_sha.lower())
|
||||
self.assertEqual(res["hashes"]["sha256"], valid_sha.lower())
|
||||
serialized = res.to_dict()
|
||||
self.assertEqual(serialized["source"], "civitai")
|
||||
self.assertEqual(serialized["model_id"], 123)
|
||||
self.assertEqual(serialized["version_id"], 456)
|
||||
self.assertEqual(serialized["sha256"], valid_sha.lower())
|
||||
self.assertEqual(serialized["hashes"]["sha256"], valid_sha.lower())
|
||||
|
||||
def test_job_progress_tracker_update_from_payload(self):
|
||||
from core.progress import JobProgressTracker
|
||||
|
||||
+108
-1
@@ -3,7 +3,19 @@ from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
from core.request_utils import extract_request_sha256, read_optional_object_payload
|
||||
from core.request_utils import (
|
||||
coerce_integer_identifier,
|
||||
extract_request_sha256,
|
||||
read_bool_field,
|
||||
read_first_bool_field,
|
||||
read_first_identifier_field,
|
||||
read_first_text_field,
|
||||
read_identifier_field,
|
||||
read_int_field,
|
||||
read_optional_object_payload,
|
||||
read_optional_text_field,
|
||||
read_text_field,
|
||||
)
|
||||
|
||||
|
||||
def test_extract_request_sha256_preserves_alias_precedence_and_normalization():
|
||||
@@ -26,6 +38,101 @@ def test_extract_request_sha256_preserves_alias_precedence_and_normalization():
|
||||
keys=("sha256", "hash", "SHA256"),
|
||||
) == sha256
|
||||
assert extract_request_sha256({}, keys=("sha256", "hash")) == ""
|
||||
with pytest.raises(TypeError, match="Request sha256 must be a string"):
|
||||
extract_request_sha256(
|
||||
{"sha256": 123},
|
||||
keys=("sha256", "hash"),
|
||||
)
|
||||
|
||||
|
||||
def test_read_text_field_preserves_optional_null_and_rejects_scalars():
|
||||
assert read_text_field({}, "name", default="fallback") == "fallback"
|
||||
assert read_text_field({"name": None}, "name", default="fallback") == (
|
||||
"fallback"
|
||||
)
|
||||
assert read_text_field({"name": " value "}, "name") == "value"
|
||||
with pytest.raises(TypeError, match="Request name must be a string"):
|
||||
read_text_field({"name": 123}, "name")
|
||||
|
||||
|
||||
def test_read_optional_text_field_preserves_missing_value_and_validates():
|
||||
assert read_optional_text_field({}, "name") is None
|
||||
assert read_optional_text_field({"name": " value "}, "name") == "value"
|
||||
assert read_optional_text_field({"name": ""}, "name") is None
|
||||
with pytest.raises(TypeError, match="Request name must be a string"):
|
||||
read_optional_text_field({"name": 123}, "name")
|
||||
|
||||
|
||||
def test_read_first_text_field_preserves_alias_order_and_rejects_scalars():
|
||||
assert read_first_text_field(
|
||||
{"url": "", "custom_url": " https://example.test/model "},
|
||||
("url", "custom_url"),
|
||||
) == "https://example.test/model"
|
||||
with pytest.raises(TypeError, match="Request url must be a string"):
|
||||
read_first_text_field(
|
||||
{"url": 123, "custom_url": "https://example.test/model"},
|
||||
("url", "custom_url"),
|
||||
)
|
||||
|
||||
|
||||
def test_read_identifier_field_accepts_only_string_or_integer_identifiers():
|
||||
assert read_identifier_field({"id": " 12 "}, "id") == "12"
|
||||
assert read_identifier_field({"id": 12}, "id") == 12
|
||||
assert read_identifier_field({}, "id") is None
|
||||
with pytest.raises(TypeError, match="Request id must be an integer or string"):
|
||||
read_identifier_field({"id": True}, "id")
|
||||
|
||||
|
||||
def test_coerce_integer_identifier_rejects_non_numeric_values():
|
||||
assert coerce_integer_identifier(" 12 ", "id") == 12
|
||||
assert coerce_integer_identifier(34, "id") == 34
|
||||
assert coerce_integer_identifier("", "id") is None
|
||||
with pytest.raises(ValueError, match="Request id must be an integer"):
|
||||
coerce_integer_identifier("repo/name", "id")
|
||||
|
||||
|
||||
def test_read_first_identifier_field_preserves_alias_order_and_rejects_scalars():
|
||||
assert read_first_identifier_field(
|
||||
{"model_id": "", "modelId": " 12 "},
|
||||
("model_id", "modelId"),
|
||||
) == "12"
|
||||
with pytest.raises(
|
||||
TypeError,
|
||||
match="Request model_id must be an integer or string",
|
||||
):
|
||||
read_first_identifier_field(
|
||||
{"model_id": {"value": 12}, "modelId": 12},
|
||||
("model_id", "modelId"),
|
||||
)
|
||||
|
||||
|
||||
def test_read_bool_field_accepts_supported_forms_and_rejects_scalars():
|
||||
assert read_bool_field({"value": "true"}, "value") is True
|
||||
assert read_bool_field({"value": 0}, "value") is False
|
||||
assert read_bool_field({}, "value", default=True) is True
|
||||
with pytest.raises(TypeError, match="Request value must be a boolean"):
|
||||
read_bool_field({"value": {"enabled": True}}, "value")
|
||||
|
||||
|
||||
def test_read_first_bool_field_preserves_alias_order_and_rejects_scalars():
|
||||
assert read_first_bool_field(
|
||||
{"force": "", "force_rescan": "true"},
|
||||
("force", "force_rescan"),
|
||||
) is True
|
||||
with pytest.raises(TypeError, match="Request force must be a boolean"):
|
||||
read_first_bool_field(
|
||||
{"force": {"enabled": True}, "force_rescan": True},
|
||||
("force", "force_rescan"),
|
||||
)
|
||||
|
||||
|
||||
def test_read_int_field_rejects_boolean_and_invalid_numeric_values():
|
||||
assert read_int_field({"value": " 12 "}, "value") == 12
|
||||
assert read_int_field({}, "value", default=5) == 5
|
||||
with pytest.raises(TypeError, match="Request value must be an integer"):
|
||||
read_int_field({"value": True}, "value")
|
||||
with pytest.raises(ValueError, match="Request value must be an integer"):
|
||||
read_int_field({"value": "not-a-number"}, "value")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -18,6 +18,7 @@ from core.download.api import (
|
||||
from core.download.api import context as downloader_context
|
||||
from core.download.metadata import read_completed_metadata_sha256
|
||||
from core.download.state import download_lock, download_progress
|
||||
from core.contracts import Resolution, ResolvedModel
|
||||
from core.path_utils import get_metadata_sidecar_path, get_path_key
|
||||
from core.workflow_updater import (
|
||||
convert_to_relative_path,
|
||||
@@ -179,12 +180,14 @@ class ModelResolverRobustnessTests(unittest.TestCase):
|
||||
# Act
|
||||
success = update_model_path(
|
||||
workflow=workflow,
|
||||
node_id=15,
|
||||
widget_index=2,
|
||||
resolved_path="new_lora_name.safetensors",
|
||||
category="loras",
|
||||
resolved_model=resolved_model,
|
||||
mapping=mapping
|
||||
resolution=Resolution(
|
||||
node_id=15,
|
||||
widget_index=2,
|
||||
resolved_path="new_lora_name.safetensors",
|
||||
category="loras",
|
||||
resolved_model=ResolvedModel.from_mapping(resolved_model),
|
||||
custom_node_metadata=mapping,
|
||||
),
|
||||
)
|
||||
|
||||
# Assert
|
||||
@@ -229,12 +232,14 @@ class ModelResolverRobustnessTests(unittest.TestCase):
|
||||
# Act
|
||||
success = update_model_path(
|
||||
workflow=workflow,
|
||||
node_id=99,
|
||||
widget_index=0,
|
||||
resolved_path="new_model.safetensors",
|
||||
category="checkpoints",
|
||||
subgraph_id="subgraph_a",
|
||||
is_top_level=False
|
||||
resolution=Resolution(
|
||||
node_id=99,
|
||||
widget_index=0,
|
||||
resolved_path="new_model.safetensors",
|
||||
category="checkpoints",
|
||||
subgraph_id="subgraph_a",
|
||||
is_top_level=False,
|
||||
),
|
||||
)
|
||||
|
||||
# Assert
|
||||
@@ -582,12 +587,14 @@ class ModelResolverRobustnessTests(unittest.TestCase):
|
||||
# Act
|
||||
success = update_model_path(
|
||||
workflow=workflow,
|
||||
node_id=1,
|
||||
widget_index=0,
|
||||
resolved_path="model.safetensors",
|
||||
category="checkpoints",
|
||||
subgraph_id="nonexistent_subgraph",
|
||||
is_top_level=False
|
||||
resolution=Resolution(
|
||||
node_id=1,
|
||||
widget_index=0,
|
||||
resolved_path="model.safetensors",
|
||||
category="checkpoints",
|
||||
subgraph_id="nonexistent_subgraph",
|
||||
is_top_level=False,
|
||||
),
|
||||
)
|
||||
|
||||
# Assert
|
||||
|
||||
+36
-9
@@ -1,11 +1,34 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from core.scanner import find_local_file_path, scan_directory
|
||||
from core.contracts import ResolvedModel
|
||||
from core.scanner import find_local_file_path, scan_all_directories, scan_directory
|
||||
|
||||
|
||||
class ScannerTests(unittest.TestCase):
|
||||
|
||||
@patch("core.scanner.get_model_directories")
|
||||
@patch("core.scanner.scan_directory")
|
||||
def test_scan_all_directories_normalizes_scalar_registry_paths(
|
||||
self, mock_scan_directory, mock_get_model_directories
|
||||
):
|
||||
mock_get_model_directories.return_value = {
|
||||
"checkpoints": (r"C:\\models\\checkpoints", {".safetensors"})
|
||||
}
|
||||
model = ResolvedModel(
|
||||
path=r"C:\\models\\checkpoints\\model.safetensors",
|
||||
filename="model.safetensors",
|
||||
category="checkpoints",
|
||||
)
|
||||
mock_scan_directory.return_value = [model]
|
||||
|
||||
assert scan_all_directories() == [model]
|
||||
mock_scan_directory.assert_called_once_with(
|
||||
r"C:\\models\\checkpoints",
|
||||
{".safetensors"},
|
||||
"checkpoints",
|
||||
)
|
||||
|
||||
def test_model_path_identity_empty_and_valid(self):
|
||||
from core.path_utils import get_model_path_identity
|
||||
|
||||
@@ -37,17 +60,18 @@ class ScannerTests(unittest.TestCase):
|
||||
|
||||
# Verify that we found the two models and ignored readme.txt
|
||||
self.assertEqual(len(models), 2)
|
||||
self.assertTrue(all(isinstance(model, ResolvedModel) for model in models))
|
||||
|
||||
# First model check
|
||||
m1 = next(m for m in models if m["filename"] == "root_model.safetensors")
|
||||
self.assertEqual(m1["relative_path"].replace("\\", "/"), "root_model.safetensors")
|
||||
self.assertEqual(m1["category"], "checkpoints")
|
||||
m1 = next(m for m in models if m.filename == "root_model.safetensors")
|
||||
self.assertEqual(m1.relative_path.replace("\\", "/"), "root_model.safetensors")
|
||||
self.assertEqual(m1.category, "checkpoints")
|
||||
import os
|
||||
self.assertEqual(m1["base_directory"], os.path.abspath("/models/checkpoints"))
|
||||
self.assertEqual(m1.base_directory, os.path.abspath("/models/checkpoints"))
|
||||
|
||||
# Second model check
|
||||
m2 = next(m for m in models if m["filename"] == "nested_model.ckpt")
|
||||
self.assertEqual(m2["relative_path"].replace("\\", "/"), "subdir/nested_model.ckpt")
|
||||
m2 = next(m for m in models if m.filename == "nested_model.ckpt")
|
||||
self.assertEqual(m2.relative_path.replace("\\", "/"), "subdir/nested_model.ckpt")
|
||||
|
||||
@patch("os.walk")
|
||||
@patch("os.path.exists")
|
||||
@@ -69,7 +93,7 @@ class ScannerTests(unittest.TestCase):
|
||||
"/models/diffusion_models", {".gguf"}, "model_gguf"
|
||||
)
|
||||
|
||||
self.assertEqual([model["filename"] for model in models], ["quantized.gguf"])
|
||||
self.assertEqual([model.filename for model in models], ["quantized.gguf"])
|
||||
|
||||
@patch("core.scanner.folder_paths")
|
||||
@patch("core.scanner.get_model_files")
|
||||
@@ -89,7 +113,10 @@ class ScannerTests(unittest.TestCase):
|
||||
# 3. Test found via local scanner fallback
|
||||
mock_folder_paths.get_full_path.return_value = None
|
||||
mock_get_files.return_value = [
|
||||
{"filename": "fallback.safetensors", "path": "/models/other/fallback.safetensors"}
|
||||
ResolvedModel(
|
||||
filename="fallback.safetensors",
|
||||
path="/models/other/fallback.safetensors",
|
||||
)
|
||||
]
|
||||
self.assertEqual(
|
||||
find_local_file_path("fallback.safetensors", "checkpoints"),
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
import asyncio
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from aiohttp import web
|
||||
|
||||
from core.contracts import ResolvedModel
|
||||
from core.path_utils import is_path_within
|
||||
from core.routes.context import RouteContext
|
||||
from core.routes.directories import register_directory_routes
|
||||
@@ -113,6 +114,15 @@ def _request(query=None):
|
||||
return SimpleNamespace(query=query or {}, match_info={})
|
||||
|
||||
|
||||
def _body_request(payload):
|
||||
return SimpleNamespace(
|
||||
can_read_body=True,
|
||||
json=AsyncMock(return_value=payload),
|
||||
query={},
|
||||
match_info={},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_models_route_forwards_force_rescan_and_invalidates_hash_cache():
|
||||
handlers, values = _build_metadata_routes()
|
||||
@@ -125,6 +135,20 @@ async def test_models_route_forwards_force_rescan_and_invalidates_hash_cache():
|
||||
values["get_model_files"].assert_called_once_with(force_rescan=True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_models_route_rejects_malformed_force_flag():
|
||||
handlers, values = _build_metadata_routes()
|
||||
handler = handlers[("GET", "/model_resolver/models")]
|
||||
|
||||
response = await handler(_request({"force": {"enabled": True}}))
|
||||
|
||||
assert response.status == 400
|
||||
assert json.loads(response.text) == {
|
||||
"error": "Models request force must be a boolean"
|
||||
}
|
||||
values["get_model_files"].assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_path_suggestions_route_scans_and_infers_in_background():
|
||||
handlers, values = _build_directory_routes()
|
||||
@@ -140,6 +164,36 @@ async def test_path_suggestions_route_scans_and_infers_in_background():
|
||||
assert json.loads(response.text) == [{"category": "checkpoints"}]
|
||||
values["get_model_files"].assert_called_once_with(True)
|
||||
values["infer_download_path_templates"].assert_called_once_with(
|
||||
[{"filename": "model.safetensors"}],
|
||||
[ResolvedModel(filename="model.safetensors")],
|
||||
base_models,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_metadata_size_audit_rejects_malformed_numeric_options():
|
||||
handlers, values = _build_metadata_routes()
|
||||
handler = handlers[("POST", "/model_resolver/metadata-size-audit")]
|
||||
|
||||
response = await handler(_body_request({"worker_count": {"value": 4}}))
|
||||
|
||||
assert response.status == 400
|
||||
assert json.loads(response.text) == {
|
||||
"error": "Metadata size audit request worker_count must be an integer"
|
||||
}
|
||||
values["audit_metadata_sizes"].assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_metadata_build_start_rejects_malformed_options():
|
||||
handlers, values = _build_metadata_routes()
|
||||
handler = handlers[("POST", "/model_resolver/metadata-build/start")]
|
||||
|
||||
response = await handler(
|
||||
_body_request({"force_rescan": [], "worker_count": 2})
|
||||
)
|
||||
|
||||
assert response.status == 400
|
||||
assert json.loads(response.text) == {
|
||||
"error": "Metadata build request force_rescan must be a boolean"
|
||||
}
|
||||
values["run_in_background_thread"].assert_not_called()
|
||||
|
||||
@@ -63,6 +63,7 @@ civitai_sources = importlib.import_module("comfyui-model-resolver.core.sources.c
|
||||
huggingface_sources = importlib.import_module("comfyui-model-resolver.core.sources.huggingface")
|
||||
civarchive_sources = importlib.import_module("comfyui-model-resolver.core.sources.civarchive")
|
||||
resolver_core = importlib.import_module("comfyui-model-resolver.core.resolver")
|
||||
typed_contracts = importlib.import_module("comfyui-model-resolver.core.contracts")
|
||||
|
||||
|
||||
@contextmanager
|
||||
@@ -88,30 +89,36 @@ class LocalMatchDeduplicationTests(unittest.TestCase):
|
||||
def test_search_local_matches_keeps_highest_confidence_for_same_path(self):
|
||||
shared_path = os.path.join(os.getcwd(), "models", "shared.safetensors")
|
||||
matches = [
|
||||
{
|
||||
"model": {
|
||||
"path": shared_path,
|
||||
"relative_path": "models/shared.safetensors",
|
||||
},
|
||||
"filename": "shared.safetensors",
|
||||
"confidence": 0.42,
|
||||
},
|
||||
{
|
||||
"model": {
|
||||
"path": shared_path,
|
||||
"relative_path": "models/shared.safetensors",
|
||||
},
|
||||
"filename": "shared.safetensors",
|
||||
"confidence": 0.91,
|
||||
},
|
||||
{
|
||||
"model": {
|
||||
"path": os.path.join(os.getcwd(), "models", "other.safetensors"),
|
||||
"relative_path": "models/other.safetensors",
|
||||
},
|
||||
"filename": "other.safetensors",
|
||||
"confidence": 0.73,
|
||||
},
|
||||
resolver_core.ModelMatch.from_mapping(
|
||||
{
|
||||
"model": {
|
||||
"path": shared_path,
|
||||
"relative_path": "models/shared.safetensors",
|
||||
},
|
||||
"filename": "shared.safetensors",
|
||||
"confidence": 0.42,
|
||||
}
|
||||
),
|
||||
resolver_core.ModelMatch.from_mapping(
|
||||
{
|
||||
"model": {
|
||||
"path": shared_path,
|
||||
"relative_path": "models/shared.safetensors",
|
||||
},
|
||||
"filename": "shared.safetensors",
|
||||
"confidence": 0.91,
|
||||
}
|
||||
),
|
||||
resolver_core.ModelMatch.from_mapping(
|
||||
{
|
||||
"model": {
|
||||
"path": os.path.join(os.getcwd(), "models", "other.safetensors"),
|
||||
"relative_path": "models/other.safetensors",
|
||||
},
|
||||
"filename": "other.safetensors",
|
||||
"confidence": 0.73,
|
||||
}
|
||||
),
|
||||
]
|
||||
|
||||
with (
|
||||
@@ -126,7 +133,7 @@ class LocalMatchDeduplicationTests(unittest.TestCase):
|
||||
result = resolver_core.search_local_matches("shared.safetensors")
|
||||
|
||||
self.assertEqual(
|
||||
[match["confidence"] for match in result],
|
||||
[match.confidence for match in result],
|
||||
[0.91, 0.73],
|
||||
)
|
||||
|
||||
@@ -157,8 +164,15 @@ class SearchOrchestrationTests(unittest.IsolatedAsyncioTestCase):
|
||||
|
||||
async def test_search_sources_orchestrator_success(self):
|
||||
# Setup mocks
|
||||
self.mock_civitai_search.return_value = {"name": "CivitAI Match", "confidence": 100.0}
|
||||
self.mock_hf_search.return_value = {"name": "HF Match"}
|
||||
self.mock_civitai_search.return_value = typed_contracts.SearchResult(
|
||||
source="civitai",
|
||||
name="CivitAI Match",
|
||||
confidence=100.0,
|
||||
)
|
||||
self.mock_hf_search.return_value = typed_contracts.SearchResult(
|
||||
source="huggingface",
|
||||
name="HF Match",
|
||||
)
|
||||
|
||||
# Construct a mock request
|
||||
payload = {
|
||||
@@ -270,7 +284,11 @@ class SearchOrchestrationTests(unittest.IsolatedAsyncioTestCase):
|
||||
def side_effect(filename, base_model_context=None, **kwargs):
|
||||
if base_model_context is not None:
|
||||
return None
|
||||
return {"name": "Fallback Match", "confidence": 90.0}
|
||||
return typed_contracts.SearchResult(
|
||||
source="civitai",
|
||||
name="Fallback Match",
|
||||
confidence=90.0,
|
||||
)
|
||||
|
||||
self.mock_civitai_search.side_effect = side_effect
|
||||
|
||||
@@ -300,17 +318,16 @@ class SearchOrchestrationTests(unittest.IsolatedAsyncioTestCase):
|
||||
"resolved_path": model_path,
|
||||
"sha256": file_hash,
|
||||
}
|
||||
civarchive_result = {
|
||||
"source": "civarchive",
|
||||
"filename": os.path.basename(model_path),
|
||||
"model_name": "Archive Match",
|
||||
"name": "Archive Match",
|
||||
"sha256": file_hash,
|
||||
"hashes": {"SHA256": file_hash},
|
||||
"download_url": "https://civarchive.com/api/download/models/123.safetensors",
|
||||
"url": "https://civarchive.com/models/123",
|
||||
"version_url": "https://civarchive.com/models/123?modelVersionId=456",
|
||||
}
|
||||
civarchive_result = typed_contracts.SearchResult(
|
||||
source="civarchive",
|
||||
filename=os.path.basename(model_path),
|
||||
name="Archive Match",
|
||||
sha256=file_hash,
|
||||
hashes={"SHA256": file_hash},
|
||||
download_url="https://civarchive.com/api/download/models/123.safetensors",
|
||||
url="https://civarchive.com/models/123",
|
||||
version_url="https://civarchive.com/models/123?modelVersionId=456",
|
||||
)
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
@@ -477,14 +494,14 @@ class SearchOrchestrationTests(unittest.IsolatedAsyncioTestCase):
|
||||
with open(metadata_path, "w", encoding="utf-8") as metadata_file:
|
||||
json.dump({"hashes": {"SHA256": file_hash}}, metadata_file)
|
||||
|
||||
self.mock_civitai_search.return_value = {
|
||||
"source": "civitai",
|
||||
"name": "Remote Match",
|
||||
"filename": "remote.safetensors",
|
||||
"download_url": "https://example.test/remote.safetensors",
|
||||
"confidence": 52.0,
|
||||
"hashes": {"SHA256": file_hash},
|
||||
}
|
||||
self.mock_civitai_search.return_value = typed_contracts.SearchResult(
|
||||
source="civitai",
|
||||
name="Remote Match",
|
||||
filename="remote.safetensors",
|
||||
download_url="https://example.test/remote.safetensors",
|
||||
confidence=52.0,
|
||||
hashes={"SHA256": file_hash},
|
||||
)
|
||||
mock_request = MagicMock()
|
||||
mock_request.json = AsyncMock(return_value={
|
||||
"filename": "remote.safetensors",
|
||||
@@ -569,4 +586,4 @@ class SearchOrchestrationTests(unittest.IsolatedAsyncioTestCase):
|
||||
category="checkpoints",
|
||||
)
|
||||
|
||||
self.assertEqual([match["filename"] for match in matches], ["cached.safetensors"])
|
||||
self.assertEqual([match.filename for match in matches], ["cached.safetensors"])
|
||||
|
||||
@@ -4,11 +4,17 @@ from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from core.contracts import ModelCatalogEntry, ModelMatch, SearchResult
|
||||
from core.routes.context import RouteContext
|
||||
from core.services.search_cache import SearchResultCache
|
||||
from core.services.search_dependencies import SearchDependencies
|
||||
from core.services.search_orchestrator import SearchOrchestrator
|
||||
from core.services.search_providers import SearchCancelled, SearchProviderRunner
|
||||
from core.services.search_providers import (
|
||||
SearchCancelled,
|
||||
SearchProviderRunner,
|
||||
SearchRequest,
|
||||
)
|
||||
from core.type_utils import build_model_result
|
||||
|
||||
|
||||
def _request(**overrides):
|
||||
@@ -21,9 +27,128 @@ def _request(**overrides):
|
||||
"force_search": False,
|
||||
}
|
||||
values.update(overrides)
|
||||
request_data = values.get("data")
|
||||
if isinstance(request_data, dict):
|
||||
values.setdefault("model_id", request_data.get("model_id"))
|
||||
values.setdefault("version_id", request_data.get("version_id"))
|
||||
return SimpleNamespace(**values)
|
||||
|
||||
|
||||
def _typed_search_request(**overrides):
|
||||
values = {
|
||||
"filename": "model.safetensors",
|
||||
"category": "checkpoints",
|
||||
"base_model_context": "SDXL",
|
||||
"progress_id": "search-1",
|
||||
"progress_source": "civitai",
|
||||
"civitai_candidate_limit": 5,
|
||||
"civarchive_candidate_limit": 10,
|
||||
"is_urn": False,
|
||||
"civitai_key": "",
|
||||
"civitai_session_token": "",
|
||||
"hf_token": "",
|
||||
"brave_search_api_key": "",
|
||||
"civitai_use_trpc_search": True,
|
||||
"civitai_use_api_search": True,
|
||||
"civitai_use_html_fallback": True,
|
||||
"hf_use_api_search": True,
|
||||
"hf_use_comfy_org_fallback": True,
|
||||
"hf_use_brave_fallback": True,
|
||||
"force_search": False,
|
||||
"normalized_sources": frozenset({"civitai"}),
|
||||
"sha256": "",
|
||||
}
|
||||
values.update(overrides)
|
||||
return SearchRequest(**values)
|
||||
|
||||
|
||||
def test_search_request_from_mapping_adapts_http_payload_once():
|
||||
request = SearchRequest.from_mapping(
|
||||
{
|
||||
"filename": " model.safetensors ",
|
||||
"sources": ["CivitAI"],
|
||||
"civitai_candidate_limit": 100,
|
||||
"is_urn": "true",
|
||||
"model_id": " 12 ",
|
||||
"version_id": 34,
|
||||
"sha256": "A" * 64,
|
||||
},
|
||||
)
|
||||
|
||||
assert request.filename == "model.safetensors"
|
||||
assert request.normalized_sources == frozenset({"civitai"})
|
||||
assert request.civitai_candidate_limit == 20
|
||||
assert request.is_urn is True
|
||||
assert request.model_id == "12"
|
||||
assert request.version_id == 34
|
||||
assert request.sha256 == "a" * 64
|
||||
|
||||
|
||||
def test_search_request_from_mapping_rejects_non_object_payloads():
|
||||
with pytest.raises(TypeError, match="Search request must be an object"):
|
||||
SearchRequest.from_mapping(
|
||||
[],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("payload", "message"),
|
||||
[
|
||||
({"filename": 123}, "SearchRequest filename must be a string"),
|
||||
(
|
||||
{"filename": "model.safetensors", "category": 123},
|
||||
"SearchRequest category must be a string",
|
||||
),
|
||||
(
|
||||
{"filename": "model.safetensors", "sources": {"civitai": True}},
|
||||
"SearchRequest sources must be a string or array",
|
||||
),
|
||||
(
|
||||
{"filename": "model.safetensors", "sources": ["civitai", 1]},
|
||||
"SearchRequest sources must contain only strings",
|
||||
),
|
||||
(
|
||||
{"filename": "model.safetensors", "sources": None},
|
||||
"SearchRequest sources must be a string or array",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_search_request_from_mapping_rejects_malformed_http_fields(payload, message):
|
||||
with pytest.raises(TypeError, match=message):
|
||||
SearchRequest.from_mapping(
|
||||
payload,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field_name", "value"),
|
||||
[
|
||||
("category", 123),
|
||||
("civitai_key", 123),
|
||||
("is_urn", "false"),
|
||||
("force_search", {"enabled": True}),
|
||||
("civitai_candidate_limit", "5"),
|
||||
("normalized_sources", {"civitai"}),
|
||||
("model_id", 1.5),
|
||||
("version_id", False),
|
||||
],
|
||||
)
|
||||
def test_search_request_rejects_invalid_normalized_field_types(field_name, value):
|
||||
with pytest.raises((TypeError, ValueError), match=field_name):
|
||||
_typed_search_request(**{field_name: value})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field_name",
|
||||
["civitai_candidate_limit", "civarchive_candidate_limit"],
|
||||
)
|
||||
def test_search_request_from_mapping_rejects_invalid_limit_types(field_name):
|
||||
with pytest.raises((TypeError, ValueError), match=field_name):
|
||||
SearchRequest.from_mapping(
|
||||
{"filename": "model.safetensors", field_name: "invalid"},
|
||||
)
|
||||
|
||||
|
||||
def _build_search_orchestrator():
|
||||
tracker = MagicMock()
|
||||
tracker.is_cancelled.return_value = False
|
||||
@@ -33,19 +158,6 @@ def _build_search_orchestrator():
|
||||
logger=MagicMock(),
|
||||
)
|
||||
|
||||
def to_bool(value, default=False):
|
||||
if value is None:
|
||||
return default
|
||||
if isinstance(value, str):
|
||||
return value.strip().lower() in {"1", "true", "yes", "on"}
|
||||
return bool(value)
|
||||
|
||||
def to_int(value, default=0):
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
def extract_sha256(metadata):
|
||||
return metadata.get("sha256") or (
|
||||
metadata.get("hashes") or {}
|
||||
@@ -55,10 +167,7 @@ def _build_search_orchestrator():
|
||||
"self": extension,
|
||||
"asyncio": asyncio,
|
||||
"CivArchiveSearchError": Exception,
|
||||
"build_model_result": lambda source, **fields: {
|
||||
"source": source,
|
||||
**fields,
|
||||
},
|
||||
"build_model_result": build_model_result,
|
||||
"clear_civarchive_search_cache": MagicMock(),
|
||||
"clear_civitai_search_cache": MagicMock(),
|
||||
"clear_huggingface_search_cache": MagicMock(),
|
||||
@@ -78,8 +187,6 @@ def _build_search_orchestrator():
|
||||
"search_local_matches_by_hash": MagicMock(return_value=[]),
|
||||
"search_lora_manager_archive_for_file": MagicMock(),
|
||||
"search_model_list": MagicMock(),
|
||||
"to_bool": to_bool,
|
||||
"to_int": to_int,
|
||||
"web": SimpleNamespace(
|
||||
json_response=lambda payload, status=200: SimpleNamespace(
|
||||
payload=payload,
|
||||
@@ -200,7 +307,7 @@ def test_search_provider_runner_retries_without_base_model_context():
|
||||
calls.append((base_model_context, progress_callback))
|
||||
if len(calls) == 1:
|
||||
return None
|
||||
return {"name": "Fallback model"}
|
||||
return SearchResult(source="civitai", name="Fallback model")
|
||||
|
||||
result = runner.execute_search_with_fallback(
|
||||
_request(),
|
||||
@@ -210,9 +317,9 @@ def test_search_provider_runner_retries_without_base_model_context():
|
||||
)
|
||||
|
||||
assert [call[0] for call in calls] == ["SDXL", None]
|
||||
assert result["any_model_match"] is True
|
||||
assert result["base_model_fallback"] is True
|
||||
assert result["requested_base_model"] == "SDXL"
|
||||
assert result.extra_value("any_model_match") is True
|
||||
assert result.extra_value("base_model_fallback") is True
|
||||
assert result.extra_value("requested_base_model") == "SDXL"
|
||||
|
||||
|
||||
def test_search_provider_runner_searches_local_sources():
|
||||
@@ -220,15 +327,20 @@ def test_search_provider_runner_searches_local_sources():
|
||||
logger=MagicMock(),
|
||||
search_tracker=MagicMock(),
|
||||
format_log_fields=MagicMock(return_value="file=model.safetensors"),
|
||||
build_model_result=build_model_result,
|
||||
get_popular_model_url=MagicMock(
|
||||
return_value={"download_url": "https://example.test/model"}
|
||||
return_value=ModelCatalogEntry(
|
||||
filename="model.safetensors",
|
||||
download_url="https://example.test/model",
|
||||
)
|
||||
),
|
||||
search_model_list=MagicMock(
|
||||
return_value={
|
||||
"filename": "model.safetensors",
|
||||
"size": 123,
|
||||
"confidence": 95,
|
||||
}
|
||||
return_value=SearchResult(
|
||||
source="model_list",
|
||||
filename="model.safetensors",
|
||||
size=123,
|
||||
confidence=95,
|
||||
)
|
||||
),
|
||||
log_search_result=MagicMock(),
|
||||
)
|
||||
@@ -238,9 +350,9 @@ def test_search_provider_runner_searches_local_sources():
|
||||
result, found = runner.search_local_sources(_request(is_urn=False))
|
||||
|
||||
assert found is True
|
||||
assert result["popular"]["size"] == 123
|
||||
assert result["popular"]["download_url"] == "https://example.test/model"
|
||||
assert result["model_list"]["confidence"] == 95
|
||||
assert result["popular"].size == 123
|
||||
assert result["popular"].download_url == "https://example.test/model"
|
||||
assert result["model_list"].confidence == 95
|
||||
|
||||
|
||||
def test_search_provider_runner_scales_provider_progress():
|
||||
@@ -358,7 +470,7 @@ def test_search_provider_runner_resolves_civitai_urn():
|
||||
}
|
||||
),
|
||||
get_civitai_download_url=MagicMock(return_value="https://example.test/download"),
|
||||
build_model_result=MagicMock(side_effect=lambda source, **fields: {"source": source, **fields}),
|
||||
build_model_result=MagicMock(side_effect=build_model_result),
|
||||
)
|
||||
owner.search_tracker.is_cancelled.return_value = False
|
||||
owner.log_search_result = MagicMock()
|
||||
@@ -373,20 +485,20 @@ def test_search_provider_runner_resolves_civitai_urn():
|
||||
)
|
||||
|
||||
assert found is True
|
||||
assert result["civitai"]["model_id"] == 10
|
||||
assert result["civitai"]["version_id"] == 20
|
||||
assert result["civitai"]["name"] == "URN Model"
|
||||
assert result["civitai"]["version_name"] == "v1"
|
||||
assert result["civitai"]["filename"] == "urn.safetensors"
|
||||
assert result["civitai"]["type"] == "checkpoints"
|
||||
assert result["civitai"]["download_url"] == "https://example.test/download"
|
||||
assert result["civitai"]["url"] == "https://civitai.com/models/10?modelVersionId=20"
|
||||
assert result["civitai"]["size"] is None
|
||||
assert result["civitai"]["base_model"] == "SDXL"
|
||||
assert result["civitai"]["match_type"] == "exact"
|
||||
assert result["civitai"]["confidence"] == 100.0
|
||||
assert result["civitai"]["sha256"] == "a" * 64
|
||||
assert result["civitai"]["hashes"] == {}
|
||||
assert result["civitai"].model_id == 10
|
||||
assert result["civitai"].version_id == 20
|
||||
assert result["civitai"].name == "URN Model"
|
||||
assert result["civitai"].version_name == "v1"
|
||||
assert result["civitai"].filename == "urn.safetensors"
|
||||
assert result["civitai"].model_type == "checkpoints"
|
||||
assert result["civitai"].download_url == "https://example.test/download"
|
||||
assert result["civitai"].url == "https://civitai.com/models/10?modelVersionId=20"
|
||||
assert result["civitai"].size is None
|
||||
assert result["civitai"].base_model == "SDXL"
|
||||
assert result["civitai"].match_type == "exact"
|
||||
assert result["civitai"].confidence == 100.0
|
||||
assert result["civitai"].sha256 == "a" * 64
|
||||
assert result["civitai"].hashes == {}
|
||||
|
||||
|
||||
def test_search_provider_runner_keeps_first_file_when_expected_name_is_missing():
|
||||
@@ -413,7 +525,7 @@ def test_search_provider_runner_keeps_first_file_when_expected_name_is_missing()
|
||||
}
|
||||
),
|
||||
get_civitai_download_url=MagicMock(return_value="https://example.test/download"),
|
||||
build_model_result=MagicMock(side_effect=lambda source, **fields: {"source": source, **fields}),
|
||||
build_model_result=MagicMock(side_effect=build_model_result),
|
||||
)
|
||||
owner.search_tracker.is_cancelled.return_value = False
|
||||
owner.log_search_result = MagicMock()
|
||||
@@ -428,7 +540,7 @@ def test_search_provider_runner_keeps_first_file_when_expected_name_is_missing()
|
||||
)
|
||||
|
||||
assert found is True
|
||||
assert result["civitai"]["sha256"] == "a" * 64
|
||||
assert result["civitai"].sha256 == "a" * 64
|
||||
|
||||
|
||||
def test_search_provider_runner_falls_back_for_civitai_urn_without_ids():
|
||||
@@ -438,18 +550,19 @@ def test_search_provider_runner_falls_back_for_civitai_urn_without_ids():
|
||||
format_log_fields=MagicMock(return_value="file=model.safetensors"),
|
||||
search_civitai=MagicMock(
|
||||
return_value=[
|
||||
{
|
||||
"model_id": 30,
|
||||
"version_id": 40,
|
||||
"name": "Fallback model",
|
||||
"filename": "model.safetensors",
|
||||
"type": "Checkpoint",
|
||||
"download_url": "https://example.test/fallback",
|
||||
"url": "https://example.test/model/30",
|
||||
"size": 456,
|
||||
"base_model": "SDXL",
|
||||
"tags": ["fallback"],
|
||||
}
|
||||
SearchResult(
|
||||
source="civitai",
|
||||
model_id=30,
|
||||
version_id=40,
|
||||
name="Fallback model",
|
||||
filename="model.safetensors",
|
||||
model_type="Checkpoint",
|
||||
download_url="https://example.test/fallback",
|
||||
url="https://example.test/model/30",
|
||||
size=456,
|
||||
base_model="SDXL",
|
||||
tags=["fallback"],
|
||||
)
|
||||
]
|
||||
),
|
||||
build_model_result=MagicMock(
|
||||
@@ -465,8 +578,9 @@ def test_search_provider_runner_falls_back_for_civitai_urn_without_ids():
|
||||
)
|
||||
|
||||
assert found is True
|
||||
assert result["civitai"]["model_id"] == 30
|
||||
assert result["civitai"]["download_url"] == "https://example.test/fallback"
|
||||
assert isinstance(result["civitai"], SearchResult)
|
||||
assert result["civitai"].model_id == 30
|
||||
assert result["civitai"].download_url == "https://example.test/fallback"
|
||||
owner.search_civitai.assert_called_once_with(
|
||||
"model.safetensors",
|
||||
model_type="checkpoints",
|
||||
@@ -480,7 +594,11 @@ def test_search_provider_runner_resolves_civarchive_urn():
|
||||
search_tracker=MagicMock(),
|
||||
format_log_fields=MagicMock(return_value="ids=10@20"),
|
||||
resolve_civarchive_model_version=MagicMock(
|
||||
return_value={"name": "Archived model", "version_id": 20}
|
||||
return_value=SearchResult(
|
||||
source="civarchive",
|
||||
name="Archived model",
|
||||
version_id=20,
|
||||
)
|
||||
),
|
||||
log_search_result=MagicMock(),
|
||||
)
|
||||
@@ -495,7 +613,8 @@ def test_search_provider_runner_resolves_civarchive_urn():
|
||||
)
|
||||
|
||||
assert found is True
|
||||
assert result["civarchive"] == {"name": "Archived model", "version_id": 20}
|
||||
assert result["civarchive"].name == "Archived model"
|
||||
assert result["civarchive"].version_id == 20
|
||||
owner.resolve_civarchive_model_version.assert_called_once_with(
|
||||
10,
|
||||
20,
|
||||
@@ -509,7 +628,7 @@ def test_search_provider_runner_searches_lora_manager_archive():
|
||||
search_tracker=MagicMock(),
|
||||
format_log_fields=MagicMock(return_value="file=model.safetensors"),
|
||||
search_lora_manager_archive_for_file=MagicMock(
|
||||
return_value={"name": "Archived LoRA"}
|
||||
return_value=SearchResult(source="lora_manager_archive", name="Archived LoRA")
|
||||
),
|
||||
log_search_result=MagicMock(),
|
||||
)
|
||||
@@ -521,7 +640,7 @@ def test_search_provider_runner_searches_lora_manager_archive():
|
||||
)
|
||||
|
||||
assert found is True
|
||||
assert result == {"lora_manager_archive": {"name": "Archived LoRA"}}
|
||||
assert result["lora_manager_archive"].name == "Archived LoRA"
|
||||
owner.search_lora_manager_archive_for_file.assert_called_once()
|
||||
|
||||
|
||||
@@ -710,7 +829,7 @@ async def test_search_orchestrator_returns_cancelled_response():
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_orchestrator_returns_error_response_for_unexpected_failure():
|
||||
async def test_search_orchestrator_rejects_invalid_request_payload():
|
||||
orchestrator, dependencies = _build_search_orchestrator()
|
||||
request = SimpleNamespace(
|
||||
json=AsyncMock(side_effect=ValueError("invalid request"))
|
||||
@@ -718,12 +837,32 @@ async def test_search_orchestrator_returns_error_response_for_unexpected_failure
|
||||
|
||||
response = await orchestrator.search_sources(request)
|
||||
|
||||
assert response.status == 500
|
||||
assert response.payload == {"error": "invalid request"}
|
||||
error_update = dependencies["self"].search_tracker.update.call_args
|
||||
assert error_update.args[:5] == ("", "", "error", "invalid request", 100)
|
||||
assert error_update.kwargs == {"status": "error"}
|
||||
dependencies["self"].logger.exception.assert_called_once()
|
||||
assert response.status == 400
|
||||
assert response.payload == {
|
||||
"error": "Filename is required for non-URN, or model_id+version_id for URN"
|
||||
}
|
||||
dependencies["self"].search_tracker.update.assert_not_called()
|
||||
dependencies["self"].logger.exception.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"payload",
|
||||
[
|
||||
{"filename": "model.safetensors", "category": 123},
|
||||
{"filename": "model.safetensors", "sources": {"civitai": True}},
|
||||
],
|
||||
)
|
||||
async def test_search_orchestrator_returns_bad_request_for_contract_errors(payload):
|
||||
orchestrator, dependencies = _build_search_orchestrator()
|
||||
request = SimpleNamespace(json=AsyncMock(return_value=payload))
|
||||
|
||||
response = await orchestrator.search_sources(request)
|
||||
|
||||
assert response.status == 400
|
||||
assert response.payload["error"]
|
||||
dependencies["self"].search_tracker.update.assert_not_called()
|
||||
dependencies["self"].logger.exception.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -735,8 +874,18 @@ async def test_search_orchestrator_deduplicates_local_hash_matches_and_keeps_err
|
||||
def lookup_by_hash(sha256, **kwargs):
|
||||
if sha256 == first_hash:
|
||||
return [
|
||||
{"model": {"path": r"C:\\Models\\Local.safetensors"}},
|
||||
{"path": r"c:\\models\\local.safetensors"},
|
||||
ModelMatch.from_mapping(
|
||||
{
|
||||
"model": {"path": r"C:\\Models\\Local.safetensors"},
|
||||
"filename": "Local.safetensors",
|
||||
}
|
||||
),
|
||||
ModelMatch.from_mapping(
|
||||
{
|
||||
"model": {"path": r"c:\\models\\local.safetensors"},
|
||||
"filename": "local.safetensors",
|
||||
}
|
||||
),
|
||||
]
|
||||
raise RuntimeError("local index unavailable")
|
||||
|
||||
@@ -744,12 +893,20 @@ async def test_search_orchestrator_deduplicates_local_hash_matches_and_keeps_err
|
||||
orchestrator.provider_runner = _StaticSearchRunner(
|
||||
{
|
||||
"civitai": [
|
||||
{
|
||||
"filename": "remote.safetensors",
|
||||
"hashes": {"SHA256": first_hash},
|
||||
},
|
||||
{"filename": "without-hash.safetensors"},
|
||||
{"filename": "failing.safetensors", "sha256": failing_hash},
|
||||
SearchResult(
|
||||
source="civitai",
|
||||
filename="remote.safetensors",
|
||||
hashes={"SHA256": first_hash},
|
||||
),
|
||||
SearchResult(
|
||||
source="civitai",
|
||||
filename="without-hash.safetensors",
|
||||
),
|
||||
SearchResult(
|
||||
source="civitai",
|
||||
filename="failing.safetensors",
|
||||
sha256=failing_hash,
|
||||
),
|
||||
],
|
||||
"source_errors": {"civarchive": "provider unavailable"},
|
||||
},
|
||||
|
||||
+155
-26
@@ -18,6 +18,8 @@ from unittest.mock import MagicMock, patch
|
||||
|
||||
import requests
|
||||
|
||||
from core.contracts import ResolvedModel, SearchResult
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# civarchive internals
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -35,7 +37,10 @@ from core.sources.civarchive import (
|
||||
_resolve_file_size_bytes,
|
||||
_transform_file_entry,
|
||||
build_civarchive_custom_result,
|
||||
get_civarchive_model_details,
|
||||
parse_civarchive_url,
|
||||
search_civarchive,
|
||||
search_civarchive_for_file,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -62,8 +67,8 @@ from core.sources.civitai import (
|
||||
get_model_info_by_hash,
|
||||
get_model_info_for_file,
|
||||
parse_civitai_url,
|
||||
resolve_urn,
|
||||
resolve_civitai_version_custom_result,
|
||||
resolve_urn,
|
||||
search_civitai,
|
||||
search_civitai_for_file,
|
||||
)
|
||||
@@ -174,6 +179,8 @@ class CivitaiResultBuilderTests(unittest.TestCase):
|
||||
tags=["character"],
|
||||
)
|
||||
|
||||
self.assertIsInstance(result, SearchResult)
|
||||
result = result.to_dict()
|
||||
self.assertEqual("civitai", result["source"])
|
||||
self.assertEqual(123, result["model_id"])
|
||||
self.assertEqual(456, result["version_id"])
|
||||
@@ -196,7 +203,10 @@ class CivitaiCustomVersionLookupTests(unittest.TestCase):
|
||||
) as get_details,
|
||||
patch(
|
||||
"core.sources.civitai.build_civitai_custom_result",
|
||||
return_value={"source": "civitai", "version_id": 456},
|
||||
return_value=SearchResult(
|
||||
source="civitai",
|
||||
version_id=456,
|
||||
),
|
||||
),
|
||||
):
|
||||
result = resolve_civitai_version_custom_result(
|
||||
@@ -205,7 +215,9 @@ class CivitaiCustomVersionLookupTests(unittest.TestCase):
|
||||
api_key="secret",
|
||||
)
|
||||
|
||||
self.assertEqual({"source": "civitai", "version_id": 456}, result)
|
||||
self.assertIsInstance(result, SearchResult)
|
||||
self.assertEqual("civitai", result.source)
|
||||
self.assertEqual(456, result.version_id)
|
||||
request_json.assert_called_once_with(
|
||||
"CivitAI custom URL version lookup",
|
||||
"https://civitai.com/api/v1/model-versions/456",
|
||||
@@ -242,6 +254,113 @@ class CivitaiCustomVersionLookupTests(unittest.TestCase):
|
||||
|
||||
class CivarchiveResultBuilderTests(unittest.TestCase):
|
||||
|
||||
def test_name_search_sorts_typed_results(self):
|
||||
lower_result = SearchResult(
|
||||
source="civarchive",
|
||||
name="typed-civarchive-lower",
|
||||
filename="typed-civarchive-lower.safetensors",
|
||||
confidence=55.0,
|
||||
match_type="similar",
|
||||
)
|
||||
higher_result = SearchResult(
|
||||
source="civarchive",
|
||||
name="typed-civarchive-higher",
|
||||
filename="typed-civarchive-higher.safetensors",
|
||||
confidence=90.0,
|
||||
match_type="exact",
|
||||
)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"core.sources.civarchive._search_page",
|
||||
return_value=[
|
||||
{"id": 1, "url": "/models/1", "name": "lower"},
|
||||
{"id": 2, "url": "/models/2", "name": "higher"},
|
||||
],
|
||||
),
|
||||
patch(
|
||||
"core.sources.civarchive._resolve_search_candidate",
|
||||
side_effect=[lower_result, higher_result],
|
||||
),
|
||||
):
|
||||
results = search_civarchive(
|
||||
"typed-civarchive-sort-regression",
|
||||
limit=2,
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
[higher_result, lower_result],
|
||||
results,
|
||||
)
|
||||
|
||||
def test_details_fallback_serializes_typed_version_result(self):
|
||||
resolved = SearchResult(
|
||||
source="civarchive",
|
||||
model_id=123,
|
||||
version_id=456,
|
||||
name="Typed fallback model",
|
||||
version_name="v1",
|
||||
model_type="LORA",
|
||||
filename="typed-fallback.safetensors",
|
||||
download_url="https://civarchive.com/download/456",
|
||||
base_model="SDXL",
|
||||
)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"core.sources.civarchive._request_model_payload",
|
||||
return_value=None,
|
||||
),
|
||||
patch(
|
||||
"core.sources.civarchive.resolve_civarchive_model_version",
|
||||
return_value=resolved,
|
||||
),
|
||||
patch(
|
||||
"core.sources.civarchive._normalize_archive_version",
|
||||
return_value={
|
||||
"id": 456,
|
||||
"url": "https://civarchive.com/models/123?modelVersionId=456",
|
||||
},
|
||||
),
|
||||
):
|
||||
result = get_civarchive_model_details(123, version_id=456)
|
||||
|
||||
self.assertEqual("civarchive", result["source"])
|
||||
self.assertEqual(456, result["version_id"])
|
||||
self.assertEqual("Typed fallback model", result["name"])
|
||||
|
||||
def test_filename_search_keeps_typed_resolved_result(self):
|
||||
filename = "typed-civarchive-model.safetensors"
|
||||
resolved = SearchResult(
|
||||
source="civarchive",
|
||||
name="typed-civarchive-model",
|
||||
version_name="v1",
|
||||
filename=filename,
|
||||
download_url="https://civarchive.com/api/download/models/456",
|
||||
)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"core.sources.civarchive._search_page",
|
||||
return_value=[
|
||||
{
|
||||
"name": "typed-civarchive-model",
|
||||
"url": "/models/123?modelVersionId=456",
|
||||
}
|
||||
],
|
||||
),
|
||||
patch(
|
||||
"core.sources.civarchive._resolve_search_candidate",
|
||||
return_value=resolved,
|
||||
),
|
||||
):
|
||||
result = search_civarchive_for_file(filename, limit=1)
|
||||
|
||||
self.assertIsInstance(result, SearchResult)
|
||||
self.assertEqual(filename, result.filename)
|
||||
self.assertEqual(100.0, result.confidence)
|
||||
self.assertEqual("exact", result.match_type)
|
||||
|
||||
def test_normalized_version_builder_preserves_search_result_contract(self):
|
||||
result = _build_result_from_normalized_version(
|
||||
model_details={
|
||||
@@ -268,6 +387,8 @@ class CivarchiveResultBuilderTests(unittest.TestCase):
|
||||
match_type="similar",
|
||||
)
|
||||
|
||||
self.assertIsInstance(result, SearchResult)
|
||||
result = result.to_dict()
|
||||
self.assertEqual("civarchive", result["source"])
|
||||
self.assertEqual(123, result["model_id"])
|
||||
self.assertEqual(456, result["version_id"])
|
||||
@@ -310,6 +431,8 @@ class CivarchiveResultBuilderTests(unittest.TestCase):
|
||||
preferred_filename="archive.safetensors",
|
||||
)
|
||||
|
||||
self.assertIsInstance(result, SearchResult)
|
||||
result = result.to_dict()
|
||||
self.assertEqual("civarchive", result["source"])
|
||||
self.assertEqual(123, result["model_id"])
|
||||
self.assertEqual(456, result["version_id"])
|
||||
@@ -448,8 +571,9 @@ class CivitaiPrimaryFileSelectionTests(unittest.TestCase):
|
||||
|
||||
results = search_civitai("example", limit=1)
|
||||
|
||||
self.assertEqual("first.safetensors", results[0]["filename"])
|
||||
self.assertEqual(1024, results[0]["size"])
|
||||
self.assertIsInstance(results[0], SearchResult)
|
||||
self.assertEqual("first.safetensors", results[0].filename)
|
||||
self.assertEqual(1024, results[0].size)
|
||||
clear_search_cache()
|
||||
|
||||
|
||||
@@ -542,7 +666,7 @@ class CustomUrlResultBuilderTests(unittest.TestCase):
|
||||
"hashes": {"SHA256": "abc123"},
|
||||
"match_type": "custom_url",
|
||||
"custom_url": True,
|
||||
}, result)
|
||||
}, result.to_dict())
|
||||
|
||||
def test_civarchive_custom_result_preserves_provider_fields(self):
|
||||
result = build_civarchive_custom_result({
|
||||
@@ -606,7 +730,7 @@ class CustomUrlResultBuilderTests(unittest.TestCase):
|
||||
"platform": "civitai",
|
||||
"match_type": "custom_url",
|
||||
"custom_url": True,
|
||||
}, result)
|
||||
}, result.to_dict())
|
||||
|
||||
|
||||
class CivitaiModelDetailsTests(unittest.TestCase):
|
||||
@@ -1153,13 +1277,13 @@ class ParseCivarchiveUrlTests(unittest.TestCase):
|
||||
def test_model_url_parsed(self):
|
||||
result = parse_civarchive_url("https://civarchive.com/models/123")
|
||||
self.assertIsNotNone(result)
|
||||
self.assertEqual(result.get("model_id"), 123)
|
||||
self.assertEqual(result.model_id, 123)
|
||||
|
||||
def test_model_version_url_parsed(self):
|
||||
result = parse_civarchive_url("https://civarchive.com/models/123?modelVersionId=456")
|
||||
self.assertIsNotNone(result)
|
||||
self.assertEqual(result.get("model_id"), 123)
|
||||
self.assertEqual(result.get("version_id"), 456)
|
||||
self.assertEqual(result.model_id, 123)
|
||||
self.assertEqual(result.version_id, 456)
|
||||
|
||||
def test_non_civarchive_url_returns_none(self):
|
||||
self.assertIsNone(parse_civarchive_url("https://huggingface.co/user/repo"))
|
||||
@@ -1339,24 +1463,24 @@ class ParseCivitaiUrlTests(unittest.TestCase):
|
||||
def test_standard_model_url(self):
|
||||
result = parse_civitai_url("https://civitai.com/models/123456")
|
||||
self.assertIsNotNone(result)
|
||||
self.assertEqual(result.get("model_id"), 123456)
|
||||
self.assertEqual(result.model_id, 123456)
|
||||
|
||||
def test_model_url_with_version(self):
|
||||
result = parse_civitai_url("https://civitai.com/models/123456?modelVersionId=789")
|
||||
self.assertIsNotNone(result)
|
||||
self.assertEqual(result.get("model_id"), 123456)
|
||||
self.assertEqual(result.get("version_id"), 789)
|
||||
self.assertEqual(result.model_id, 123456)
|
||||
self.assertEqual(result.version_id, 789)
|
||||
|
||||
def test_civitai_red_model_url_with_version(self):
|
||||
result = parse_civitai_url("https://civitai.red/models/123456?modelVersionId=789")
|
||||
self.assertIsNotNone(result)
|
||||
self.assertEqual(result.get("model_id"), 123456)
|
||||
self.assertEqual(result.get("version_id"), 789)
|
||||
self.assertEqual(result.model_id, 123456)
|
||||
self.assertEqual(result.version_id, 789)
|
||||
|
||||
def test_civitai_red_download_url(self):
|
||||
result = parse_civitai_url("https://civitai.red/api/download/models/789")
|
||||
self.assertIsNotNone(result)
|
||||
self.assertEqual(result.get("version_id"), 789)
|
||||
self.assertEqual(result.version_id, 789)
|
||||
|
||||
def test_non_civitai_url_returns_none(self):
|
||||
self.assertIsNone(parse_civitai_url("https://civarchive.com/models/123"))
|
||||
@@ -1539,13 +1663,13 @@ class CivitaiTrpcSearchTests(unittest.TestCase):
|
||||
|
||||
def test_public_api_candidates_used_when_trpc_disabled_and_html_empty(self):
|
||||
clear_search_cache()
|
||||
expected = {
|
||||
"source": "civitai",
|
||||
"model_id": 2676616,
|
||||
"version_id": 3005748,
|
||||
"filename": "sickOllie_v1.safetensors",
|
||||
"confidence": 100.0,
|
||||
}
|
||||
expected = SearchResult(
|
||||
source="civitai",
|
||||
model_id=2676616,
|
||||
version_id=3005748,
|
||||
filename="sickOllie_v1.safetensors",
|
||||
confidence=100.0,
|
||||
)
|
||||
with patch(
|
||||
"core.sources.civitai._search_civitai_red_candidates",
|
||||
return_value=[],
|
||||
@@ -1564,7 +1688,12 @@ class CivitaiTrpcSearchTests(unittest.TestCase):
|
||||
use_html_fallback=True,
|
||||
)
|
||||
|
||||
self.assertEqual(result, expected)
|
||||
self.assertIsInstance(result, SearchResult)
|
||||
self.assertEqual(expected.source, result.source)
|
||||
self.assertEqual(expected.model_id, result.model_id)
|
||||
self.assertEqual(expected.version_id, result.version_id)
|
||||
self.assertEqual(expected.filename, result.filename)
|
||||
self.assertEqual(expected.confidence, result.confidence)
|
||||
html_search.assert_called_once()
|
||||
api_search.assert_called_once()
|
||||
clear_search_cache()
|
||||
@@ -1906,12 +2035,12 @@ class ConvertToRelativePathTests(unittest.TestCase):
|
||||
class GetBaseDirectoryForModelTests(unittest.TestCase):
|
||||
|
||||
def test_returns_base_directory_if_provided(self):
|
||||
model = {"base_directory": "/some/path/models"}
|
||||
model = ResolvedModel(base_directory="/some/path/models")
|
||||
result = get_base_directory_for_model(model, "checkpoints")
|
||||
self.assertEqual(result, "/some/path/models")
|
||||
|
||||
def test_returns_none_without_path_or_base_directory(self):
|
||||
result = get_base_directory_for_model({}, "checkpoints")
|
||||
result = get_base_directory_for_model(ResolvedModel(), "checkpoints")
|
||||
self.assertIsNone(result)
|
||||
|
||||
|
||||
|
||||
@@ -1,15 +1,16 @@
|
||||
import sys
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch, AsyncMock
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
if parent_dir not in sys.path:
|
||||
sys.path.insert(0, parent_dir)
|
||||
|
||||
from core.sources.civitai import _find_model_title_match_in_model
|
||||
from core.sources.civarchive import _find_model_title_match_in_model_details
|
||||
from core.path_utils import calculate_file_sha256
|
||||
from core.sources.civarchive import _find_model_title_match_in_model_details
|
||||
from core.sources.civitai import _find_model_title_match_in_model
|
||||
|
||||
|
||||
class TestTitleMatchingAndHashing(unittest.IsolatedAsyncioTestCase):
|
||||
|
||||
@@ -44,11 +45,11 @@ class TestTitleMatchingAndHashing(unittest.IsolatedAsyncioTestCase):
|
||||
base_model_context="SDXL 1.0"
|
||||
)
|
||||
self.assertIsNotNone(result)
|
||||
self.assertEqual(result["model_id"], 1)
|
||||
self.assertEqual(result["version_id"], 123)
|
||||
self.assertEqual(result["filename"], "sdxl_base.safetensors")
|
||||
self.assertEqual(result["base_model"], "SDXL 1.0")
|
||||
self.assertIn("anime", result["tags"])
|
||||
self.assertEqual(result.model_id, 1)
|
||||
self.assertEqual(result.version_id, 123)
|
||||
self.assertEqual(result.filename, "sdxl_base.safetensors")
|
||||
self.assertEqual(result.base_model, "SDXL 1.0")
|
||||
self.assertIn("anime", result.tags)
|
||||
|
||||
def test_civitai_find_model_title_match_rejected_confidence(self):
|
||||
model_data = {
|
||||
@@ -92,10 +93,10 @@ class TestTitleMatchingAndHashing(unittest.IsolatedAsyncioTestCase):
|
||||
base_model_context="SD 1.5"
|
||||
)
|
||||
self.assertIsNotNone(result)
|
||||
self.assertEqual(result["model_id"], 10)
|
||||
self.assertEqual(result["version_id"], 999)
|
||||
self.assertEqual(result["filename"], "v1-5-pruned.safetensors")
|
||||
self.assertEqual(result["base_model"], "SD 1.5")
|
||||
self.assertEqual(result.model_id, 10)
|
||||
self.assertEqual(result.version_id, 999)
|
||||
self.assertEqual(result.filename, "v1-5-pruned.safetensors")
|
||||
self.assertEqual(result.base_model, "SD 1.5")
|
||||
|
||||
async def test_calculate_file_sha256_with_mock_progress(self):
|
||||
import tempfile
|
||||
|
||||
@@ -57,6 +57,7 @@ class UnifiedHelpersTests(unittest.TestCase):
|
||||
self.assertEqual("", extract_file_sha256(None))
|
||||
self.assertEqual("", extract_file_sha256({"hashes": []}))
|
||||
self.assertEqual("fallback", extract_file_sha256({"sha256": "", "hash": "fallback"}))
|
||||
self.assertEqual("", extract_file_sha256({"sha256": 123}))
|
||||
|
||||
def test_extract_sha256_from_metadata_uses_first_valid_source(self):
|
||||
preferred = "a" * 64
|
||||
@@ -1038,6 +1039,7 @@ class UnifiedHelpersTests(unittest.TestCase):
|
||||
self.assertEqual(normalize_sha256("SHA256:0X" + h.upper()), h)
|
||||
self.assertEqual(normalize_sha256("invalid_hash"), "")
|
||||
self.assertEqual(normalize_sha256(None), "")
|
||||
self.assertEqual(normalize_sha256(123), "")
|
||||
|
||||
def test_unique_ordered_strings(self):
|
||||
self.assertEqual(unique_ordered_strings(["a", "b", "a", "c", "b"]), ["a", "b", "c"])
|
||||
@@ -1405,11 +1407,14 @@ class UnifiedHelpersTests(unittest.TestCase):
|
||||
|
||||
# Valid url download format
|
||||
res = parse_provider_model_url("https://civitai.com/api/download/models/12345", allowed)
|
||||
self.assertEqual(res, {"version_id": 12345})
|
||||
self.assertEqual(res.to_dict(), {"version_id": 12345})
|
||||
|
||||
# Valid model page format
|
||||
res = parse_provider_model_url("https://civitai.red/models/9876?modelVersionId=54321", allowed)
|
||||
self.assertEqual(res, {"model_id": 9876, "version_id": 54321})
|
||||
self.assertEqual(
|
||||
res.to_dict(),
|
||||
{"model_id": 9876, "version_id": 54321},
|
||||
)
|
||||
|
||||
# Non-matching hostname
|
||||
res = parse_provider_model_url("https://example.com/api/download/models/12345", allowed)
|
||||
@@ -1417,7 +1422,9 @@ class UnifiedHelpersTests(unittest.TestCase):
|
||||
|
||||
# Mirror sha256 path pattern
|
||||
res = parse_provider_model_url("https://civarchive.com/sha256/a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2", ["civarchive.com"])
|
||||
self.assertEqual(res, {"sha256": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2"})
|
||||
self.assertIsNotNone(res)
|
||||
self.assertEqual(len(res.sha256), 64)
|
||||
self.assertEqual(res.to_dict(), {"sha256": res.sha256})
|
||||
|
||||
def test_resolve_model_category(self):
|
||||
from core.type_utils import resolve_model_category
|
||||
|
||||
+107
-73
@@ -5,6 +5,13 @@ from types import SimpleNamespace
|
||||
from unittest.mock import ANY, patch
|
||||
|
||||
from core import resolver as resolver_core
|
||||
from core.contracts import (
|
||||
ModelMatch,
|
||||
ModelReference,
|
||||
ResolvedModel,
|
||||
WorkflowAnalysisResult,
|
||||
WorkflowModelInventory,
|
||||
)
|
||||
from core.scanner import invalidate_model_files_cache, scan_directory
|
||||
from core.workflow import analysis, dynamic_widgets, references
|
||||
from core.workflow.analysis import analyze_workflow_models, identify_missing_models
|
||||
@@ -54,10 +61,10 @@ class WorkflowAnalyzerCaseSensitivityTests(unittest.TestCase):
|
||||
)
|
||||
missing = identify_missing_models(refs, available_models)
|
||||
|
||||
self.assertFalse(refs[0]["exists"])
|
||||
self.assertFalse(refs[0].exists)
|
||||
self.assertEqual(1, len(missing))
|
||||
self.assertEqual(
|
||||
r"Qwen\qwen_3_4b.safetensors", missing[0]["original_path"]
|
||||
r"Qwen\qwen_3_4b.safetensors", missing[0].reference.original_path
|
||||
)
|
||||
|
||||
def test_exact_folder_case_is_not_reported_missing(self):
|
||||
@@ -84,7 +91,7 @@ class WorkflowAnalyzerCaseSensitivityTests(unittest.TestCase):
|
||||
)
|
||||
missing = identify_missing_models(refs, available_models)
|
||||
|
||||
self.assertTrue(refs[0]["exists"])
|
||||
self.assertTrue(refs[0].exists)
|
||||
self.assertEqual([], missing)
|
||||
|
||||
|
||||
@@ -348,10 +355,10 @@ class WorkflowAnalyzerCategoryHintTests(unittest.TestCase):
|
||||
refs = analyze_workflow_models(workflow, available_models=[])
|
||||
|
||||
self.assertEqual(1, len(refs))
|
||||
self.assertEqual(3, refs[0]["widget_index"])
|
||||
self.assertEqual("start_ckpt_name", refs[0]["widget_name"])
|
||||
self.assertEqual(r"LTXV\model.safetensors", refs[0]["original_path"])
|
||||
self.assertEqual("checkpoints", refs[0]["category"])
|
||||
self.assertEqual(3, refs[0].widget_index)
|
||||
self.assertEqual("start_ckpt_name", refs[0].extra_value("widget_name"))
|
||||
self.assertEqual(r"LTXV\model.safetensors", refs[0].original_path)
|
||||
self.assertEqual("checkpoints", refs[0].category)
|
||||
|
||||
def test_implicit_seed_control_keeps_following_model_aligned(self):
|
||||
dynamic_hints = {
|
||||
@@ -415,14 +422,14 @@ class WorkflowAnalyzerCategoryHintTests(unittest.TestCase):
|
||||
refs = analyze_workflow_models(workflow, available_models=[])
|
||||
|
||||
self.assertEqual(1, len(refs))
|
||||
self.assertEqual(2, refs[0]["widget_index"])
|
||||
self.assertEqual("clip_name", refs[0]["widget_name"])
|
||||
self.assertEqual(2, refs[0].widget_index)
|
||||
self.assertEqual("clip_name", refs[0].extra_value("widget_name"))
|
||||
self.assertEqual(
|
||||
"clip-vit-large-patch14.safetensors",
|
||||
refs[0]["original_path"],
|
||||
refs[0].original_path,
|
||||
)
|
||||
self.assertEqual("clip_vision", refs[0]["category"])
|
||||
self.assertFalse(refs[0]["exists"])
|
||||
self.assertEqual("clip_vision", refs[0].category)
|
||||
self.assertFalse(refs[0].exists)
|
||||
|
||||
def test_non_model_static_choices_in_hybrid_model_dropdowns_are_skipped(self):
|
||||
cases = [
|
||||
@@ -546,7 +553,7 @@ class WorkflowAnalyzerCategoryHintTests(unittest.TestCase):
|
||||
refs = analyze_workflow_models(workflow, available_models=[])
|
||||
|
||||
self.assertEqual(1, len(refs))
|
||||
self.assertEqual("diffusion_models", refs[0]["category"])
|
||||
self.assertEqual("diffusion_models", refs[0].category)
|
||||
|
||||
def test_show_anything_cached_model_text_is_not_a_model_reference(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
@@ -605,7 +612,7 @@ class WorkflowAnalyzerCategoryHintTests(unittest.TestCase):
|
||||
refs = analyze_workflow_models(workflow, available_models=[])
|
||||
|
||||
self.assertEqual(1, len(refs))
|
||||
self.assertEqual("custom_model.safetensors", refs[0]["original_path"])
|
||||
self.assertEqual("custom_model.safetensors", refs[0].original_path)
|
||||
|
||||
def test_set_node_model_slot_name_does_not_turn_constant_into_model(self):
|
||||
workflow = {
|
||||
@@ -740,7 +747,7 @@ class WorkflowAnalyzerCategoryHintTests(unittest.TestCase):
|
||||
refs = analyze_workflow_models(workflow, available_models=[])
|
||||
|
||||
self.assertEqual(1, len(refs))
|
||||
self.assertEqual("upscale_models", refs[0]["category"])
|
||||
self.assertEqual("upscale_models", refs[0].category)
|
||||
|
||||
def test_upscale_output_hint_does_not_mark_every_text_widget_as_model(self):
|
||||
workflow = {
|
||||
@@ -758,8 +765,8 @@ class WorkflowAnalyzerCategoryHintTests(unittest.TestCase):
|
||||
refs = analyze_workflow_models(workflow, available_models=[])
|
||||
|
||||
self.assertEqual(1, len(refs))
|
||||
self.assertEqual("4x_NMKD-Siax_200k.pth", refs[0]["original_path"])
|
||||
self.assertEqual("upscale_models", refs[0]["category"])
|
||||
self.assertEqual("4x_NMKD-Siax_200k.pth", refs[0].original_path)
|
||||
self.assertEqual("upscale_models", refs[0].category)
|
||||
|
||||
def test_impact_sam_loader_uses_sams_category(self):
|
||||
workflow = {
|
||||
@@ -778,8 +785,8 @@ class WorkflowAnalyzerCategoryHintTests(unittest.TestCase):
|
||||
refs = analyze_workflow_models(workflow, available_models=[])
|
||||
|
||||
self.assertEqual(1, len(refs))
|
||||
self.assertEqual("sam_vit_b_01ec64.pth", refs[0]["original_path"])
|
||||
self.assertEqual("sams", refs[0]["category"])
|
||||
self.assertEqual("sam_vit_b_01ec64.pth", refs[0].original_path)
|
||||
self.assertEqual("sams", refs[0].category)
|
||||
|
||||
def test_impact_sam_loader_esam_option_is_not_missing_model(self):
|
||||
workflow = {
|
||||
@@ -819,8 +826,8 @@ class WorkflowAnalyzerCategoryHintTests(unittest.TestCase):
|
||||
refs = analyze_workflow_models(workflow, available_models=[])
|
||||
|
||||
self.assertEqual(1, len(refs))
|
||||
self.assertEqual("segm/person_yolov8m-seg.pt", refs[0]["original_path"])
|
||||
self.assertEqual("ultralytics", refs[0]["category"])
|
||||
self.assertEqual("segm/person_yolov8m-seg.pt", refs[0].original_path)
|
||||
self.assertEqual("ultralytics", refs[0].category)
|
||||
|
||||
def test_core_extra_loader_categories_match_comfyui_folder_paths(self):
|
||||
cases = [
|
||||
@@ -855,7 +862,7 @@ class WorkflowAnalyzerCategoryHintTests(unittest.TestCase):
|
||||
)
|
||||
|
||||
self.assertEqual(1, len(refs))
|
||||
self.assertEqual(expected_category, refs[0]["category"])
|
||||
self.assertEqual(expected_category, refs[0].category)
|
||||
|
||||
def test_multi_clip_loader_indexes_are_text_encoders(self):
|
||||
refs = analyze_workflow_models(
|
||||
@@ -878,7 +885,7 @@ class WorkflowAnalyzerCategoryHintTests(unittest.TestCase):
|
||||
)
|
||||
|
||||
self.assertEqual(4, len(refs))
|
||||
self.assertEqual({"text_encoders"}, {ref["category"] for ref in refs})
|
||||
self.assertEqual({"text_encoders"}, {ref.category for ref in refs})
|
||||
|
||||
def test_gguf_unet_loader_keeps_diffusion_badge_and_raw_folder_hint(self):
|
||||
for node_type in (
|
||||
@@ -910,9 +917,15 @@ class WorkflowAnalyzerCategoryHintTests(unittest.TestCase):
|
||||
)
|
||||
|
||||
self.assertEqual(1, len(refs))
|
||||
self.assertEqual("diffusion_models", refs[0]["category"])
|
||||
self.assertEqual(["diffusion_models"], refs[0]["category_hints"])
|
||||
self.assertEqual(["model_gguf"], refs[0]["folder_key_hints"])
|
||||
self.assertEqual("diffusion_models", refs[0].category)
|
||||
self.assertEqual(
|
||||
("diffusion_models",),
|
||||
refs[0].extra_value("category_hints"),
|
||||
)
|
||||
self.assertEqual(
|
||||
("model_gguf",),
|
||||
refs[0].extra_value("folder_key_hints"),
|
||||
)
|
||||
|
||||
def test_gguf_loader_resolves_scanner_category_alias(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
@@ -953,9 +966,9 @@ class WorkflowAnalyzerCategoryHintTests(unittest.TestCase):
|
||||
)
|
||||
|
||||
self.assertEqual(1, len(refs))
|
||||
self.assertTrue(refs[0]["exists"])
|
||||
self.assertEqual(model_path, refs[0]["full_path"])
|
||||
self.assertEqual("diffusion_models", refs[0]["category"])
|
||||
self.assertTrue(refs[0].exists)
|
||||
self.assertEqual(model_path, refs[0].extra_value("full_path"))
|
||||
self.assertEqual("diffusion_models", refs[0].category)
|
||||
|
||||
|
||||
class ScannerFolderModelTests(unittest.TestCase):
|
||||
@@ -971,8 +984,8 @@ class ScannerFolderModelTests(unittest.TestCase):
|
||||
models = scan_directory(tmpdir, {"folder"}, "diffusers")
|
||||
|
||||
self.assertEqual(1, len(models))
|
||||
self.assertEqual("wan_diffusers", models[0]["relative_path"])
|
||||
self.assertEqual(model_dir, models[0]["path"])
|
||||
self.assertEqual("wan_diffusers", models[0].relative_path)
|
||||
self.assertEqual(model_dir, models[0].path)
|
||||
|
||||
|
||||
class WorkflowCategoryHintsTests(unittest.TestCase):
|
||||
@@ -1007,8 +1020,8 @@ class WorkflowCategoryHintsTests(unittest.TestCase):
|
||||
refs = analyze_workflow_models(workflow, available_models=[])
|
||||
|
||||
self.assertEqual(1, len(refs))
|
||||
self.assertEqual("model.safetensors", refs[0]["original_path"])
|
||||
self.assertEqual("checkpoints", refs[0]["category"])
|
||||
self.assertEqual("model.safetensors", refs[0].original_path)
|
||||
self.assertEqual("checkpoints", refs[0].category)
|
||||
|
||||
def test_node_type_to_category_hints_is_populated(self):
|
||||
from core.workflow.widgets import NODE_TYPE_TO_CATEGORY_HINTS
|
||||
@@ -1186,10 +1199,12 @@ class WorkflowCustomNodeAdapterTests(unittest.TestCase):
|
||||
refs = analyze_workflow_models(workflow, available_models=[])
|
||||
|
||||
self.assertEqual(1, len(refs))
|
||||
self.assertEqual("lora-manager", refs[0]["custom_node_adapter"])
|
||||
self.assertEqual("missing_style", refs[0]["original_path"])
|
||||
self.assertEqual(0.65, refs[0]["strength"])
|
||||
self.assertTrue(refs[0]["active"])
|
||||
self.assertEqual(
|
||||
"lora-manager", refs[0].extra_value("custom_node_adapter")
|
||||
)
|
||||
self.assertEqual("missing_style", refs[0].original_path)
|
||||
self.assertEqual(0.65, refs[0].extra_value("strength"))
|
||||
self.assertTrue(refs[0].extra_value("active"))
|
||||
|
||||
|
||||
class WorkflowMissingReferenceGroupingTests(unittest.TestCase):
|
||||
@@ -1219,17 +1234,17 @@ class WorkflowMissingReferenceGroupingTests(unittest.TestCase):
|
||||
self.assertEqual(2, len(refs))
|
||||
self.assertEqual(
|
||||
["rgthree-power-lora-loader", "rgthree-power-lora-loader"],
|
||||
[ref["custom_node_adapter"] for ref in refs],
|
||||
[ref.extra_value("custom_node_adapter") for ref in refs],
|
||||
)
|
||||
self.assertEqual(1, len(missing))
|
||||
self.assertEqual(2, missing[0]["reference_count"])
|
||||
self.assertEqual(2, missing[0].reference_count)
|
||||
self.assertEqual(
|
||||
[401, 402],
|
||||
[ref["node_id"] for ref in missing[0]["all_node_refs"]],
|
||||
[ref.node_id for ref in missing[0].all_node_refs],
|
||||
)
|
||||
self.assertEqual(
|
||||
["lora", "lora"],
|
||||
[ref["nested_key"] for ref in missing[0]["all_node_refs"]],
|
||||
[ref.extra["nested_key"] for ref in missing[0].all_node_refs],
|
||||
)
|
||||
|
||||
def test_same_filename_different_categories_are_not_merged(self):
|
||||
@@ -1255,7 +1270,7 @@ class WorkflowMissingReferenceGroupingTests(unittest.TestCase):
|
||||
self.assertEqual(2, len(missing))
|
||||
self.assertEqual(
|
||||
{"checkpoints", "loras"},
|
||||
{item["category"] for item in missing},
|
||||
{item.reference.category for item in missing},
|
||||
)
|
||||
|
||||
|
||||
@@ -1263,22 +1278,26 @@ class WorkflowResolverMatchingTests(unittest.TestCase):
|
||||
def test_missing_model_matches_keep_highest_confidence_for_same_path(self):
|
||||
shared_path = os.path.join(os.getcwd(), "models", "shared.safetensors")
|
||||
matches = [
|
||||
{
|
||||
"model": {
|
||||
"path": shared_path,
|
||||
"relative_path": "models/shared.safetensors",
|
||||
},
|
||||
"filename": "shared.safetensors",
|
||||
"confidence": 0.35,
|
||||
},
|
||||
{
|
||||
"model": {
|
||||
"path": shared_path,
|
||||
"relative_path": "models/shared.safetensors",
|
||||
},
|
||||
"filename": "shared.safetensors",
|
||||
"confidence": 0.88,
|
||||
},
|
||||
ModelMatch.from_mapping(
|
||||
{
|
||||
"model": {
|
||||
"path": shared_path,
|
||||
"relative_path": "models/shared.safetensors",
|
||||
},
|
||||
"filename": "shared.safetensors",
|
||||
"confidence": 0.35,
|
||||
}
|
||||
),
|
||||
ModelMatch.from_mapping(
|
||||
{
|
||||
"model": {
|
||||
"path": shared_path,
|
||||
"relative_path": "models/shared.safetensors",
|
||||
},
|
||||
"filename": "shared.safetensors",
|
||||
"confidence": 0.88,
|
||||
}
|
||||
),
|
||||
]
|
||||
workflow_ref = {
|
||||
"node_id": 1,
|
||||
@@ -1293,10 +1312,9 @@ class WorkflowResolverMatchingTests(unittest.TestCase):
|
||||
patch.object(
|
||||
resolver_core,
|
||||
"get_workflow_model_inventory",
|
||||
return_value={
|
||||
"available_models": [],
|
||||
"model_refs": [workflow_ref],
|
||||
},
|
||||
return_value=WorkflowModelInventory(
|
||||
model_refs=(ModelReference.from_mapping(workflow_ref),)
|
||||
),
|
||||
),
|
||||
patch.object(resolver_core, "find_matches", return_value=matches),
|
||||
patch.object(resolver_core, "_get_active_downloads_by_path", return_value={}),
|
||||
@@ -1305,8 +1323,13 @@ class WorkflowResolverMatchingTests(unittest.TestCase):
|
||||
_workflow_with_model("shared.safetensors")
|
||||
)
|
||||
|
||||
self.assertIsInstance(result, WorkflowAnalysisResult)
|
||||
serialized = result.to_dict()
|
||||
self.assertEqual(
|
||||
[match["confidence"] for match in result["missing_models"][0]["matches"]],
|
||||
[
|
||||
match["confidence"]
|
||||
for match in serialized["missing_models"][0]["matches"]
|
||||
],
|
||||
[0.88],
|
||||
)
|
||||
|
||||
@@ -1335,10 +1358,13 @@ class WorkflowResolverMatchingTests(unittest.TestCase):
|
||||
patch.object(
|
||||
resolver_core,
|
||||
"get_workflow_model_inventory",
|
||||
return_value={
|
||||
"available_models": available_models,
|
||||
"model_refs": [resolved_ref],
|
||||
},
|
||||
return_value=WorkflowModelInventory(
|
||||
available_models=tuple(
|
||||
ResolvedModel.from_mapping(model)
|
||||
for model in available_models
|
||||
),
|
||||
model_refs=(ModelReference.from_mapping(resolved_ref),),
|
||||
),
|
||||
),
|
||||
patch.object(resolver_core, "find_matches") as find_matches,
|
||||
):
|
||||
@@ -1347,9 +1373,10 @@ class WorkflowResolverMatchingTests(unittest.TestCase):
|
||||
progress_callback=progress.append,
|
||||
)
|
||||
|
||||
self.assertIsInstance(result, WorkflowAnalysisResult)
|
||||
find_matches.assert_not_called()
|
||||
self.assertEqual(1, result["total_resolved"])
|
||||
self.assertEqual([], result["resolved_models"][0]["matches"])
|
||||
self.assertEqual(1, result.total_resolved)
|
||||
self.assertEqual([], result.to_dict()["resolved_models"][0]["matches"])
|
||||
self.assertFalse(
|
||||
any(
|
||||
"Analyzing resolved model" in str(update.get("message") or "")
|
||||
@@ -1384,8 +1411,15 @@ class WorkflowModelInventoryCacheTests(unittest.TestCase):
|
||||
first = get_workflow_model_inventory(workflow)
|
||||
second = get_workflow_model_inventory({"nodes": workflow["nodes"]})
|
||||
|
||||
self.assertIs(first["available_models"], available_models)
|
||||
self.assertIs(second["model_refs"], model_refs)
|
||||
self.assertEqual(
|
||||
("shared.safetensors",),
|
||||
tuple(model.filename for model in first.available_models),
|
||||
)
|
||||
self.assertIsInstance(second.model_refs[0], ModelReference)
|
||||
self.assertEqual(
|
||||
[ref.to_dict() for ref in first.model_refs],
|
||||
[ref.to_dict() for ref in second.model_refs],
|
||||
)
|
||||
get_models.assert_called_once_with(force_rescan=False)
|
||||
analyze_models.assert_called_once_with(
|
||||
workflow,
|
||||
@@ -1503,7 +1537,7 @@ class WorkflowModelInventoryCacheTests(unittest.TestCase):
|
||||
self.assertEqual(3, get_node_info.call_count)
|
||||
self.assertEqual(
|
||||
{"second.safetensors", "unchanged.safetensors"},
|
||||
{ref["original_path"] for ref in result["model_refs"]},
|
||||
{ref.original_path for ref in result.model_refs},
|
||||
)
|
||||
|
||||
def test_changed_loader_keeps_previous_result_position(self):
|
||||
@@ -1555,7 +1589,7 @@ class WorkflowModelInventoryCacheTests(unittest.TestCase):
|
||||
"middle-updated.safetensors",
|
||||
"last.safetensors",
|
||||
],
|
||||
[ref["original_path"] for ref in result["model_refs"]],
|
||||
[ref.original_path for ref in result.model_refs],
|
||||
)
|
||||
|
||||
def test_new_loader_does_not_reanalyze_existing_loader(self):
|
||||
|
||||
@@ -2,6 +2,7 @@ from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from core import resolver as resolver_core
|
||||
from core.contracts import ModelReference, Resolution
|
||||
from core.workflow import analysis, references, subgraphs, traversal
|
||||
from core.workflow_updater import update_workflow_nodes
|
||||
|
||||
@@ -274,7 +275,9 @@ def test_apply_instance_promoted_context_resolves_existing_model(tmp_path):
|
||||
"path": str(model_path),
|
||||
}
|
||||
]
|
||||
reference = {"original_path": "promoted.safetensors", "exists": False}
|
||||
reference = ModelReference.from_mapping(
|
||||
{"original_path": "promoted.safetensors", "exists": False}
|
||||
)
|
||||
context = {
|
||||
"proxy_widget_name": "ckpt_name",
|
||||
"node_id": 10,
|
||||
@@ -285,16 +288,16 @@ def test_apply_instance_promoted_context_resolves_existing_model(tmp_path):
|
||||
"category": "checkpoints",
|
||||
}
|
||||
|
||||
subgraphs._apply_instance_promoted_widget_context(
|
||||
reference = subgraphs._apply_instance_promoted_widget_context(
|
||||
reference,
|
||||
context,
|
||||
available_models,
|
||||
)
|
||||
|
||||
assert reference["promoted_inner_node_id"] == 10
|
||||
assert reference["category"] == "checkpoints"
|
||||
assert reference["full_path"] == str(model_path)
|
||||
assert reference["exists"] is True
|
||||
assert reference.extra_value("promoted_inner_node_id") == 10
|
||||
assert reference.category == "checkpoints"
|
||||
assert reference.extra_value("full_path") == str(model_path)
|
||||
assert reference.exists is True
|
||||
|
||||
|
||||
def test_apply_promoted_locator_supports_name_fallback_and_value_selection():
|
||||
@@ -307,20 +310,22 @@ def test_apply_promoted_locator_supports_name_fallback_and_value_selection():
|
||||
]
|
||||
},
|
||||
}
|
||||
reference = {
|
||||
"subgraph_id": "subgraph-1",
|
||||
"node_id": 10,
|
||||
"widget_index": 99,
|
||||
"widget_name": "ckpt_name",
|
||||
"original_path": "second",
|
||||
}
|
||||
reference = ModelReference.from_mapping(
|
||||
{
|
||||
"subgraph_id": "subgraph-1",
|
||||
"node_id": 10,
|
||||
"widget_index": 99,
|
||||
"widget_name": "ckpt_name",
|
||||
"original_path": "second",
|
||||
}
|
||||
)
|
||||
|
||||
subgraphs._apply_promoted_widget_locator(reference, contexts)
|
||||
reference = subgraphs._apply_promoted_widget_locator(reference, contexts)
|
||||
|
||||
assert reference["locate_node_id"] == 2
|
||||
assert reference["locate_subgraph_name"] == "Two"
|
||||
assert reference["locate_is_top_level"] is True
|
||||
assert reference["locate_via_promoted_widget"] is True
|
||||
assert reference.extra_value("locate_node_id") == 2
|
||||
assert reference.extra_value("locate_subgraph_name") == "Two"
|
||||
assert reference.extra_value("locate_is_top_level") is True
|
||||
assert reference.extra_value("locate_via_promoted_widget") is True
|
||||
|
||||
|
||||
def test_subgraph_input_reference_targets_instance_for_update_and_location():
|
||||
@@ -377,14 +382,14 @@ def test_subgraph_input_reference_targets_instance_for_update_and_location():
|
||||
refs = analysis.analyze_workflow_models(workflow, available_models=[])
|
||||
|
||||
assert len(refs) == 1
|
||||
assert refs[0]["node_id"] == 1
|
||||
assert refs[0]["widget_index"] == 0
|
||||
assert refs[0]["subgraph_id"] == "subgraph-1"
|
||||
assert refs[0]["is_top_level"] is True
|
||||
assert refs[0]["subgraph_path"] is None
|
||||
assert refs[0]["locate_node_id"] == 1
|
||||
assert refs[0]["locate_subgraph_id"] == ""
|
||||
assert refs[0]["promoted_inner_node_id"] == 10
|
||||
assert refs[0].node_id == 1
|
||||
assert refs[0].widget_index == 0
|
||||
assert refs[0].subgraph_id == "subgraph-1"
|
||||
assert refs[0].is_top_level is True
|
||||
assert refs[0].extra_value("subgraph_path") is None
|
||||
assert refs[0].extra_value("locate_node_id") == 1
|
||||
assert refs[0].extra_value("locate_subgraph_id") == ""
|
||||
assert refs[0].extra_value("promoted_inner_node_id") == 10
|
||||
|
||||
no_proxy_workflow = {
|
||||
**workflow,
|
||||
@@ -398,23 +403,21 @@ def test_subgraph_input_reference_targets_instance_for_update_and_location():
|
||||
no_proxy_refs = analysis.analyze_workflow_models(
|
||||
no_proxy_workflow, available_models=[]
|
||||
)
|
||||
assert no_proxy_refs[0]["node_id"] == 1
|
||||
assert no_proxy_refs[0]["promoted_inner_node_id"] == 10
|
||||
assert no_proxy_refs[0].node_id == 1
|
||||
assert no_proxy_refs[0].extra_value("promoted_inner_node_id") == 10
|
||||
|
||||
update_workflow_nodes(
|
||||
workflow,
|
||||
[
|
||||
{
|
||||
"node_id": refs[0]["node_id"],
|
||||
"widget_index": refs[0]["widget_index"],
|
||||
"resolved_path": "replacement.safetensors",
|
||||
"category": "checkpoints",
|
||||
"subgraph_id": refs[0]["subgraph_id"],
|
||||
"is_top_level": refs[0]["is_top_level"],
|
||||
"promoted_widget_name": refs[0]["promoted_widget_name"],
|
||||
}
|
||||
],
|
||||
resolution = Resolution.from_mapping(
|
||||
{
|
||||
"node_id": refs[0].node_id,
|
||||
"widget_index": refs[0].widget_index,
|
||||
"resolved_path": "replacement.safetensors",
|
||||
"category": "checkpoints",
|
||||
"subgraph_id": refs[0].subgraph_id,
|
||||
"is_top_level": refs[0].is_top_level,
|
||||
"promoted_widget_name": refs[0].extra_value("promoted_widget_name"),
|
||||
}
|
||||
)
|
||||
update_workflow_nodes(workflow, [resolution])
|
||||
|
||||
assert workflow["nodes"][0]["widgets_values"] == ["replacement.safetensors"]
|
||||
assert workflow["definitions"]["subgraphs"][0]["nodes"][0]["widgets_values"] == [
|
||||
@@ -479,9 +482,9 @@ def test_reused_subgraph_instance_refreshes_promoted_model_context():
|
||||
)
|
||||
|
||||
assert len(refs) == 1
|
||||
assert refs[0]["node_id"] == 1
|
||||
assert refs[0]["original_path"] == "new.safetensors"
|
||||
assert refs[0]["promoted_inner_node_id"] == 10
|
||||
assert refs[0].node_id == 1
|
||||
assert refs[0].original_path == "new.safetensors"
|
||||
assert refs[0].extra_value("promoted_inner_node_id") == 10
|
||||
|
||||
|
||||
def test_reference_matching_handles_placeholders_paths_and_scanner_records(tmp_path):
|
||||
|
||||
+155
-25
@@ -7,6 +7,14 @@ from unittest.mock import AsyncMock, MagicMock
|
||||
import pytest
|
||||
from aiohttp import web
|
||||
|
||||
from core.contracts import (
|
||||
MissingModel,
|
||||
ModelCatalogEntry,
|
||||
ModelMatch,
|
||||
Resolution,
|
||||
SearchResult,
|
||||
WorkflowAnalysisResult,
|
||||
)
|
||||
from core.path_utils import get_filename_from_path
|
||||
from core.routes.context import RouteContext
|
||||
from core.routes.workflow_analysis import register_workflow_analysis_routes
|
||||
@@ -40,6 +48,21 @@ def _request(payload):
|
||||
return SimpleNamespace(json=AsyncMock(return_value=payload))
|
||||
|
||||
|
||||
def _analysis_result(missing_models=(), **counts):
|
||||
typed_missing = tuple(
|
||||
item
|
||||
if isinstance(item, MissingModel)
|
||||
else MissingModel.from_mapping(item)
|
||||
for item in missing_models
|
||||
)
|
||||
return WorkflowAnalysisResult(
|
||||
missing_models=typed_missing,
|
||||
total_resolved=counts.get("total_resolved", 0),
|
||||
total_missing=counts.get("total_missing", len(typed_missing)),
|
||||
total_models_analyzed=counts.get("total_models_analyzed", len(typed_missing)),
|
||||
)
|
||||
|
||||
|
||||
def _build_routes(overrides=None):
|
||||
routes = _Routes()
|
||||
extension = SimpleNamespace(
|
||||
@@ -49,7 +72,7 @@ def _build_routes(overrides=None):
|
||||
)
|
||||
values = {
|
||||
"analyze_and_find_matches": MagicMock(
|
||||
return_value={"missing_models": [], "total_missing": 0}
|
||||
return_value=WorkflowAnalysisResult()
|
||||
),
|
||||
"apply_resolution": MagicMock(side_effect=lambda workflow, _resolutions: workflow),
|
||||
"asyncio": asyncio,
|
||||
@@ -94,6 +117,20 @@ async def test_analyze_route_validates_workflow_payload(payload, message):
|
||||
assert json.loads(response.text) == {"error": message}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_analyze_route_rejects_non_text_analysis_id():
|
||||
handlers, values = _build_routes()
|
||||
response = await handlers[("POST", "/model_resolver/analyze")](
|
||||
_request({"workflow": {"nodes": []}, "analysis_id": 123})
|
||||
)
|
||||
|
||||
assert response.status == 400
|
||||
assert "Workflow analysis request analysis_id must be a string" in (
|
||||
json.loads(response.text)["error"]
|
||||
)
|
||||
values["analyze_and_find_matches"].assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_analyze_route_filters_references_and_uses_local_download_sources():
|
||||
missing_models = [
|
||||
@@ -114,7 +151,16 @@ async def test_analyze_route_filters_references_and_uses_local_download_sources(
|
||||
},
|
||||
{
|
||||
"original_path": "already-local.safetensors",
|
||||
"matches": [{"confidence": 100}],
|
||||
"matches": [
|
||||
{
|
||||
"model": {
|
||||
"path": "models/already-local.safetensors",
|
||||
"filename": "already-local.safetensors",
|
||||
"category": "checkpoints",
|
||||
},
|
||||
"confidence": 100,
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"original_path": "installed-custom-node.safetensors",
|
||||
@@ -124,33 +170,39 @@ async def test_analyze_route_filters_references_and_uses_local_download_sources(
|
||||
|
||||
def analyze(_workflow, _threshold, _limit, progress_callback, **_kwargs):
|
||||
progress_callback({"stage": "matching", "current": 1, "total": 5})
|
||||
return {"missing_models": missing_models}
|
||||
return _analysis_result(missing_models)
|
||||
|
||||
def popular_url(filename):
|
||||
if filename == "popular.safetensors":
|
||||
return {
|
||||
"url": "https://example.com/popular.safetensors",
|
||||
"type": "checkpoint",
|
||||
"directory": "checkpoints",
|
||||
"size": 100,
|
||||
}
|
||||
return ModelCatalogEntry(
|
||||
filename=filename,
|
||||
url="https://example.com/popular.safetensors",
|
||||
model_type="checkpoint",
|
||||
directory="checkpoints",
|
||||
size=100,
|
||||
)
|
||||
return None
|
||||
|
||||
def model_list(filename, exact_only=False):
|
||||
assert exact_only is True
|
||||
if filename == "popular.safetensors":
|
||||
return {"size": 200}
|
||||
return SearchResult(
|
||||
source="model_list",
|
||||
filename=filename,
|
||||
size=200,
|
||||
)
|
||||
if filename == "listed.safetensors":
|
||||
return {
|
||||
"url": "https://example.com/listed.safetensors",
|
||||
"filename": "listed.safetensors",
|
||||
"name": "Listed model",
|
||||
"type": "checkpoint",
|
||||
"directory": "checkpoints",
|
||||
"size": 300,
|
||||
"match_type": "exact",
|
||||
"confidence": 100,
|
||||
}
|
||||
return SearchResult(
|
||||
source="model_list",
|
||||
url="https://example.com/listed.safetensors",
|
||||
filename="listed.safetensors",
|
||||
name="Listed model",
|
||||
model_type="checkpoint",
|
||||
size=300,
|
||||
match_type="exact",
|
||||
confidence=100,
|
||||
extra={"directory": "checkpoints"},
|
||||
)
|
||||
return None
|
||||
|
||||
handlers, values = _build_routes(
|
||||
@@ -160,7 +212,7 @@ async def test_analyze_route_filters_references_and_uses_local_download_sources(
|
||||
"get_popular_model_url": MagicMock(side_effect=popular_url),
|
||||
"search_model_list": MagicMock(side_effect=model_list),
|
||||
"should_skip_existing_custom_node_reference": MagicMock(
|
||||
side_effect=lambda item: item["original_path"].startswith(
|
||||
side_effect=lambda item: item.original_path.startswith(
|
||||
"installed-custom"
|
||||
)
|
||||
),
|
||||
@@ -244,7 +296,14 @@ async def test_workflow_resolution_route_returns_updated_workflow():
|
||||
_request(
|
||||
{
|
||||
"workflow": workflow,
|
||||
"resolutions": [{"node_id": 1, "widget_index": 0}],
|
||||
"resolutions": [
|
||||
{
|
||||
"node_id": 1,
|
||||
"widget_index": 0,
|
||||
"resolved_path": "resolved.safetensors",
|
||||
"category": "checkpoints",
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
)
|
||||
@@ -252,13 +311,68 @@ async def test_workflow_resolution_route_returns_updated_workflow():
|
||||
assert response.status == 200
|
||||
assert json.loads(response.text) == {"workflow": updated, "success": True}
|
||||
values["apply_resolution"].assert_called_once_with(
|
||||
workflow, [{"node_id": 1, "widget_index": 0}]
|
||||
workflow,
|
||||
[
|
||||
Resolution.from_mapping(
|
||||
{
|
||||
"node_id": 1,
|
||||
"widget_index": 0,
|
||||
"resolved_path": "resolved.safetensors",
|
||||
"category": "checkpoints",
|
||||
}
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"resolution",
|
||||
[
|
||||
{
|
||||
"node_id": 1,
|
||||
"widget_index": "0",
|
||||
"resolved_path": "resolved.safetensors",
|
||||
},
|
||||
{
|
||||
"node_id": 1,
|
||||
"widget_index": -1,
|
||||
"resolved_path": "resolved.safetensors",
|
||||
},
|
||||
{
|
||||
"node_id": 1,
|
||||
"widget_index": 0,
|
||||
"resolved_path": 123,
|
||||
},
|
||||
],
|
||||
)
|
||||
async def test_workflow_resolution_route_rejects_invalid_resolution_types(resolution):
|
||||
handlers, values = _build_routes()
|
||||
response = await handlers[("POST", "/model_resolver/resolve")](
|
||||
_request(
|
||||
{
|
||||
"workflow": {"nodes": []},
|
||||
"resolutions": [resolution],
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
assert response.status == 400
|
||||
assert json.loads(response.text)["error"].startswith("Invalid resolution:")
|
||||
values["apply_resolution"].assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_local_matches_route_rescans_and_passes_search_options():
|
||||
matches = [{"path": os.path.join("models", "model.safetensors")}]
|
||||
matches = [
|
||||
ModelMatch.from_mapping(
|
||||
{
|
||||
"model": {
|
||||
"path": os.path.join("models", "model.safetensors")
|
||||
}
|
||||
}
|
||||
)
|
||||
]
|
||||
handlers, values = _build_routes(
|
||||
{"search_local_matches": MagicMock(return_value=matches)}
|
||||
)
|
||||
@@ -273,7 +387,9 @@ async def test_local_matches_route_rescans_and_passes_search_options():
|
||||
)
|
||||
|
||||
assert response.status == 200
|
||||
assert json.loads(response.text) == {"matches": matches}
|
||||
assert json.loads(response.text) == {
|
||||
"matches": [match.to_dict() for match in matches]
|
||||
}
|
||||
values["invalidate_local_hash_match_cache"].assert_called_once_with()
|
||||
values["search_local_matches"].assert_called_once_with(
|
||||
"model.safetensors",
|
||||
@@ -284,6 +400,20 @@ async def test_local_matches_route_rescans_and_passes_search_options():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_local_matches_route_rejects_non_text_fields():
|
||||
handlers, values = _build_routes()
|
||||
response = await handlers[("POST", "/model_resolver/local-matches")](
|
||||
_request({"filename": 123, "category": "checkpoints"})
|
||||
)
|
||||
|
||||
assert response.status == 400
|
||||
assert json.loads(response.text) == {
|
||||
"error": "Local matches request filename must be a string"
|
||||
}
|
||||
values["search_local_matches"].assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_workflow_progress_route_delegates_analysis_tracker():
|
||||
handlers, values = _build_routes()
|
||||
|
||||
Reference in New Issue
Block a user