diff --git a/core/matcher.py b/core/matcher.py index b551a60..240fdc9 100644 --- a/core/matcher.py +++ b/core/matcher.py @@ -12,6 +12,11 @@ from difflib import SequenceMatcher from .log_system.log_funcs import log_debug, log_info, log_warn, log_error, log_exception +try: + from rapidfuzz import fuzz as rapidfuzz_fuzz +except ImportError: + rapidfuzz_fuzz = None + def normalize_filename(filename: str) -> str: """ @@ -45,7 +50,7 @@ def calculate_similarity(str1: str, str2: str) -> float: """ Calculate similarity score between two strings (0.0 to 1.0). - Uses SequenceMatcher to compute a ratio. + Uses RapidFuzz when available, with SequenceMatcher as a fallback. Args: str1: First string @@ -54,6 +59,9 @@ def calculate_similarity(str1: str, str2: str) -> float: Returns: Similarity score from 0.0 (completely different) to 1.0 (identical) """ + if rapidfuzz_fuzz is not None: + return rapidfuzz_fuzz.ratio(str1, str2) / 100.0 + return SequenceMatcher(None, str1, str2).ratio() @@ -190,7 +198,7 @@ def find_matches( # Exact match after normalization = 100% confidence similarity = 1.0 else: - # Calculate similarity using SequenceMatcher + # Calculate similarity using the configured fuzzy matcher # This gives a ratio between 0.0 and 1.0 based on longest common subsequence similarity = calculate_similarity_with_normalization( target_filename, candidate_filename @@ -207,7 +215,7 @@ def find_matches( similarity = max(similarity, similarity_no_ext) # Cap similarity at 0.999 for non-exact matches to prevent false 100% scores - # SequenceMatcher can sometimes give 1.0 for very similar but not identical strings + # Fuzzy matchers can sometimes give 1.0 for very similar but not identical strings # due to normalization artifacts if similarity >= 0.999 and target_norm != candidate_norm: similarity = 0.999 diff --git a/core/resolver.py b/core/resolver.py index 485e12a..64357b1 100644 --- a/core/resolver.py +++ b/core/resolver.py @@ -34,6 +34,17 @@ URL_PATTERN = re.compile(r'(https?://(?:huggingface\.co|civitai\.com)[^\s"\'<>\) MODEL_EXTENSIONS = (".safetensors", ".ckpt", ".pt", ".pth", ".bin", ".onnx", ".gguf") +def get_path_identity(path: str) -> str: + """Return a stable path identity across symlinks/junctions.""" + if not path: + return "" + + try: + return os.path.normcase(os.path.realpath(os.path.abspath(path))) + except (OSError, ValueError): + return os.path.normcase(os.path.abspath(path)) + + def strip_known_model_extension(filename: str) -> str: """Strip only known model extensions, preserving names like v4.0.""" if not isinstance(filename, str): @@ -132,18 +143,20 @@ def search_local_matches( for match in matches: model_dict = match["model"] absolute_path = model_dict.get("path", "") - if absolute_path: - absolute_path = os.path.normpath(absolute_path) + path_identity = get_path_identity(absolute_path) if absolute_path else "" + dedupe_key = path_identity or os.path.normcase( + model_dict.get("relative_path", "") or match.get("filename", "") + ) - if absolute_path not in seen_absolute_paths: - seen_absolute_paths[absolute_path] = match + if dedupe_key not in seen_absolute_paths: + seen_absolute_paths[dedupe_key] = match deduplicated_matches.append(match) else: - existing_match = seen_absolute_paths[absolute_path] + existing_match = seen_absolute_paths[dedupe_key] if match["confidence"] > existing_match["confidence"]: idx = deduplicated_matches.index(existing_match) deduplicated_matches[idx] = match - seen_absolute_paths[absolute_path] = match + seen_absolute_paths[dedupe_key] = match return deduplicated_matches @@ -532,18 +545,20 @@ def analyze_and_find_matches( for match in matches: model_dict = match["model"] absolute_path = model_dict.get("path", "") - if absolute_path: - absolute_path = os.path.normpath(absolute_path) + path_identity = get_path_identity(absolute_path) if absolute_path else "" + dedupe_key = path_identity or os.path.normcase( + model_dict.get("relative_path", "") or match.get("filename", "") + ) - if absolute_path not in seen_absolute_paths: - seen_absolute_paths[absolute_path] = match + if dedupe_key not in seen_absolute_paths: + seen_absolute_paths[dedupe_key] = match deduplicated_matches.append(match) else: - existing_match = seen_absolute_paths[absolute_path] + existing_match = seen_absolute_paths[dedupe_key] if match["confidence"] > existing_match["confidence"]: idx = deduplicated_matches.index(existing_match) deduplicated_matches[idx] = match - seen_absolute_paths[absolute_path] = match + seen_absolute_paths[dedupe_key] = match return deduplicated_matches diff --git a/core/scanner.py b/core/scanner.py index dbd639c..dbd6418 100644 --- a/core/scanner.py +++ b/core/scanner.py @@ -43,9 +43,25 @@ _MODEL_FILES_CACHE_AT: float = 0.0 _MODEL_FILES_CACHE_TTL_SECONDS = 2.0 +def _path_identity(path: str) -> str: + """Return a stable identity for dedupe across symlinks/junctions.""" + return os.path.normcase(os.path.realpath(os.path.abspath(path))) + + def _directory_identity(path: str) -> str: """Return a stable identity for loop detection across symlinks/junctions.""" - return os.path.normcase(os.path.realpath(os.path.abspath(path))) + return _path_identity(path) + + +def _model_identity(model: Dict[str, str]) -> str: + path = model.get("path", "") + if not path: + return "" + + try: + return _path_identity(path) + except (OSError, ValueError): + return os.path.normcase(os.path.abspath(path)) def get_model_directories() -> Dict[str, Tuple[List[str], set]]: @@ -206,6 +222,8 @@ def scan_all_directories() -> List[Dict[str, str]]: """ all_models = [] directories = get_model_directories() + seen_scan_roots = set() + seen_models = set() for category, value in directories.items(): # Skip categories that aren't typically model directories @@ -245,8 +263,25 @@ def scan_all_directories() -> List[Dict[str, str]]: for directory_path in paths: try: + try: + root_identity = _directory_identity(directory_path) + except (OSError, ValueError): + root_identity = os.path.normcase(os.path.abspath(directory_path)) + + root_key = (category, root_identity) + if root_key in seen_scan_roots: + continue + seen_scan_roots.add(root_key) + models = scan_directory(directory_path, extensions, category) - all_models.extend(models) + for model in models: + identity = _model_identity(model) + model_key = (model.get("category", category), identity) + if identity and model_key in seen_models: + continue + if identity: + seen_models.add(model_key) + all_models.append(model) #log_debug(f"Found {len(models)} models in {category}/{directory_path}") except Exception as e: log_warn(f"Error scanning {category} directory {directory_path}: {e}") diff --git a/requirements.txt b/requirements.txt index 4cdb2b6..8b13721 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ requests aiohttp +rapidfuzz