Improve symlink/junction handling and add RapidFuzz for faster fuzzy matching

- Add rapidfuzz as optional dependency with SequenceMatcher fallback
- Use stable path identity (realpath) for deduplication in scanner and resolver
- Prevent duplicate scans and duplicate model entries across symlinks/junctions
- Improve matching performance with RapidFuzz when available

core/matcher.py

@@ -12,6 +12,11 @@ from difflib import SequenceMatcher
 
 from .log_system.log_funcs import log_debug, log_info, log_warn, log_error, log_exception
 
+try:
+    from rapidfuzz import fuzz as rapidfuzz_fuzz
+except ImportError:
+    rapidfuzz_fuzz = None
+
 
 def normalize_filename(filename: str) -> str:
     """
@@ -45,7 +50,7 @@ def calculate_similarity(str1: str, str2: str) -> float:
     """
     Calculate similarity score between two strings (0.0 to 1.0).
 
-    Uses SequenceMatcher to compute a ratio.
+    Uses RapidFuzz when available, with SequenceMatcher as a fallback.
 
     Args:
         str1: First string
@@ -54,6 +59,9 @@ def calculate_similarity(str1: str, str2: str) -> float:
     Returns:
         Similarity score from 0.0 (completely different) to 1.0 (identical)
     """
+    if rapidfuzz_fuzz is not None:
+        return rapidfuzz_fuzz.ratio(str1, str2) / 100.0
+
     return SequenceMatcher(None, str1, str2).ratio()
 
 
@@ -190,7 +198,7 @@ def find_matches(
             # Exact match after normalization = 100% confidence
             similarity = 1.0
         else:
-            # Calculate similarity using SequenceMatcher
+            # Calculate similarity using the configured fuzzy matcher
             # This gives a ratio between 0.0 and 1.0 based on longest common subsequence
             similarity = calculate_similarity_with_normalization(
                 target_filename, candidate_filename
@@ -207,7 +215,7 @@ def find_matches(
             similarity = max(similarity, similarity_no_ext)
 
             # Cap similarity at 0.999 for non-exact matches to prevent false 100% scores
-            # SequenceMatcher can sometimes give 1.0 for very similar but not identical strings
+            # Fuzzy matchers can sometimes give 1.0 for very similar but not identical strings
             # due to normalization artifacts
             if similarity >= 0.999 and target_norm != candidate_norm:
                 similarity = 0.999

core/resolver.py

@@ -34,6 +34,17 @@ URL_PATTERN = re.compile(r'(https?://(?:huggingface\.co|civitai\.com)[^\s"\'<>\)
 MODEL_EXTENSIONS = (".safetensors", ".ckpt", ".pt", ".pth", ".bin", ".onnx", ".gguf")
 
 
+def get_path_identity(path: str) -> str:
+    """Return a stable path identity across symlinks/junctions."""
+    if not path:
+        return ""
+
+    try:
+        return os.path.normcase(os.path.realpath(os.path.abspath(path)))
+    except (OSError, ValueError):
+        return os.path.normcase(os.path.abspath(path))
+
+
 def strip_known_model_extension(filename: str) -> str:
     """Strip only known model extensions, preserving names like v4.0."""
     if not isinstance(filename, str):
@@ -132,18 +143,20 @@ def search_local_matches(
     for match in matches:
         model_dict = match["model"]
         absolute_path = model_dict.get("path", "")
-        if absolute_path:
-            absolute_path = os.path.normpath(absolute_path)
+        path_identity = get_path_identity(absolute_path) if absolute_path else ""
+        dedupe_key = path_identity or os.path.normcase(
+            model_dict.get("relative_path", "") or match.get("filename", "")
+        )
 
-        if absolute_path not in seen_absolute_paths:
-            seen_absolute_paths[absolute_path] = match
+        if dedupe_key not in seen_absolute_paths:
+            seen_absolute_paths[dedupe_key] = match
             deduplicated_matches.append(match)
         else:
-            existing_match = seen_absolute_paths[absolute_path]
+            existing_match = seen_absolute_paths[dedupe_key]
             if match["confidence"] > existing_match["confidence"]:
                 idx = deduplicated_matches.index(existing_match)
                 deduplicated_matches[idx] = match
-                seen_absolute_paths[absolute_path] = match
+                seen_absolute_paths[dedupe_key] = match
 
     return deduplicated_matches
 
@@ -532,18 +545,20 @@ def analyze_and_find_matches(
         for match in matches:
             model_dict = match["model"]
             absolute_path = model_dict.get("path", "")
-            if absolute_path:
-                absolute_path = os.path.normpath(absolute_path)
+            path_identity = get_path_identity(absolute_path) if absolute_path else ""
+            dedupe_key = path_identity or os.path.normcase(
+                model_dict.get("relative_path", "") or match.get("filename", "")
+            )
 
-            if absolute_path not in seen_absolute_paths:
-                seen_absolute_paths[absolute_path] = match
+            if dedupe_key not in seen_absolute_paths:
+                seen_absolute_paths[dedupe_key] = match
                 deduplicated_matches.append(match)
             else:
-                existing_match = seen_absolute_paths[absolute_path]
+                existing_match = seen_absolute_paths[dedupe_key]
                 if match["confidence"] > existing_match["confidence"]:
                     idx = deduplicated_matches.index(existing_match)
                     deduplicated_matches[idx] = match
-                    seen_absolute_paths[absolute_path] = match
+                    seen_absolute_paths[dedupe_key] = match
 
         return deduplicated_matches
 

core/scanner.py

@@ -43,9 +43,25 @@ _MODEL_FILES_CACHE_AT: float = 0.0
 _MODEL_FILES_CACHE_TTL_SECONDS = 2.0
 
 
+def _path_identity(path: str) -> str:
+    """Return a stable identity for dedupe across symlinks/junctions."""
+    return os.path.normcase(os.path.realpath(os.path.abspath(path)))
+
+
 def _directory_identity(path: str) -> str:
     """Return a stable identity for loop detection across symlinks/junctions."""
-    return os.path.normcase(os.path.realpath(os.path.abspath(path)))
+    return _path_identity(path)
+
+
+def _model_identity(model: Dict[str, str]) -> str:
+    path = model.get("path", "")
+    if not path:
+        return ""
+
+    try:
+        return _path_identity(path)
+    except (OSError, ValueError):
+        return os.path.normcase(os.path.abspath(path))
 
 
 def get_model_directories() -> Dict[str, Tuple[List[str], set]]:
@@ -206,6 +222,8 @@ def scan_all_directories() -> List[Dict[str, str]]:
     """
     all_models = []
     directories = get_model_directories()
+    seen_scan_roots = set()
+    seen_models = set()
 
     for category, value in directories.items():
         # Skip categories that aren't typically model directories
@@ -245,8 +263,25 @@ def scan_all_directories() -> List[Dict[str, str]]:
 
         for directory_path in paths:
             try:
+                try:
+                    root_identity = _directory_identity(directory_path)
+                except (OSError, ValueError):
+                    root_identity = os.path.normcase(os.path.abspath(directory_path))
+
+                root_key = (category, root_identity)
+                if root_key in seen_scan_roots:
+                    continue
+                seen_scan_roots.add(root_key)
+
                 models = scan_directory(directory_path, extensions, category)
-                all_models.extend(models)
+                for model in models:
+                    identity = _model_identity(model)
+                    model_key = (model.get("category", category), identity)
+                    if identity and model_key in seen_models:
+                        continue
+                    if identity:
+                        seen_models.add(model_key)
+                    all_models.append(model)
                 #log_debug(f"Found {len(models)} models in {category}/{directory_path}")
             except Exception as e:
                 log_warn(f"Error scanning {category} directory {directory_path}: {e}")

requirements.txt

@@ -1,2 +1,3 @@
 requests
 aiohttp
+rapidfuzz