From ab4bdd41ea862b34ccf1aabb67cfe9d5bfba84f5 Mon Sep 17 00:00:00 2001 From: Ewald ALOEBOETOE <129623145+MajoorWaldi@users.noreply.github.com> Date: Wed, 4 Nov 2026 09:16:16 +0000 Subject: [PATCH] Match single-word searches anywhere in the file name Typing "F" only found file names with a word starting with F (FTS5 prefix match), so the grid did not list e.g. sunflower.png or ComfyUI_00001_.png. Single-word queries now add a filename LIKE '%text%' branch to the global and scoped FTS queries and their counts, ranked after FTS hits, with LIKE wildcards escaped. Multi-word and quoted queries stay pure FTS. --- CHANGELOG.md | 1 + mjr_am_backend/features/index/searcher.py | 74 ++++++++++++++++++++--- tests/features/test_search_substring.py | 63 +++++++++++++++++++ 3 files changed, 130 insertions(+), 8 deletions(-) create mode 100644 tests/features/test_search_substring.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a12c8ca0..2a6e2fa7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ All notable changes to this project are documented in this file. - **`mjr_am_backend.adapters.tools` missing from Registry installs**: `.comfyignore`'s unanchored `tools/` pattern also matched `mjr_am_backend/adapters/tools/`, stripping the ExifTool/FFProbe adapters out of every Registry-published package and causing `ModuleNotFoundError: No module named 'mjr_am_backend.adapters.tools'` at startup. The pattern is now anchored to the repo-root `/tools/` dev-scripts folder only. ### Fixed (follow-ups) +- **Search by part of a file name**: typing `F` (or `flow`) only found names with a *word starting* with it (`Flux…`, `field…`), never `sunflower.png` or `ComfyUI_…`. Single-word searches now also list file names that contain the text (ranked after word matches); `_` and `%` are matched literally. Multi-word and quoted searches are unchanged. - **Sidecar JSON added after a file was indexed** is now picked up: the watcher maps a new/changed `.json` / `.json` to its media, invalidates that file's metadata cache and scan journal, and forces a re-extraction (a non-incremental re-index no longer trusts an existing metadata row when the cache was invalidated). - **Duplicate index rows for one file (Windows)**: rows whose paths differ only by case or separator (left by older versions) are merged at startup, keeping the highest rating, all tags, the stack cover, execution ids and the embedding. - **Generated feed** drops items deleted from the UI or removed on disk. diff --git a/mjr_am_backend/features/index/searcher.py b/mjr_am_backend/features/index/searcher.py index 578c7f93..5b193643 100644 --- a/mjr_am_backend/features/index/searcher.py +++ b/mjr_am_backend/features/index/searcher.py @@ -128,6 +128,44 @@ def _strip_diacritics(text: str) -> str: _SEARCH_TOKEN_RE = re.compile(r"\w+", re.UNICODE) +# Rank given to rows that only match by substring: worse than any bm25 score +# (bm25 is negative, lower is better), so word/prefix hits stay on top. +_SUBSTRING_MATCH_RANK = 50.0 + + +def _filename_substring_pattern(query: str) -> str | None: + """``LIKE`` pattern for "file name contains " (single-word queries). + + FTS5 only matches words that *start* with the query: typing ``f`` found + ``field.png`` but not ``sunflower.png`` or ``ComfyUI_00001_.png``. + Single-word searches therefore also match file names containing the text. + Multi-word and quoted searches stay pure FTS. + """ + text = str(query or "").strip() + if not text or text == "*" or any(ch.isspace() for ch in text) or text[0] in "\"'": + return None + escaped = text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + return f"%{escaped}%" + + +def _substring_matches_sql(pattern: str | None, *, with_rank: bool) -> str: + if not pattern: + return "" + if with_rank: + return ( + "\n UNION ALL\n\n" + f" SELECT id AS asset_id, {_SUBSTRING_MATCH_RANK} AS rank\n" + " FROM assets\n" + " WHERE filename LIKE ? ESCAPE '\\'\n" + ) + return ( + "\n UNION\n\n" + " SELECT id AS asset_id\n" + " FROM assets\n" + " WHERE filename LIKE ? ESCAPE '\\'\n" + ) + + def _is_browse_query(query: str) -> bool: """``*`` or input without any word character (a lone quote while typing a phrase, "--", "()"): nothing to match, so browse instead of erroring.""" @@ -1245,13 +1283,16 @@ class IndexSearcher: filters: dict[str, Any] | None, include_total: bool, metadata_tags_text_clause: str, + substring_pattern: str | None = None, ) -> Result[dict[str, Any]]: # Never use COUNT(*) OVER() here — on large result sets the window # function forces SQLite to materialise every matching row before it # can return page 1, making first-page loads slow at scale. # _global_fts_total_count runs a lean UNION count query instead. - sql_parts = [self._global_fts_select_sql("", metadata_tags_text_clause)] + sql_parts = [self._global_fts_select_sql("", metadata_tags_text_clause, substring_pattern)] params: list[Any] = [fts_query, fts_query] + if substring_pattern: + params.append(substring_pattern) filter_clauses, filter_params = self._filter_clauses(filters) sql_parts.extend(filter_clauses) @@ -1271,11 +1312,15 @@ class IndexSearcher: total: int | None = None if include_total: - total = await self._global_fts_total_count(fts_query, filter_clauses, filter_params) + total = await self._global_fts_total_count( + fts_query, filter_clauses, filter_params, substring_pattern=substring_pattern + ) return Result.Ok({"rows": rows, "total": total}) @staticmethod - def _global_fts_select_sql(total_field: str, metadata_tags_text_clause: str) -> str: + def _global_fts_select_sql( + total_field: str, metadata_tags_text_clause: str, substring_pattern: str | None = None + ) -> str: return f""" WITH matches AS ( SELECT rowid AS asset_id, bm25(assets_fts, 8.0, 1.25) AS rank @@ -1287,7 +1332,7 @@ class IndexSearcher: SELECT rowid AS asset_id, (bm25(asset_metadata_fts, 7.0, 4.0, 1.5) + 2.0) AS rank FROM asset_metadata_fts WHERE asset_metadata_fts MATCH ? - ), +{_substring_matches_sql(substring_pattern, with_rank=True)} ), best AS ( SELECT asset_id, MIN(rank) AS rank FROM matches @@ -1319,8 +1364,10 @@ class IndexSearcher: fts_query: str, filter_clauses: list[str], filter_params: list[Any], + *, + substring_pattern: str | None = None, ) -> int: - count_sql = """ + count_sql = f""" WITH matches AS ( SELECT rowid AS asset_id FROM assets_fts @@ -1331,7 +1378,7 @@ class IndexSearcher: SELECT rowid AS asset_id FROM asset_metadata_fts WHERE asset_metadata_fts MATCH ? - ) +{_substring_matches_sql(substring_pattern, with_rank=False)} ) SELECT COUNT(*) as total FROM (SELECT DISTINCT asset_id FROM matches) t JOIN assets a ON t.asset_id = a.id @@ -1339,6 +1386,8 @@ class IndexSearcher: WHERE 1=1 """ count_params: list[Any] = [fts_query, fts_query] + if substring_pattern: + count_params.append(substring_pattern) if filter_clauses: count_sql += " " + " ".join(filter_clauses) count_params.extend(filter_params) @@ -1436,6 +1485,7 @@ class IndexSearcher: include_total: bool, metadata_tags_text_clause: str, sort: str | None, + substring_pattern: str | None = None, ) -> Result[dict[str, Any]]: sql_parts = [ f""" @@ -1449,7 +1499,7 @@ class IndexSearcher: SELECT rowid AS asset_id, (bm25(asset_metadata_fts, 7.0, 4.0, 1.5) + 2.0) AS rank FROM asset_metadata_fts WHERE asset_metadata_fts MATCH ? - ), +{_substring_matches_sql(substring_pattern, with_rank=True)} ), best AS ( SELECT asset_id, MIN(rank) AS rank FROM matches @@ -1475,6 +1525,8 @@ class IndexSearcher: """ ] params: list[Any] = [fts_query, fts_query] + if substring_pattern: + params.append(substring_pattern) params.extend(roots_params) filter_clauses, filter_params = self._filter_clauses(filters, assert_safe=True) @@ -1506,7 +1558,7 @@ class IndexSearcher: SELECT rowid AS asset_id FROM asset_metadata_fts WHERE asset_metadata_fts MATCH ? - ) +{_substring_matches_sql(substring_pattern, with_rank=False)} ) SELECT COUNT(*) as total FROM (SELECT DISTINCT asset_id FROM matches) t JOIN assets a ON t.asset_id = a.id @@ -1514,6 +1566,8 @@ class IndexSearcher: WHERE {roots_clause} """ count_params: list[Any] = [fts_query, fts_query] + if substring_pattern: + count_params.append(substring_pattern) count_params.extend(roots_params) if filter_clauses: count_sql += " " + " ".join(filter_clauses) @@ -1715,6 +1769,7 @@ class IndexSearcher: return Result.Err("INVALID_INPUT", "Invalid search query syntax") return await self._search_global_fts_rows( fts_query=fts_query, + substring_pattern=_filename_substring_pattern(query), limit=raw_limit, offset=raw_offset, filters=filters, @@ -1744,6 +1799,7 @@ class IndexSearcher: return Result.Err("INVALID_INPUT", "Invalid search query syntax") return await self._search_scoped_fts_rows( fts_query=fts_query, + substring_pattern=_filename_substring_pattern(query), roots_clause=roots_clause, roots_params=roots_params, limit=raw_limit, @@ -1783,6 +1839,7 @@ class IndexSearcher: return Result.Err("INVALID_INPUT", "Invalid search query syntax") rows_total_res = await self._search_global_fts_rows( fts_query=fts_query, + substring_pattern=_filename_substring_pattern(query), limit=limit, offset=offset, filters=filters, @@ -1833,6 +1890,7 @@ class IndexSearcher: return Result.Err("INVALID_INPUT", "Invalid search query syntax") rows_total_res = await self._search_scoped_fts_rows( fts_query=fts_query, + substring_pattern=_filename_substring_pattern(query), roots_clause=roots_clause, roots_params=roots_params, limit=limit, diff --git a/tests/features/test_search_substring.py b/tests/features/test_search_substring.py new file mode 100644 index 00000000..4fcbcc5f --- /dev/null +++ b/tests/features/test_search_substring.py @@ -0,0 +1,63 @@ +"""Single-word searches also match file names that *contain* the text.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from mjr_am_backend import deps +from mjr_am_backend.features.index.searcher import _filename_substring_pattern + +NAMES = ["ComfyUI_00001_.png", "Flux_portrait.png", "field.png", "sunflower.png", "cat.png", "50%_off.png", "a_b.png", "axb.png"] + + +def test_pattern_rules(): + assert _filename_substring_pattern("F") == "%F%" + assert _filename_substring_pattern("red car") is None # multi-word: FTS only + assert _filename_substring_pattern('"exact"') is None + assert _filename_substring_pattern("*") is None + assert _filename_substring_pattern("a_b") == "%a\\_b%" # LIKE wildcards escaped + assert _filename_substring_pattern("50%") == "%50\\%%" + + +@pytest.mark.asyncio +async def test_single_letter_lists_every_file_containing_it(tmp_path: Path, monkeypatch): + root = tmp_path / "out" + root.mkdir() + for name in NAMES: + (root / name).write_bytes(b"\x89PNG\r\n\x1a\n") + monkeypatch.setattr(deps, "WATCHER_ENABLED", False) + res = await deps.build_services(str(tmp_path / "db.sqlite")) + assert res.ok, res.error + services = res.data + try: + index = services["index"] + scan = await index.scan_directory( + directory=str(root), recursive=False, incremental=True, source="output", root_id=None + ) + assert scan.ok, scan.error + + async def names(query: str, *, scoped: bool) -> set[str]: + if scoped: + r = await index.search_scoped( + query, roots=[str(root)], limit=50, offset=0, filters=None, include_total=True + ) + else: + r = await index.search(query, limit=50, offset=0, filters=None, include_total=True) + assert r.ok, r.error + found = {a["filename"] for a in r.data["assets"]} + assert r.data.get("total") in (None, len(found)) + return found + + for scoped in (True, False): + assert await names("F", scoped=scoped) == { + "ComfyUI_00001_.png", "Flux_portrait.png", "field.png", "sunflower.png", "50%_off.png", + } + assert await names("flow", scoped=scoped) == {"sunflower.png"} + # "_" and "%" are literal, not LIKE wildcards. + assert await names("a_b", scoped=scoped) == {"a_b.png"} + finally: + worker = services.get("rating_tags_sync") + if worker is not None: + worker.stop() + await services["db"].aclose()