From 8462ecbb03d2990f6684e67659a364d23208e26c Mon Sep 17 00:00:00 2001 From: Vito Sansevero Date: Thu, 8 Oct 2026 18:33:18 -0700 Subject: [PATCH] fix: Windows case-folded gallery roots in tests, "/" subfolders everywhere, longer busy timeout Second CI run on Windows: - gallery roots are stored resolved and case-folded there; the admin API tests now compare against that form instead of the mixed-case temp path - the subfolder fallback for images outside every root also uses "/", and the test expects it - a relative diagnostics db path is normalised after joining - "database is locked" under many writers on a slow disk: connections now wait up to 30 seconds (connect timeout and busy_timeout), long enough for a scan or thumbnail run while ComfyUI saves images --- database/models.py | 22 ++++++---- database/operations.py | 2 +- tests/test_api_admin.py | 92 ++++++++++++++++++++++++----------------- tests/test_database.py | 6 ++- utils/diagnostics.py | 2 +- 5 files changed, 73 insertions(+), 51 deletions(-) diff --git a/database/models.py b/database/models.py index 351835d..e6761c2 100644 --- a/database/models.py +++ b/database/models.py @@ -49,6 +49,10 @@ def normalize_image_path(path: str) -> str: return os.path.normcase(os.path.normpath(os.path.abspath(path))) +# How long a connection waits for a locked database before giving up. Thirty +# seconds covers a scan or a thumbnail run writing while ComfyUI saves images, +# and the slow disks of CI runners. +BUSY_TIMEOUT_SECONDS = 30 SQLITE_HEADER = b"SQLite format 3\x00" REQUIRED_PROMPT_COLUMNS = ("id", "text", "created_at") @@ -103,10 +107,10 @@ class PromptModel: if key in PromptModel._initialized_paths and os.path.exists(self.db_path): return try: - conn = sqlite3.connect(self.db_path) + conn = sqlite3.connect(self.db_path, timeout=BUSY_TIMEOUT_SECONDS) try: conn.execute("PRAGMA journal_mode = WAL") - conn.execute("PRAGMA busy_timeout = 5000") + conn.execute(f"PRAGMA busy_timeout = {BUSY_TIMEOUT_SECONDS * 1000}") conn.execute("PRAGMA foreign_keys = ON") self._create_tables(conn) self._create_indexes(conn) @@ -275,10 +279,10 @@ class PromptModel: ``threading.local``), created on first use and reused by that thread thereafter. Every connection is configured with WAL journaling, ``row_factory = sqlite3.Row``, ``PRAGMA foreign_keys = ON`` and - ``PRAGMA busy_timeout = 5000`` so concurrent writers wait instead of - failing. Callers use it as a context manager (``with conn:``), which - commits or rolls back that thread's transaction and leaves the - connection open. No lock is held while a connection is in use. + ``PRAGMA busy_timeout`` of BUSY_TIMEOUT_SECONDS so concurrent writers + wait instead of failing. Callers use it as a context manager + (``with conn:``), which commits or rolls back that thread's transaction + and leaves the connection open. No lock is held while a connection is in use. Returns: sqlite3.Connection: The current thread's configured connection @@ -289,11 +293,13 @@ class PromptModel: # check_same_thread=False only so restore_from_file can close other # threads' connections; each connection is still used by one thread. - conn = sqlite3.connect(self.db_path, check_same_thread=False) + conn = sqlite3.connect( + self.db_path, check_same_thread=False, timeout=BUSY_TIMEOUT_SECONDS + ) conn.row_factory = sqlite3.Row conn.execute("PRAGMA journal_mode = WAL") conn.execute("PRAGMA foreign_keys = ON") - conn.execute("PRAGMA busy_timeout = 5000") + conn.execute(f"PRAGMA busy_timeout = {BUSY_TIMEOUT_SECONDS * 1000}") self._local.conn = conn self._local.generation = self._generation with self._conn_lock: diff --git a/database/operations.py b/database/operations.py index f942a18..a1b8bcb 100644 --- a/database/operations.py +++ b/database/operations.py @@ -683,7 +683,7 @@ class PromptDatabase: # Preserve full relative-style path instead of collapsing # to just the basename, which loses hierarchy and creates # ambiguity (e.g. foo/bar and baz/bar both become "bar"). - folders.add(parent) + folders.add(parent.replace(os.sep, "/")) if include_ancestors: ancestors = set() diff --git a/tests/test_api_admin.py b/tests/test_api_admin.py index fdb2d1b..a6a5e04 100644 --- a/tests/test_api_admin.py +++ b/tests/test_api_admin.py @@ -95,6 +95,11 @@ class AdminAPITestCase(AioHTTPTestCase): # ── helpers ──────────────────────────────────────────────────────── + @staticmethod + def _root(path): + """A gallery root as the config stores it: resolved, case-folded on Windows.""" + return GalleryConfig.resolve_gallery_root(str(path)) + def _write_config(self, content): with open(self.config_path, "w") as f: json.dump(content, f) @@ -127,7 +132,7 @@ class TestSaveSettingsGalleryRoots(AdminAPITestCase): resp = await self.client.request( "POST", "/prompt_manager/settings", - json={"gallery_root_paths": [str(self.outside_dir)]}, + json={"gallery_root_paths": [self._root(self.outside_dir)]}, ) self.assertEqual(resp.status, 400) @@ -168,16 +173,18 @@ class TestSaveSettingsGalleryRoots(AdminAPITestCase): resp = await self.client.request( "POST", "/prompt_manager/settings", - json={"gallery_root_paths": [str(good)]}, + json={"gallery_root_paths": [self._root(good)]}, ) self.assertEqual(resp.status, 200) data = await resp.json() self.assertTrue(data["success"]) self.assertTrue(data["restart_required"]) - self.assertEqual(GalleryConfig.MONITORING_DIRECTORIES, [str(good)]) + self.assertEqual(GalleryConfig.MONITORING_DIRECTORIES, [self._root(good)]) saved = self._read_config() - self.assertEqual(saved["gallery"]["monitoring"]["directories"], [str(good)]) + self.assertEqual( + saved["gallery"]["monitoring"]["directories"], [self._root(good)] + ) async def test_accepts_env_listed_directory(self): os.environ[EXTRA_ROOTS_ENV] = str(self.outside_dir) @@ -185,11 +192,13 @@ class TestSaveSettingsGalleryRoots(AdminAPITestCase): resp = await self.client.request( "POST", "/prompt_manager/settings", - json={"gallery_root_paths": [str(self.outside_dir)]}, + json={"gallery_root_paths": [self._root(self.outside_dir)]}, ) self.assertEqual(resp.status, 200) - self.assertEqual(GalleryConfig.MONITORING_DIRECTORIES, [str(self.outside_dir)]) + self.assertEqual( + GalleryConfig.MONITORING_DIRECTORIES, [self._root(self.outside_dir)] + ) async def test_legacy_single_path_accepted(self): resp = await self.client.request( @@ -199,10 +208,12 @@ class TestSaveSettingsGalleryRoots(AdminAPITestCase): ) self.assertEqual(resp.status, 200) - self.assertEqual(GalleryConfig.MONITORING_DIRECTORIES, [str(self.output_dir)]) + self.assertEqual( + GalleryConfig.MONITORING_DIRECTORIES, [self._root(self.output_dir)] + ) async def test_empty_paths_clear_roots(self): - GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)] + GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)] resp = await self.client.request( "POST", @@ -293,8 +304,8 @@ class TestNoAbsolutePathsInResponses(AdminAPITestCase): ) async def test_diagnostics_paths_are_relative(self): - GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)] - self._install_monitor([str(self.output_dir)]) + GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)] + self._install_monitor([self._root(self.output_dir)]) resp, body = await self._body("GET", "/prompt_manager/diagnostics") @@ -322,7 +333,7 @@ class TestNoAbsolutePathsInResponses(AdminAPITestCase): (self.output_dir / "a.png").write_bytes(b"same-bytes") (self.output_dir / "sub").mkdir() (self.output_dir / "sub" / "b.png").write_bytes(b"same-bytes") - GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)] + GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)] resp, body = await self._body("GET", "/prompt_manager/scan_duplicates") @@ -337,7 +348,7 @@ class TestNoAbsolutePathsInResponses(AdminAPITestCase): target = self.output_dir / "sub" / "b.png" target.parent.mkdir() target.write_bytes(b"bytes") - GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)] + GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)] resp, body = await self._body( "POST", @@ -351,7 +362,7 @@ class TestNoAbsolutePathsInResponses(AdminAPITestCase): self.assertFalse(target.exists()) async def test_scan_images_stream_has_no_absolute_paths(self): - GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)] + GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)] resp, body = await self._body("POST", "/prompt_manager/scan") @@ -396,7 +407,7 @@ class TestScansRunOffTheEventLoop(AdminAPITestCase): async def test_scan_duplicates_runs_in_executor(self): (self.output_dir / "a.png").write_bytes(b"dup") (self.output_dir / "b.png").write_bytes(b"dup") - GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)] + GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)] calls = self._spy_executor() resp = await self.client.request("GET", "/prompt_manager/scan_duplicates") @@ -407,7 +418,7 @@ class TestScansRunOffTheEventLoop(AdminAPITestCase): async def test_delete_duplicates_runs_in_executor(self): (self.output_dir / "a.png").write_bytes(b"dup") - GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)] + GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)] calls = self._spy_executor() resp = await self.client.request( @@ -422,7 +433,7 @@ class TestScansRunOffTheEventLoop(AdminAPITestCase): async def test_output_scan_runs_in_executor(self): _write_png(self.output_dir / "gen.png", prompt_text="a red square") - GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)] + GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)] calls = self._spy_executor() resp = await self.client.request("POST", "/prompt_manager/scan") @@ -440,7 +451,7 @@ class TestScansRunOffTheEventLoop(AdminAPITestCase): async def test_output_scan_links_existing_prompt_on_second_pass(self): _write_png(self.output_dir / "gen.png", prompt_text="a red square") - GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)] + GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)] first = await (await self.client.request("POST", "/prompt_manager/scan")).text() second = await ( @@ -454,7 +465,7 @@ class TestScansRunOffTheEventLoop(AdminAPITestCase): self.assertEqual([p["text"] for p in prompts], ["a red square"]) async def test_output_scan_without_directories_reports_error(self): - GalleryConfig.MONITORING_DIRECTORIES = [str(self.outside_dir)] + GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.outside_dir)] sys.modules.pop("folder_paths", None) self.api._find_comfyui_output_dir = lambda: None @@ -717,7 +728,7 @@ class TestSettingsReadMonitorThroughModule(AdminAPITestCase): im_mod._monitor_instance = None self.addCleanup(setattr, im_mod, "_monitor_instance", orig) monitor = get_image_monitor(MagicMock(), MagicMock()) - monitor.monitored_directories = [str(self.output_dir)] + monitor.monitored_directories = [self._root(self.output_dir)] resp = await self.client.request("GET", "/prompt_manager/settings") @@ -731,7 +742,7 @@ class TestSettingsReadMonitorThroughModule(AdminAPITestCase): orig = im_mod._monitor_instance im_mod._monitor_instance = None self.addCleanup(setattr, im_mod, "_monitor_instance", orig) - GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)] + GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)] resp = await self.client.request("GET", "/prompt_manager/settings") @@ -809,7 +820,7 @@ class TestDeleteDuplicatesEndpoint(AdminAPITestCase): self.assertEqual(resp.status, 400) async def test_failures_are_reported(self): - GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)] + GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)] resp = await self._post({"image_paths": ["ghost.png"]}) data = await resp.json() self.assertEqual(resp.status, 200) @@ -986,7 +997,10 @@ class TestSettingsMisc(AdminAPITestCase): resp = await self.client.request( "POST", "/prompt_manager/settings", - json={"result_timeout": 7, "gallery_root_paths": [str(self.output_dir)]}, + json={ + "result_timeout": 7, + "gallery_root_paths": [self._root(self.output_dir)], + }, ) self.assertEqual(resp.status, 200) @@ -997,7 +1011,7 @@ class TestSettingsMisc(AdminAPITestCase): self.assertEqual(saved["web_ui"]["result_timeout"], 7) self.assertEqual(saved["web_ui"]["show_test_button"], True) self.assertEqual( - saved["gallery"]["monitoring"]["directories"], [str(self.output_dir)] + saved["gallery"]["monitoring"]["directories"], [self._root(self.output_dir)] ) async def test_save_replaces_a_corrupt_config_file(self): @@ -1042,7 +1056,7 @@ class TestSettingsMisc(AdminAPITestCase): resp = await self.client.request( "POST", "/prompt_manager/settings", - json={"gallery_root_paths": [str(self.output_dir)]}, + json={"gallery_root_paths": [self._root(self.output_dir)]}, ) data = await resp.json() self.assertEqual(resp.status, 200) @@ -1084,7 +1098,7 @@ class TestDiagnostics(AdminAPITestCase): async def test_healthy_report(self): self.api.db.save_prompt(text="hello", prompt_hash="h1") - self._install_monitor(FakeMonitor([str(self.output_dir)])) + self._install_monitor(FakeMonitor([self._root(self.output_dir)])) resp, diag = await self._diag() @@ -1172,7 +1186,7 @@ class TestTestImageLink(AdminAPITestCase): self.assertIsInstance(data["result"]["image_id"], int) async def test_client_supplied_path_is_not_stored(self): - GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)] + GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)] pid = self.api.db.save_prompt(text="hello", prompt_hash="h1") evil = os.path.join(self.tmpdir, "etc", "passwd") @@ -1535,7 +1549,7 @@ class TestOutputScanInternals(AdminAPITestCase): self.assertEqual(counts, {"processed": 2, "found": 0, "added": 0, "linked": 0}) async def test_scan_internal_error_streams_error_event(self): - GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)] + GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)] self.api._collect_output_media_sync = _raise body = await (await self.client.request("POST", "/prompt_manager/scan")).text() self.assertIn('"type": "error"', body) @@ -1566,7 +1580,7 @@ class TestOutputScanIsABackgroundJob(AdminAPITestCase): from unittest.mock import AsyncMock, patch _write_png(self.output_dir / "gen.png", prompt_text="a red square") - GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)] + GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)] with self.assertNoLogs("aiohttp.server", level="ERROR"): with patch.object( @@ -1586,7 +1600,7 @@ class TestOutputScanIsABackgroundJob(AdminAPITestCase): async def test_second_request_attaches_to_the_running_scan(self): _write_png(self.output_dir / "gen.png", prompt_text="a red square") - GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)] + GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)] gate, calls = self._gate_collect() first = asyncio.ensure_future( @@ -1609,7 +1623,7 @@ class TestOutputScanIsABackgroundJob(AdminAPITestCase): self.assertIn('"added": 1', second_body) async def test_status_reports_running_scan_and_its_last_event(self): - GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)] + GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)] gate, _ = self._gate_collect() first = asyncio.ensure_future( @@ -1639,7 +1653,7 @@ class TestOutputScanIsABackgroundJob(AdminAPITestCase): async def test_scan_after_a_finished_one_starts_fresh(self): _write_png(self.output_dir / "gen.png", prompt_text="a red square") - GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)] + GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)] gate, calls = self._gate_collect() gate.set() @@ -1650,7 +1664,7 @@ class TestOutputScanIsABackgroundJob(AdminAPITestCase): self.assertIn('"linked": 1', body) async def test_failed_scan_is_not_left_running(self): - GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)] + GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)] self.api._collect_output_media_sync = _raise await (await self.client.request("POST", "/prompt_manager/scan")).text() @@ -1707,7 +1721,7 @@ class TestWorkerThreadsSetting(AdminAPITestCase): for i in range(8): _write_png(self.output_dir / f"gen{i}.png", prompt_text=f"prompt {i}") - GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)] + GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)] PromptManagerConfig.WORKER_THREADS = 4 seen = set() lock = threading.Lock() @@ -1728,7 +1742,7 @@ class TestWorkerThreadsSetting(AdminAPITestCase): async def test_scan_with_one_worker_still_completes(self): _write_png(self.output_dir / "gen.png", prompt_text="solo") - GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)] + GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)] PromptManagerConfig.WORKER_THREADS = 1 body = await (await self.client.request("POST", "/prompt_manager/scan")).text() @@ -1777,7 +1791,7 @@ class TestScanReadsVideos(AdminAPITestCase): _write_mp4( self.output_dir / "clip.mp4", json.dumps({"prompt": json.dumps(graph)}) ) - GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)] + GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)] with self.assertNoLogs(level="ERROR"): body = await ( @@ -1795,7 +1809,7 @@ class TestScanReadsVideos(AdminAPITestCase): from utils import video_metadata (self.output_dir / "clip.mp4").write_bytes(b"\x00\x00\x00\x18ftypisom") - GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)] + GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)] with patch.object(video_metadata.shutil, "which", return_value=None): with self.assertNoLogs("prompt_manager", level="WARNING"): @@ -1808,7 +1822,7 @@ class TestScanReadsVideos(AdminAPITestCase): async def test_unreadable_image_is_a_warning_not_an_error(self): (self.output_dir / "bad.png").write_bytes(b"\x89PNG not really") - GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)] + GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)] with self.assertLogs(level="WARNING") as cm: body = await ( @@ -1837,7 +1851,7 @@ class TestDuplicateScanIsABackgroundJob(AdminAPITestCase): (self.output_dir / "a.png").write_bytes(b"same bytes") (self.output_dir / "b.png").write_bytes(b"same bytes") (self.output_dir / "c.png").write_bytes(b"other bytes") - GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)] + GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)] def _gate_hashing(self): import threading @@ -1906,7 +1920,7 @@ class TestDuplicateScanIsABackgroundJob(AdminAPITestCase): self.assertEqual(data, {"success": True, "running": False, "last_event": None}) async def test_failed_scan_streams_an_error_and_releases_the_lock(self): - GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)] + GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)] self.api._find_comfyui_output_dir = _raise body = await (await self.client.request("POST", self.STREAM)).text() diff --git a/tests/test_database.py b/tests/test_database.py index b33d233..e5f58b7 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -1058,8 +1058,10 @@ class TestMaintenanceOperations(DatabaseTestCase): self.db.link_image_to_prompt(pid, "loose.png") flat = self.db.get_prompt_subfolders(root_dirs=[root]) - self.assertIn(os.path.join("2026", "08-Aug", "2026-08-06"), flat) - self.assertIn(os.path.join(os.sep, "elsewhere", "deep"), flat) + self.assertIn("2026/08-Aug/2026-08-06", flat) + self.assertIn( + os.path.join(os.sep, "elsewhere", "deep").replace(os.sep, "/"), flat + ) self.assertNotIn(".", flat) nested = self.db.get_prompt_subfolders(root_dirs=[root], include_ancestors=True) diff --git a/utils/diagnostics.py b/utils/diagnostics.py index c0c72d4..ff88cb8 100644 --- a/utils/diagnostics.py +++ b/utils/diagnostics.py @@ -84,7 +84,7 @@ class GalleryDiagnostics: db_path = "prompts.db" if not os.path.isabs(db_path): - db_path = os.path.join(extension_root, db_path) + db_path = os.path.normpath(os.path.join(extension_root, db_path)) return db_path def run_full_diagnostic(self) -> Dict[str, Any]: