fix: Windows case-folded gallery roots in tests, "/" subfolders everywhere, longer busy timeout

Second CI run on Windows:
- gallery roots are stored resolved and case-folded there; the admin API
  tests now compare against that form instead of the mixed-case temp path
- the subfolder fallback for images outside every root also uses "/", and
  the test expects it
- a relative diagnostics db path is normalised after joining
- "database is locked" under many writers on a slow disk: connections now
  wait up to 30 seconds (connect timeout and busy_timeout), long enough for
  a scan or thumbnail run while ComfyUI saves images
This commit is contained in:
Vito Sansevero
2026-10-08 18:33:18 -07:00
parent b1bfce702d
commit 8462ecbb03
5 changed files with 73 additions and 51 deletions
+14 -8
View File
@@ -49,6 +49,10 @@ def normalize_image_path(path: str) -> str:
return os.path.normcase(os.path.normpath(os.path.abspath(path)))
# How long a connection waits for a locked database before giving up. Thirty
# seconds covers a scan or a thumbnail run writing while ComfyUI saves images,
# and the slow disks of CI runners.
BUSY_TIMEOUT_SECONDS = 30
SQLITE_HEADER = b"SQLite format 3\x00"
REQUIRED_PROMPT_COLUMNS = ("id", "text", "created_at")
@@ -103,10 +107,10 @@ class PromptModel:
if key in PromptModel._initialized_paths and os.path.exists(self.db_path):
return
try:
conn = sqlite3.connect(self.db_path)
conn = sqlite3.connect(self.db_path, timeout=BUSY_TIMEOUT_SECONDS)
try:
conn.execute("PRAGMA journal_mode = WAL")
conn.execute("PRAGMA busy_timeout = 5000")
conn.execute(f"PRAGMA busy_timeout = {BUSY_TIMEOUT_SECONDS * 1000}")
conn.execute("PRAGMA foreign_keys = ON")
self._create_tables(conn)
self._create_indexes(conn)
@@ -275,10 +279,10 @@ class PromptModel:
``threading.local``), created on first use and reused by that thread
thereafter. Every connection is configured with WAL journaling,
``row_factory = sqlite3.Row``, ``PRAGMA foreign_keys = ON`` and
``PRAGMA busy_timeout = 5000`` so concurrent writers wait instead of
failing. Callers use it as a context manager (``with conn:``), which
commits or rolls back that thread's transaction and leaves the
connection open. No lock is held while a connection is in use.
``PRAGMA busy_timeout`` of BUSY_TIMEOUT_SECONDS so concurrent writers
wait instead of failing. Callers use it as a context manager
(``with conn:``), which commits or rolls back that thread's transaction
and leaves the connection open. No lock is held while a connection is in use.
Returns:
sqlite3.Connection: The current thread's configured connection
@@ -289,11 +293,13 @@ class PromptModel:
# check_same_thread=False only so restore_from_file can close other
# threads' connections; each connection is still used by one thread.
conn = sqlite3.connect(self.db_path, check_same_thread=False)
conn = sqlite3.connect(
self.db_path, check_same_thread=False, timeout=BUSY_TIMEOUT_SECONDS
)
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA journal_mode = WAL")
conn.execute("PRAGMA foreign_keys = ON")
conn.execute("PRAGMA busy_timeout = 5000")
conn.execute(f"PRAGMA busy_timeout = {BUSY_TIMEOUT_SECONDS * 1000}")
self._local.conn = conn
self._local.generation = self._generation
with self._conn_lock:
+1 -1
View File
@@ -683,7 +683,7 @@ class PromptDatabase:
# Preserve full relative-style path instead of collapsing
# to just the basename, which loses hierarchy and creates
# ambiguity (e.g. foo/bar and baz/bar both become "bar").
folders.add(parent)
folders.add(parent.replace(os.sep, "/"))
if include_ancestors:
ancestors = set()
+53 -39
View File
@@ -95,6 +95,11 @@ class AdminAPITestCase(AioHTTPTestCase):
# ── helpers ────────────────────────────────────────────────────────
@staticmethod
def _root(path):
"""A gallery root as the config stores it: resolved, case-folded on Windows."""
return GalleryConfig.resolve_gallery_root(str(path))
def _write_config(self, content):
with open(self.config_path, "w") as f:
json.dump(content, f)
@@ -127,7 +132,7 @@ class TestSaveSettingsGalleryRoots(AdminAPITestCase):
resp = await self.client.request(
"POST",
"/prompt_manager/settings",
json={"gallery_root_paths": [str(self.outside_dir)]},
json={"gallery_root_paths": [self._root(self.outside_dir)]},
)
self.assertEqual(resp.status, 400)
@@ -168,16 +173,18 @@ class TestSaveSettingsGalleryRoots(AdminAPITestCase):
resp = await self.client.request(
"POST",
"/prompt_manager/settings",
json={"gallery_root_paths": [str(good)]},
json={"gallery_root_paths": [self._root(good)]},
)
self.assertEqual(resp.status, 200)
data = await resp.json()
self.assertTrue(data["success"])
self.assertTrue(data["restart_required"])
self.assertEqual(GalleryConfig.MONITORING_DIRECTORIES, [str(good)])
self.assertEqual(GalleryConfig.MONITORING_DIRECTORIES, [self._root(good)])
saved = self._read_config()
self.assertEqual(saved["gallery"]["monitoring"]["directories"], [str(good)])
self.assertEqual(
saved["gallery"]["monitoring"]["directories"], [self._root(good)]
)
async def test_accepts_env_listed_directory(self):
os.environ[EXTRA_ROOTS_ENV] = str(self.outside_dir)
@@ -185,11 +192,13 @@ class TestSaveSettingsGalleryRoots(AdminAPITestCase):
resp = await self.client.request(
"POST",
"/prompt_manager/settings",
json={"gallery_root_paths": [str(self.outside_dir)]},
json={"gallery_root_paths": [self._root(self.outside_dir)]},
)
self.assertEqual(resp.status, 200)
self.assertEqual(GalleryConfig.MONITORING_DIRECTORIES, [str(self.outside_dir)])
self.assertEqual(
GalleryConfig.MONITORING_DIRECTORIES, [self._root(self.outside_dir)]
)
async def test_legacy_single_path_accepted(self):
resp = await self.client.request(
@@ -199,10 +208,12 @@ class TestSaveSettingsGalleryRoots(AdminAPITestCase):
)
self.assertEqual(resp.status, 200)
self.assertEqual(GalleryConfig.MONITORING_DIRECTORIES, [str(self.output_dir)])
self.assertEqual(
GalleryConfig.MONITORING_DIRECTORIES, [self._root(self.output_dir)]
)
async def test_empty_paths_clear_roots(self):
GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)]
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
resp = await self.client.request(
"POST",
@@ -293,8 +304,8 @@ class TestNoAbsolutePathsInResponses(AdminAPITestCase):
)
async def test_diagnostics_paths_are_relative(self):
GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)]
self._install_monitor([str(self.output_dir)])
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
self._install_monitor([self._root(self.output_dir)])
resp, body = await self._body("GET", "/prompt_manager/diagnostics")
@@ -322,7 +333,7 @@ class TestNoAbsolutePathsInResponses(AdminAPITestCase):
(self.output_dir / "a.png").write_bytes(b"same-bytes")
(self.output_dir / "sub").mkdir()
(self.output_dir / "sub" / "b.png").write_bytes(b"same-bytes")
GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)]
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
resp, body = await self._body("GET", "/prompt_manager/scan_duplicates")
@@ -337,7 +348,7 @@ class TestNoAbsolutePathsInResponses(AdminAPITestCase):
target = self.output_dir / "sub" / "b.png"
target.parent.mkdir()
target.write_bytes(b"bytes")
GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)]
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
resp, body = await self._body(
"POST",
@@ -351,7 +362,7 @@ class TestNoAbsolutePathsInResponses(AdminAPITestCase):
self.assertFalse(target.exists())
async def test_scan_images_stream_has_no_absolute_paths(self):
GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)]
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
resp, body = await self._body("POST", "/prompt_manager/scan")
@@ -396,7 +407,7 @@ class TestScansRunOffTheEventLoop(AdminAPITestCase):
async def test_scan_duplicates_runs_in_executor(self):
(self.output_dir / "a.png").write_bytes(b"dup")
(self.output_dir / "b.png").write_bytes(b"dup")
GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)]
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
calls = self._spy_executor()
resp = await self.client.request("GET", "/prompt_manager/scan_duplicates")
@@ -407,7 +418,7 @@ class TestScansRunOffTheEventLoop(AdminAPITestCase):
async def test_delete_duplicates_runs_in_executor(self):
(self.output_dir / "a.png").write_bytes(b"dup")
GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)]
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
calls = self._spy_executor()
resp = await self.client.request(
@@ -422,7 +433,7 @@ class TestScansRunOffTheEventLoop(AdminAPITestCase):
async def test_output_scan_runs_in_executor(self):
_write_png(self.output_dir / "gen.png", prompt_text="a red square")
GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)]
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
calls = self._spy_executor()
resp = await self.client.request("POST", "/prompt_manager/scan")
@@ -440,7 +451,7 @@ class TestScansRunOffTheEventLoop(AdminAPITestCase):
async def test_output_scan_links_existing_prompt_on_second_pass(self):
_write_png(self.output_dir / "gen.png", prompt_text="a red square")
GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)]
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
first = await (await self.client.request("POST", "/prompt_manager/scan")).text()
second = await (
@@ -454,7 +465,7 @@ class TestScansRunOffTheEventLoop(AdminAPITestCase):
self.assertEqual([p["text"] for p in prompts], ["a red square"])
async def test_output_scan_without_directories_reports_error(self):
GalleryConfig.MONITORING_DIRECTORIES = [str(self.outside_dir)]
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.outside_dir)]
sys.modules.pop("folder_paths", None)
self.api._find_comfyui_output_dir = lambda: None
@@ -717,7 +728,7 @@ class TestSettingsReadMonitorThroughModule(AdminAPITestCase):
im_mod._monitor_instance = None
self.addCleanup(setattr, im_mod, "_monitor_instance", orig)
monitor = get_image_monitor(MagicMock(), MagicMock())
monitor.monitored_directories = [str(self.output_dir)]
monitor.monitored_directories = [self._root(self.output_dir)]
resp = await self.client.request("GET", "/prompt_manager/settings")
@@ -731,7 +742,7 @@ class TestSettingsReadMonitorThroughModule(AdminAPITestCase):
orig = im_mod._monitor_instance
im_mod._monitor_instance = None
self.addCleanup(setattr, im_mod, "_monitor_instance", orig)
GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)]
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
resp = await self.client.request("GET", "/prompt_manager/settings")
@@ -809,7 +820,7 @@ class TestDeleteDuplicatesEndpoint(AdminAPITestCase):
self.assertEqual(resp.status, 400)
async def test_failures_are_reported(self):
GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)]
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
resp = await self._post({"image_paths": ["ghost.png"]})
data = await resp.json()
self.assertEqual(resp.status, 200)
@@ -986,7 +997,10 @@ class TestSettingsMisc(AdminAPITestCase):
resp = await self.client.request(
"POST",
"/prompt_manager/settings",
json={"result_timeout": 7, "gallery_root_paths": [str(self.output_dir)]},
json={
"result_timeout": 7,
"gallery_root_paths": [self._root(self.output_dir)],
},
)
self.assertEqual(resp.status, 200)
@@ -997,7 +1011,7 @@ class TestSettingsMisc(AdminAPITestCase):
self.assertEqual(saved["web_ui"]["result_timeout"], 7)
self.assertEqual(saved["web_ui"]["show_test_button"], True)
self.assertEqual(
saved["gallery"]["monitoring"]["directories"], [str(self.output_dir)]
saved["gallery"]["monitoring"]["directories"], [self._root(self.output_dir)]
)
async def test_save_replaces_a_corrupt_config_file(self):
@@ -1042,7 +1056,7 @@ class TestSettingsMisc(AdminAPITestCase):
resp = await self.client.request(
"POST",
"/prompt_manager/settings",
json={"gallery_root_paths": [str(self.output_dir)]},
json={"gallery_root_paths": [self._root(self.output_dir)]},
)
data = await resp.json()
self.assertEqual(resp.status, 200)
@@ -1084,7 +1098,7 @@ class TestDiagnostics(AdminAPITestCase):
async def test_healthy_report(self):
self.api.db.save_prompt(text="hello", prompt_hash="h1")
self._install_monitor(FakeMonitor([str(self.output_dir)]))
self._install_monitor(FakeMonitor([self._root(self.output_dir)]))
resp, diag = await self._diag()
@@ -1172,7 +1186,7 @@ class TestTestImageLink(AdminAPITestCase):
self.assertIsInstance(data["result"]["image_id"], int)
async def test_client_supplied_path_is_not_stored(self):
GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)]
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
pid = self.api.db.save_prompt(text="hello", prompt_hash="h1")
evil = os.path.join(self.tmpdir, "etc", "passwd")
@@ -1535,7 +1549,7 @@ class TestOutputScanInternals(AdminAPITestCase):
self.assertEqual(counts, {"processed": 2, "found": 0, "added": 0, "linked": 0})
async def test_scan_internal_error_streams_error_event(self):
GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)]
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
self.api._collect_output_media_sync = _raise
body = await (await self.client.request("POST", "/prompt_manager/scan")).text()
self.assertIn('"type": "error"', body)
@@ -1566,7 +1580,7 @@ class TestOutputScanIsABackgroundJob(AdminAPITestCase):
from unittest.mock import AsyncMock, patch
_write_png(self.output_dir / "gen.png", prompt_text="a red square")
GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)]
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
with self.assertNoLogs("aiohttp.server", level="ERROR"):
with patch.object(
@@ -1586,7 +1600,7 @@ class TestOutputScanIsABackgroundJob(AdminAPITestCase):
async def test_second_request_attaches_to_the_running_scan(self):
_write_png(self.output_dir / "gen.png", prompt_text="a red square")
GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)]
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
gate, calls = self._gate_collect()
first = asyncio.ensure_future(
@@ -1609,7 +1623,7 @@ class TestOutputScanIsABackgroundJob(AdminAPITestCase):
self.assertIn('"added": 1', second_body)
async def test_status_reports_running_scan_and_its_last_event(self):
GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)]
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
gate, _ = self._gate_collect()
first = asyncio.ensure_future(
@@ -1639,7 +1653,7 @@ class TestOutputScanIsABackgroundJob(AdminAPITestCase):
async def test_scan_after_a_finished_one_starts_fresh(self):
_write_png(self.output_dir / "gen.png", prompt_text="a red square")
GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)]
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
gate, calls = self._gate_collect()
gate.set()
@@ -1650,7 +1664,7 @@ class TestOutputScanIsABackgroundJob(AdminAPITestCase):
self.assertIn('"linked": 1', body)
async def test_failed_scan_is_not_left_running(self):
GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)]
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
self.api._collect_output_media_sync = _raise
await (await self.client.request("POST", "/prompt_manager/scan")).text()
@@ -1707,7 +1721,7 @@ class TestWorkerThreadsSetting(AdminAPITestCase):
for i in range(8):
_write_png(self.output_dir / f"gen{i}.png", prompt_text=f"prompt {i}")
GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)]
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
PromptManagerConfig.WORKER_THREADS = 4
seen = set()
lock = threading.Lock()
@@ -1728,7 +1742,7 @@ class TestWorkerThreadsSetting(AdminAPITestCase):
async def test_scan_with_one_worker_still_completes(self):
_write_png(self.output_dir / "gen.png", prompt_text="solo")
GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)]
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
PromptManagerConfig.WORKER_THREADS = 1
body = await (await self.client.request("POST", "/prompt_manager/scan")).text()
@@ -1777,7 +1791,7 @@ class TestScanReadsVideos(AdminAPITestCase):
_write_mp4(
self.output_dir / "clip.mp4", json.dumps({"prompt": json.dumps(graph)})
)
GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)]
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
with self.assertNoLogs(level="ERROR"):
body = await (
@@ -1795,7 +1809,7 @@ class TestScanReadsVideos(AdminAPITestCase):
from utils import video_metadata
(self.output_dir / "clip.mp4").write_bytes(b"\x00\x00\x00\x18ftypisom")
GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)]
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
with patch.object(video_metadata.shutil, "which", return_value=None):
with self.assertNoLogs("prompt_manager", level="WARNING"):
@@ -1808,7 +1822,7 @@ class TestScanReadsVideos(AdminAPITestCase):
async def test_unreadable_image_is_a_warning_not_an_error(self):
(self.output_dir / "bad.png").write_bytes(b"\x89PNG not really")
GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)]
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
with self.assertLogs(level="WARNING") as cm:
body = await (
@@ -1837,7 +1851,7 @@ class TestDuplicateScanIsABackgroundJob(AdminAPITestCase):
(self.output_dir / "a.png").write_bytes(b"same bytes")
(self.output_dir / "b.png").write_bytes(b"same bytes")
(self.output_dir / "c.png").write_bytes(b"other bytes")
GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)]
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
def _gate_hashing(self):
import threading
@@ -1906,7 +1920,7 @@ class TestDuplicateScanIsABackgroundJob(AdminAPITestCase):
self.assertEqual(data, {"success": True, "running": False, "last_event": None})
async def test_failed_scan_streams_an_error_and_releases_the_lock(self):
GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)]
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.output_dir)]
self.api._find_comfyui_output_dir = _raise
body = await (await self.client.request("POST", self.STREAM)).text()
+4 -2
View File
@@ -1058,8 +1058,10 @@ class TestMaintenanceOperations(DatabaseTestCase):
self.db.link_image_to_prompt(pid, "loose.png")
flat = self.db.get_prompt_subfolders(root_dirs=[root])
self.assertIn(os.path.join("2026", "08-Aug", "2026-08-06"), flat)
self.assertIn(os.path.join(os.sep, "elsewhere", "deep"), flat)
self.assertIn("2026/08-Aug/2026-08-06", flat)
self.assertIn(
os.path.join(os.sep, "elsewhere", "deep").replace(os.sep, "/"), flat
)
self.assertNotIn(".", flat)
nested = self.db.get_prompt_subfolders(root_dirs=[root], include_ancestors=True)
+1 -1
View File
@@ -84,7 +84,7 @@ class GalleryDiagnostics:
db_path = "prompts.db"
if not os.path.isabs(db_path):
db_path = os.path.join(extension_root, db_path)
db_path = os.path.normpath(os.path.join(extension_root, db_path))
return db_path
def run_full_diagnostic(self) -> Dict[str, Any]: