fix(system): keep health reports per database and off the event loop
- A restore that replaces the database forgets the stored health report, and a check that started before the swap is not kept (database generation stamped at start). - The Overview no longer walks every source for the missing-thumbnail count; the page reads /images/thumbnails/status on its own. - Image sources for the health check and the Overview resolve in the executor; a failure to start the check answers the coded envelope. - Any exception fails only its own check; missing files reads its rows in the snapshot and stats the files after it ended. - The read-only URI keeps Windows UNC paths (empty authority). - An integrity count at SQLite's limit is flagged as capped.
This commit is contained in:
+74
-25
@@ -2,19 +2,25 @@
|
||||
|
||||
``run_health_checks`` opens its own ``mode=ro`` connection, reads one
|
||||
snapshot and closes the connection in ``finally``; it never writes. Each
|
||||
check reports ``passed``, ``issues`` (with an exact ``count``),
|
||||
check reports ``passed``, ``issues`` (with an exact ``count``; the
|
||||
integrity check stops at ``INTEGRITY_LIMIT`` problems and then says
|
||||
``capped``),
|
||||
``unavailable`` (it could not be checked here, so ``count`` is None, or
|
||||
``missing_files`` checked only some sources and found nothing) or ``failed``
|
||||
(it raised). ``missing_files`` also reports each source and whether the check
|
||||
was ``partial``. At most ``MAX_DETAIL`` detail strings are returned per
|
||||
(it raised; the other checks still run). ``missing_files`` also reports each
|
||||
source and whether the check was ``partial``; it reads the image rows inside
|
||||
the snapshot and checks their files after the snapshot ended, so a slow source
|
||||
never holds the WAL. At most ``MAX_DETAIL`` detail strings are returned per
|
||||
check, and none of them carries a server path.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import logging
|
||||
import ntpath
|
||||
import os
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
from pathlib import PurePosixPath, PureWindowsPath
|
||||
from urllib.parse import quote
|
||||
|
||||
try:
|
||||
from ..utils.hashing import generate_prompt_hash
|
||||
@@ -83,13 +89,33 @@ def _counted(lines):
|
||||
return _result(count, detail)
|
||||
|
||||
|
||||
def readonly_uri(db_path, *, windows=None):
|
||||
"""The ``mode=ro`` SQLite URI of *db_path*, on this platform or the given one.
|
||||
|
||||
pathlib spells a UNC path ``\\\\server\\share\\x.db`` as
|
||||
``file://server/share/x.db``, a remote authority SQLite refuses to open.
|
||||
With an empty authority (``file:////server/share/x.db``) SQLite hands
|
||||
Windows the ``//server/share`` path, which it opens as the UNC path.
|
||||
"""
|
||||
if windows is None:
|
||||
windows = os.name == "nt"
|
||||
db_path = os.path.abspath(db_path)
|
||||
if not windows:
|
||||
return PurePosixPath(db_path).as_uri() + "?mode=ro"
|
||||
path = PureWindowsPath(db_path)
|
||||
if ntpath.splitdrive(str(path))[0].startswith("\\\\"):
|
||||
return "file:" + quote("//" + path.as_posix()) + "?mode=ro"
|
||||
return path.as_uri() + "?mode=ro"
|
||||
|
||||
|
||||
def _integrity(conn, _roots):
|
||||
rows = [
|
||||
row[0] for row in conn.execute(f"PRAGMA integrity_check({INTEGRITY_LIMIT})")
|
||||
]
|
||||
if rows == ["ok"]:
|
||||
return _result(0, [])
|
||||
return _result(len(rows), rows)
|
||||
return {**_result(0, []), "capped": False}
|
||||
# SQLite stops listing at the limit: there may be more problems than that.
|
||||
return {**_result(len(rows), rows), "capped": len(rows) >= INTEGRITY_LIMIT}
|
||||
|
||||
|
||||
def _foreign_keys(conn, _roots):
|
||||
@@ -180,10 +206,24 @@ def _missing_files(conn, roots):
|
||||
readable sources' total (None when none is readable) and ``partial`` says
|
||||
some sources were not checked. Links outside every source are named in the
|
||||
detail, not counted.
|
||||
|
||||
Inside the snapshot this only reads the image rows; it returns the step
|
||||
that checks their files, which runs after the snapshot ended.
|
||||
"""
|
||||
if not roots:
|
||||
result = _result(None, ["No image source is available."], status="unavailable")
|
||||
return {**result, "partial": False, "sources": []}
|
||||
rows = []
|
||||
cursor = conn.execute(
|
||||
"SELECT id,image_path,filename,file_path FROM generated_images ORDER BY id"
|
||||
)
|
||||
while batch := cursor.fetchmany(BATCH_ROWS):
|
||||
rows.extend(batch)
|
||||
return lambda: _check_files(rows, roots)
|
||||
|
||||
|
||||
def _check_files(rows, roots):
|
||||
"""The ``missing_files`` result of the image *rows*, from the file system."""
|
||||
readable = [_readable(root) for root in roots]
|
||||
sources = [
|
||||
{
|
||||
@@ -201,19 +241,15 @@ def _missing_files(conn, roots):
|
||||
return {**result, "partial": False, "sources": sources}
|
||||
prefixes = [_root_prefix(root) for root in roots]
|
||||
outside, missing = 0, []
|
||||
cursor = conn.execute(
|
||||
"SELECT id,image_path,filename,file_path FROM generated_images ORDER BY id"
|
||||
)
|
||||
while rows := cursor.fetchmany(BATCH_ROWS):
|
||||
for image_id, image_path, filename, file_path in rows:
|
||||
owner = _image_owner(image_path, file_path, prefixes)
|
||||
if owner is None:
|
||||
outside += 1
|
||||
elif readable[owner] and not os.path.exists(image_path):
|
||||
sources[owner]["count"] += 1
|
||||
if len(missing) < MAX_DETAIL:
|
||||
name = filename or os.path.basename(image_path)
|
||||
missing.append(f"Image {image_id}: {name} is missing")
|
||||
for image_id, image_path, filename, file_path in rows:
|
||||
owner = _image_owner(image_path, file_path, prefixes)
|
||||
if owner is None:
|
||||
outside += 1
|
||||
elif readable[owner] and not os.path.exists(image_path):
|
||||
sources[owner]["count"] += 1
|
||||
if len(missing) < MAX_DETAIL:
|
||||
name = filename or os.path.basename(image_path)
|
||||
missing.append(f"Image {image_id}: {name} is missing")
|
||||
count = sum(source["count"] or 0 for source in sources)
|
||||
detail = [_unavailable_line(unavailable, partial=True)] if unavailable else []
|
||||
detail += missing
|
||||
@@ -248,11 +284,24 @@ def _failed():
|
||||
return _result(None, [CHECK_FAILED], status="failed")
|
||||
|
||||
|
||||
def _run_check(check_id, step, *args):
|
||||
"""One check's result, or None (reported as failed) when it raised."""
|
||||
try:
|
||||
return step(*args)
|
||||
except Exception: # any error fails this check only; the others still run
|
||||
logger.exception(f"Health check {check_id} failed")
|
||||
return None
|
||||
|
||||
|
||||
def _run_database_checks(db_path, roots):
|
||||
"""Each database check's result (None when it raised), from one snapshot."""
|
||||
"""Each database check's result (None when it raised), from one snapshot.
|
||||
|
||||
A check may return a step instead of its result: the step runs after the
|
||||
snapshot ended and the connection closed (``missing_files``' file checks).
|
||||
"""
|
||||
results = dict.fromkeys(_DATABASE_CHECKS)
|
||||
try:
|
||||
conn = sqlite3.connect(Path(db_path).as_uri() + "?mode=ro", uri=True)
|
||||
conn = sqlite3.connect(readonly_uri(db_path), uri=True)
|
||||
except sqlite3.Error:
|
||||
logger.exception("Health check could not open the database")
|
||||
return results
|
||||
@@ -260,14 +309,14 @@ def _run_database_checks(db_path, roots):
|
||||
conn.execute("BEGIN") # one read snapshot for every check
|
||||
for check_id in _DATABASE_CHECKS:
|
||||
runner = globals()["_" + check_id] # looked up per run (tests patch it)
|
||||
try:
|
||||
results[check_id] = runner(conn, roots)
|
||||
except (sqlite3.Error, OSError):
|
||||
logger.exception(f"Health check {check_id} failed")
|
||||
results[check_id] = _run_check(check_id, runner, conn, roots)
|
||||
except sqlite3.Error:
|
||||
logger.exception("Health check could not read the database")
|
||||
finally:
|
||||
conn.close()
|
||||
for check_id, result in results.items():
|
||||
if callable(result):
|
||||
results[check_id] = _run_check(check_id, result)
|
||||
return results
|
||||
|
||||
|
||||
|
||||
+4
-1
@@ -328,8 +328,11 @@ class PromptManagerAPI(
|
||||
self._scan_job = None
|
||||
# The current duplicate scan (a py/api/jobs.Job outside the registry).
|
||||
self._duplicate_job = None
|
||||
# The last System health report this session (py/api/system.py).
|
||||
# The last System health report this session and the database
|
||||
# generation it describes, bumped when a restore replaces self.db
|
||||
# (py/api/system.py).
|
||||
self._health_report = None
|
||||
self._db_generation = 0
|
||||
|
||||
# Run cleanup on initialization to remove any existing duplicates
|
||||
try:
|
||||
|
||||
@@ -1271,6 +1271,7 @@ class AdminRoutesMixin:
|
||||
self.logger.info(f"Removed {len(removed)} old safety backups")
|
||||
previous = self.db
|
||||
self.db = PromptDatabase(model.db_path)
|
||||
self._database_replaced() # a health report described the old one
|
||||
# Its connections would otherwise linger (a file lock on Windows)
|
||||
previous.close_all()
|
||||
prompt_count = self.db.model.get_database_info().get("total_prompts", 0)
|
||||
|
||||
+41
-21
@@ -4,10 +4,14 @@
|
||||
(``database/health.py``) off-loop as a ``read`` job of kind ``health``: one
|
||||
at a time, beside a scan, not cancellable and with no file outcomes. The
|
||||
API keeps the last report for this session; ``GET`` returns it or null.
|
||||
A report describes the database it was taken from: replacing the database
|
||||
(a restore) forgets it, and a check that started before the swap is not
|
||||
kept.
|
||||
- ``GET /prompt_manager/system/overview`` is the System › Overview aggregate.
|
||||
Every value is real or null: there is no backup status until backups report
|
||||
one, no health result until a check ran, and no Needs-review count until
|
||||
the review queue exists.
|
||||
the review queue exists. The missing-thumbnail count is not part of it:
|
||||
the page reads ``/images/thumbnails/status``, which walks every source.
|
||||
- ``GET /prompt_manager/system/statistics`` lists runs and ratings per
|
||||
category, with the legacy ``/stats`` totals and the Overview's count of
|
||||
rated images (images whose linked prompt is rated).
|
||||
@@ -17,6 +21,7 @@ New routes answer errors with ``{success:false, code, error}``.
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from collections import namedtuple
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
@@ -31,6 +36,10 @@ _HEALTH_STATUS_LINES = {
|
||||
}
|
||||
|
||||
|
||||
# The last health report and the database generation it was taken from.
|
||||
_StoredReport = namedtuple("_StoredReport", "generation report")
|
||||
|
||||
|
||||
def _failure(message, status=500, code=None):
|
||||
return web.json_response(
|
||||
{"success": False, "code": code, "error": message}, status=status
|
||||
@@ -89,14 +98,36 @@ class SystemRoutesMixin:
|
||||
roots.append(resolved)
|
||||
return roots
|
||||
|
||||
def _database_replaced(self):
|
||||
"""Forget what described the previous database (call after swapping ``db``)."""
|
||||
self._db_generation += 1
|
||||
self._health_report = None
|
||||
|
||||
def _current_health_report(self):
|
||||
"""The last report, or None when none was kept for the current database."""
|
||||
stored = self._health_report
|
||||
generation = self._db_generation
|
||||
if isinstance(stored, _StoredReport) and stored.generation == generation:
|
||||
return stored.report
|
||||
return None
|
||||
|
||||
async def start_health_check(self, request):
|
||||
"""Start the health check job (202), or 409 naming the running check."""
|
||||
roots = self._health_roots()
|
||||
generation = self._db_generation
|
||||
db = self.db
|
||||
try:
|
||||
roots = await self._run_in_executor(self._health_roots)
|
||||
except Exception:
|
||||
self.logger.exception("Health check could not resolve the image sources")
|
||||
return _failure("Failed to start the health check")
|
||||
|
||||
async def run(job):
|
||||
job.update(status_line="Checking the database and image links")
|
||||
report = await self._run_in_executor(self.db.run_health_checks, roots)
|
||||
self._health_report = report
|
||||
report = await self._run_in_executor(db.run_health_checks, roots)
|
||||
if generation == self._db_generation:
|
||||
self._health_report = _StoredReport(generation, report)
|
||||
else:
|
||||
self.logger.info("Health report dropped: the database was replaced")
|
||||
job.update(progress=100, status_line=_HEALTH_STATUS_LINES[report["status"]])
|
||||
|
||||
try:
|
||||
@@ -113,29 +144,19 @@ class SystemRoutesMixin:
|
||||
return web.json_response({"success": True, "job": job.summary()}, status=202)
|
||||
|
||||
async def get_health_report(self, request):
|
||||
return web.json_response({"success": True, "report": self._health_report})
|
||||
return web.json_response(
|
||||
{"success": True, "report": self._current_health_report()}
|
||||
)
|
||||
|
||||
# ── overview ──────────────────────────────────────────────────────
|
||||
|
||||
def _missing_thumbnails(self, output_dirs):
|
||||
"""Pending thumbnails as ``/images/thumbnails/status`` counts them."""
|
||||
if not output_dirs:
|
||||
return None
|
||||
try:
|
||||
plan = self._plan_gallery_thumbnails(output_dirs)
|
||||
except OSError:
|
||||
self.logger.warning("Overview could not plan thumbnails", exc_info=True)
|
||||
return None
|
||||
return len(plan["targets"]) + plan["remaining"]
|
||||
|
||||
async def get_system_overview(self, request):
|
||||
from ..config import GalleryConfig
|
||||
|
||||
try:
|
||||
output_dirs, auto_detected = self._output_dirs_with_origin()
|
||||
counts, missing = await asyncio.gather(
|
||||
(output_dirs, auto_detected), counts = await asyncio.gather(
|
||||
self._run_in_executor(self._output_dirs_with_origin),
|
||||
self._run_in_executor(self.db.library_counts),
|
||||
self._run_in_executor(self._missing_thumbnails, output_dirs),
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception("System overview failed")
|
||||
@@ -151,7 +172,7 @@ class SystemRoutesMixin:
|
||||
"indexed": counts["images"],
|
||||
"rated": counts["rated_images"],
|
||||
},
|
||||
"health": health_summary(self._health_report),
|
||||
"health": health_summary(self._current_health_report()),
|
||||
"backup": None, # no verified backup status exists yet (S8)
|
||||
"needs_review": {"pending": None}, # no review queue yet (Q6)
|
||||
"sources": {
|
||||
@@ -159,7 +180,6 @@ class SystemRoutesMixin:
|
||||
"effective": len(output_dirs),
|
||||
"auto_detected": auto_detected,
|
||||
},
|
||||
"thumbnails": {"missing": missing},
|
||||
},
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
|
||||
@@ -9,6 +9,7 @@ could not run into a count.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import ntpath
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
@@ -289,6 +290,94 @@ class TestHealthReport(HealthTestCase):
|
||||
[f"Positive prompt {positive} with no negative: 3 pair rows"],
|
||||
)
|
||||
|
||||
def test_integrity_count_at_the_limit_is_flagged_capped(self):
|
||||
class Rows:
|
||||
def __init__(self, rows):
|
||||
self.rows = rows
|
||||
|
||||
def execute(self, _sql):
|
||||
return iter([(row,) for row in self.rows])
|
||||
|
||||
capped = health._integrity(Rows(["bad"] * health.INTEGRITY_LIMIT), [])
|
||||
self.assertEqual(capped["status"], "issues")
|
||||
self.assertEqual(capped["count"], health.INTEGRITY_LIMIT)
|
||||
self.assertTrue(capped["capped"])
|
||||
exact = health._integrity(Rows(["bad"] * 3), [])
|
||||
self.assertEqual(exact["count"], 3)
|
||||
self.assertFalse(exact["capped"])
|
||||
self.assertFalse(health._integrity(Rows(["ok"]), [])["capped"])
|
||||
|
||||
def test_an_unexpected_error_fails_only_its_own_check(self):
|
||||
self.save("Any")
|
||||
with (
|
||||
patch.object(health, "_pair_uniqueness", side_effect=KeyError("boom")),
|
||||
self.assertLogs("prompt_manager.database.health", "ERROR"),
|
||||
):
|
||||
report = self.run_checks(review_count=0)
|
||||
statuses = {c["id"]: c["status"] for c in report["checks"]}
|
||||
self.assertEqual(statuses.pop("pair_uniqueness"), "failed")
|
||||
self.assertEqual(set(statuses.values()), {"passed"})
|
||||
|
||||
|
||||
class TestDatabaseUri(unittest.TestCase):
|
||||
"""The ``mode=ro`` URI of the database file, for every path form SQLite opens."""
|
||||
|
||||
def test_posix_path_is_a_local_file_uri(self):
|
||||
self.assertEqual(
|
||||
health.readonly_uri("/data/my db#1?.db", windows=False),
|
||||
"file:///data/my%20db%231%3F.db?mode=ro",
|
||||
)
|
||||
|
||||
def test_windows_drive_path(self):
|
||||
self.assertEqual(
|
||||
health.readonly_uri(r"C:\Users\Me\prompts.db", windows=True),
|
||||
"file:///C:/Users/Me/prompts.db?mode=ro",
|
||||
)
|
||||
|
||||
def test_windows_unc_path_has_an_empty_authority(self):
|
||||
# pathlib spells \\server\share as file://server/share, which SQLite
|
||||
# rejects (a non-local authority); an empty one keeps the UNC path.
|
||||
uri = health.readonly_uri(r"\\server\share\Comfy UI\prompts.db", windows=True)
|
||||
self.assertEqual(uri, "file:////server/share/Comfy%20UI/prompts.db?mode=ro")
|
||||
self.assertTrue(ntpath.isabs(r"\\server\share\Comfy UI\prompts.db"))
|
||||
|
||||
def test_uri_opens_a_path_with_uri_characters(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
path = os.path.join(os.path.realpath(temp), "a #b?c%20.db")
|
||||
conn = sqlite3.connect(path)
|
||||
try:
|
||||
conn.execute("CREATE TABLE t(a)")
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
ro = sqlite3.connect(health.readonly_uri(path), uri=True)
|
||||
try:
|
||||
self.assertEqual(ro.execute("SELECT COUNT(*) FROM t").fetchone(), (0,))
|
||||
with self.assertRaises(sqlite3.OperationalError):
|
||||
ro.execute("INSERT INTO t VALUES(1)")
|
||||
finally:
|
||||
ro.close()
|
||||
|
||||
@unittest.skipUnless(os.name == "nt", "a native UNC path exists on Windows only")
|
||||
def test_native_unc_path_opens(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
path = os.path.join(os.path.realpath(temp), "prompts.db")
|
||||
conn = sqlite3.connect(path)
|
||||
try:
|
||||
conn.execute("CREATE TABLE t(a)")
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
drive, rest = ntpath.splitdrive(path)
|
||||
unc = "\\\\localhost\\" + drive[0] + "$" + rest
|
||||
if not os.path.exists(unc):
|
||||
self.skipTest("the administrative share is not reachable here")
|
||||
ro = sqlite3.connect(health.readonly_uri(unc), uri=True)
|
||||
try:
|
||||
self.assertEqual(ro.execute("SELECT COUNT(*) FROM t").fetchone(), (0,))
|
||||
finally:
|
||||
ro.close()
|
||||
|
||||
|
||||
class TestMissingFiles(HealthTestCase):
|
||||
def test_missing_files_counted_under_readable_sources_only(self):
|
||||
@@ -417,6 +506,55 @@ class TestMissingFiles(HealthTestCase):
|
||||
self.assertEqual(missing["sources"], [])
|
||||
self.assertFalse(missing["partial"])
|
||||
|
||||
def test_a_bad_stored_path_fails_only_missing_files(self):
|
||||
prompt = self.save("Images")
|
||||
self.image(prompt, "here.png")
|
||||
with self.raw() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO generated_images(prompt_id,image_path,filename)"
|
||||
" VALUES(?,?,?)",
|
||||
(prompt, str(self.root / "else\x00where.png"), "bad.png"),
|
||||
)
|
||||
conn.close()
|
||||
with self.assertLogs("prompt_manager.database.health", "ERROR"):
|
||||
report = self.run_checks(review_count=0)
|
||||
statuses = {c["id"]: c["status"] for c in report["checks"]}
|
||||
self.assertEqual(statuses.pop("missing_files"), "failed")
|
||||
self.assertEqual(set(statuses.values()), {"passed"})
|
||||
|
||||
def test_files_are_checked_after_the_read_snapshot_ends(self):
|
||||
prompt = self.save("Images")
|
||||
self.image(prompt, "here.png")
|
||||
self.image(prompt, "gone.png", exists=False)
|
||||
opened = []
|
||||
real_connect = sqlite3.connect
|
||||
|
||||
def connect(*args, **kwargs):
|
||||
conn = real_connect(*args, **kwargs)
|
||||
opened.append(conn)
|
||||
return conn
|
||||
|
||||
real_exists = os.path.exists
|
||||
in_snapshot = []
|
||||
|
||||
def exists(path):
|
||||
for conn in opened:
|
||||
try:
|
||||
in_snapshot.append(conn.in_transaction)
|
||||
except sqlite3.ProgrammingError: # closed: no snapshot
|
||||
in_snapshot.append(False)
|
||||
return real_exists(path)
|
||||
|
||||
with (
|
||||
patch.object(health.sqlite3, "connect", side_effect=connect),
|
||||
patch.object(health.os.path, "exists", side_effect=exists),
|
||||
):
|
||||
missing = self.check(self.run_checks(review_count=0), "missing_files")
|
||||
self.assertEqual((missing["status"], missing["count"]), ("issues", 1))
|
||||
self.assertEqual(len(opened), 1)
|
||||
self.assertTrue(in_snapshot)
|
||||
self.assertNotIn(True, in_snapshot)
|
||||
|
||||
|
||||
class TestAmbiguousMetadata(HealthTestCase):
|
||||
def test_ambiguous_metadata_unavailable_without_review_queue(self):
|
||||
|
||||
+116
-25
@@ -3,12 +3,15 @@
|
||||
The health check runs off-loop as a ``read`` registry job of kind ``health``
|
||||
(one at a time, beside a scan), and the API keeps its last report. The
|
||||
Overview reads only real values: Library-eligible counts, indexed images, the
|
||||
last health result or null, no backup until one exists, the effective sources
|
||||
and whether they were auto-detected, and the thumbnail plan's missing count.
|
||||
last health result or null, no backup until one exists, and the effective
|
||||
sources and whether they were auto-detected. The missing-thumbnail count is not
|
||||
part of it: the Overview reads ``/images/thumbnails/status`` on its own (its
|
||||
walk of every source must not hold the other values back).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
import threading
|
||||
from pathlib import Path
|
||||
@@ -57,6 +60,25 @@ class SystemAPITestCase(AdminAPITestCase):
|
||||
self.assertEqual(response.status, status, await response.text())
|
||||
return await response.json()
|
||||
|
||||
async def restore_snapshot(self):
|
||||
"""Restore the live database from a copy of itself through the legacy route."""
|
||||
from contextlib import closing
|
||||
|
||||
from aiohttp import FormData
|
||||
|
||||
snapshot = os.path.join(self.tmpdir, "snapshot.db")
|
||||
with closing(sqlite3.connect(self.api.db.model.db_path)) as src:
|
||||
with closing(sqlite3.connect(snapshot)) as dst:
|
||||
src.backup(dst)
|
||||
with open(snapshot, "rb") as handle:
|
||||
upload = handle.read()
|
||||
form = FormData()
|
||||
form.add_field("database_file", upload, filename="upload.db")
|
||||
response = await self.client.request(
|
||||
"POST", "/prompt_manager/restore", data=form
|
||||
)
|
||||
self.assertEqual(response.status, 200, await response.text())
|
||||
|
||||
async def run_health(self):
|
||||
body = await self.json("POST", "/prompt_manager/system/health", 202)
|
||||
job = self.api._jobs.get(body["job"]["id"])
|
||||
@@ -159,6 +181,77 @@ class TestHealthRoute(SystemAPITestCase):
|
||||
)
|
||||
self.assertEqual(report["status"], "incomplete")
|
||||
|
||||
async def test_restore_clears_health_report(self):
|
||||
self.save("Any")
|
||||
await self.run_health()
|
||||
self.assertIsNotNone(
|
||||
(await self.json("GET", "/prompt_manager/system/health"))["report"]
|
||||
)
|
||||
await self.restore_snapshot()
|
||||
body = await self.json("GET", "/prompt_manager/system/health")
|
||||
self.assertEqual(body, {"success": True, "report": None})
|
||||
overview = await self.json("GET", "/prompt_manager/system/overview")
|
||||
self.assertIsNone(overview["health"])
|
||||
|
||||
async def test_check_finishing_after_a_restore_is_not_kept(self):
|
||||
# A check that started on the old database describes it, not the new one.
|
||||
self.save("Any")
|
||||
release = threading.Event()
|
||||
entered = threading.Event()
|
||||
real = self.api.db.run_health_checks
|
||||
|
||||
def gated(*args, **kwargs):
|
||||
entered.set()
|
||||
release.wait(WAIT)
|
||||
return real(*args, **kwargs)
|
||||
|
||||
with patch.object(self.api.db, "run_health_checks", side_effect=gated):
|
||||
body = await self.json("POST", "/prompt_manager/system/health", 202)
|
||||
loop = asyncio.get_running_loop()
|
||||
await loop.run_in_executor(None, entered.wait, WAIT)
|
||||
await self.restore_snapshot()
|
||||
release.set()
|
||||
job = self.api._jobs.get(body["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(job.task), WAIT)
|
||||
self.assertEqual(job.state, "finished")
|
||||
body = await self.json("GET", "/prompt_manager/system/health")
|
||||
self.assertEqual(body, {"success": True, "report": None})
|
||||
# A check started on the restored database is kept again.
|
||||
await self.run_health()
|
||||
self.assertIsNotNone(
|
||||
(await self.json("GET", "/prompt_manager/system/health"))["report"]
|
||||
)
|
||||
|
||||
async def test_health_sources_resolve_off_the_event_loop(self):
|
||||
loop_thread = threading.get_ident()
|
||||
threads = []
|
||||
real = self.api._health_roots
|
||||
|
||||
def recording():
|
||||
threads.append(threading.get_ident())
|
||||
return real()
|
||||
|
||||
with patch.object(self.api, "_health_roots", side_effect=recording):
|
||||
await self.run_health()
|
||||
self.assertEqual(len(threads), 1)
|
||||
self.assertNotEqual(threads[0], loop_thread)
|
||||
|
||||
async def test_health_start_failure_is_an_error_envelope(self):
|
||||
with (
|
||||
patch.object(self.api, "_health_roots", side_effect=RuntimeError("secret")),
|
||||
self.assertLogs("prompt_manager.api", "ERROR"),
|
||||
):
|
||||
body = await self.json("POST", "/prompt_manager/system/health", 500)
|
||||
self.assertEqual(
|
||||
body,
|
||||
{
|
||||
"success": False,
|
||||
"code": None,
|
||||
"error": "Failed to start the health check",
|
||||
},
|
||||
)
|
||||
self.assertEqual(self.api._jobs.list(), [])
|
||||
|
||||
|
||||
class TestOverviewRoute(SystemAPITestCase):
|
||||
async def test_overview_reports_real_counts(self):
|
||||
@@ -184,7 +277,6 @@ class TestOverviewRoute(SystemAPITestCase):
|
||||
"backup",
|
||||
"needs_review",
|
||||
"sources",
|
||||
"thumbnails",
|
||||
},
|
||||
)
|
||||
|
||||
@@ -241,32 +333,31 @@ class TestOverviewRoute(SystemAPITestCase):
|
||||
body["sources"], {"configured": 1, "effective": 1, "auto_detected": True}
|
||||
)
|
||||
|
||||
async def test_overview_thumbnails_follow_the_status_plan(self):
|
||||
async def test_overview_never_walks_the_sources_for_thumbnails(self):
|
||||
(self.output_dir / "one.png").write_bytes(b"png")
|
||||
(self.output_dir / "two.png").write_bytes(b"png")
|
||||
status = await self.json("GET", "/prompt_manager/images/thumbnails/status")
|
||||
body = await self.json("GET", "/prompt_manager/system/overview")
|
||||
self.assertEqual(body["thumbnails"], {"missing": status["pending"]})
|
||||
self.assertEqual(body["thumbnails"]["missing"], 2)
|
||||
|
||||
async def test_overview_thumbnails_unknown_without_sources(self):
|
||||
self.output_dir.rmdir()
|
||||
body = await self.json("GET", "/prompt_manager/system/overview")
|
||||
self.assertEqual(body["thumbnails"], {"missing": None})
|
||||
self.assertEqual(
|
||||
body["sources"], {"configured": 0, "effective": 0, "auto_detected": False}
|
||||
)
|
||||
|
||||
async def test_overview_thumbnail_failure_is_unknown_not_zero(self):
|
||||
with (
|
||||
patch.object(
|
||||
self.api, "_plan_gallery_thumbnails", side_effect=OSError("denied")
|
||||
),
|
||||
self.assertLogs("prompt_manager.api", "WARNING"),
|
||||
with patch.object(
|
||||
self.api,
|
||||
"_plan_gallery_thumbnails",
|
||||
side_effect=AssertionError("walked"),
|
||||
):
|
||||
body = await self.json("GET", "/prompt_manager/system/overview")
|
||||
self.assertTrue(body["success"])
|
||||
self.assertEqual(body["thumbnails"], {"missing": None})
|
||||
self.assertNotIn("thumbnails", body)
|
||||
|
||||
async def test_overview_sources_resolve_off_the_event_loop(self):
|
||||
loop_thread = threading.get_ident()
|
||||
threads = []
|
||||
real = self.api._output_dirs_with_origin
|
||||
|
||||
def recording():
|
||||
threads.append(threading.get_ident())
|
||||
return real()
|
||||
|
||||
with patch.object(self.api, "_output_dirs_with_origin", side_effect=recording):
|
||||
body = await self.json("GET", "/prompt_manager/system/overview")
|
||||
self.assertEqual(body["sources"]["effective"], 1)
|
||||
self.assertEqual(len(threads), 1)
|
||||
self.assertNotEqual(threads[0], loop_thread)
|
||||
|
||||
async def test_overview_failure_is_an_error_envelope(self):
|
||||
with (
|
||||
|
||||
Reference in New Issue
Block a user