feat(maintenance): reviewed, read-only maintenance plans
POST /prompt_manager/maintenance/preview reviews duplicate_prompts, optimize, unused_components or missing_links on its own mode=ro snapshot and answers a Plan (scope, protected reasons, backup policy, media, at most 50 detail rows). The fingerprint, targets and judged sources stay in a server-side PlanStore under a single-use token (15 min, 8 plans, 16 MiB, never truncated; a restart forgets them), bound to the database it read. The previews reuse the legacy queries: duplicate groups and the merge are shared with cleanup_duplicates, ORPHANED_PROMPTS_SQL is built from the #239 protection predicates the plan reports, and missing links are judged by the health check's link_states. Legacy /maintenance keeps its shape; vacuum now reports success:false when VACUUM fails, and cleanup_orphaned_images skips links under unavailable sources.
This commit is contained in:
Vito Sansevero
@@ -0,0 +1,273 @@
|
||||
"""Maintenance routes (py/api/maintenance.py, #249 System S6).
|
||||
|
||||
``POST /prompt_manager/maintenance/preview`` answers a reviewed ``Plan`` for
|
||||
one of four operations and writes nothing; its token binds a later apply to
|
||||
exactly that plan. Errors are coded and never carry a server path or an
|
||||
exception text.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
_mock_server = MagicMock()
|
||||
_mock_server.PromptServer.instance.routes = MagicMock()
|
||||
sys.modules.setdefault("server", _mock_server)
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from database.operations import PromptDatabase # noqa: E402
|
||||
from py.api import PromptManagerAPI # noqa: E402
|
||||
from py.api.maintenance_plans import PlanExpired # noqa: E402
|
||||
from py.config import GalleryConfig # noqa: E402
|
||||
from utils.hashing import generate_prompt_hash # noqa: E402
|
||||
|
||||
try:
|
||||
from tests.test_api_admin import AdminAPITestCase # noqa: E402
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from test_api_admin import AdminAPITestCase # noqa: E402
|
||||
|
||||
WAIT = 10
|
||||
PREVIEW = "/prompt_manager/maintenance/preview"
|
||||
OPERATIONS = ("duplicate_prompts", "optimize", "unused_components", "missing_links")
|
||||
PLAN_KEYS = {
|
||||
"success",
|
||||
"token",
|
||||
"operation",
|
||||
"expires_at",
|
||||
"scope",
|
||||
"protected",
|
||||
"backup",
|
||||
"media",
|
||||
"details",
|
||||
}
|
||||
|
||||
|
||||
def _digest(path):
|
||||
if not os.path.exists(path):
|
||||
return None
|
||||
with open(path, "rb") as handle:
|
||||
return hashlib.sha256(handle.read()).hexdigest()
|
||||
|
||||
|
||||
class MaintenanceAPITestCase(AdminAPITestCase):
|
||||
async def get_application(self):
|
||||
app = await super().get_application()
|
||||
self.api._find_comfyui_output_dir = lambda: (
|
||||
str(self.output_dir) if self.output_dir.is_dir() else None
|
||||
)
|
||||
return app
|
||||
|
||||
def save(self, text, **metadata):
|
||||
return self.api.db.save_prompt(
|
||||
text, prompt_hash=generate_prompt_hash(text), **metadata
|
||||
)
|
||||
|
||||
def image(self, prompt_id, name, *, exists=True, folder=None):
|
||||
path = Path(folder or self.output_dir) / name
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if exists:
|
||||
path.write_bytes(b"png")
|
||||
return self.api.db.link_image_to_prompt(prompt_id, str(path))
|
||||
|
||||
async def post(self, url, body, status):
|
||||
response = await self.client.request("POST", url, json=body)
|
||||
text = await response.text()
|
||||
self.assertEqual(response.status, status, text)
|
||||
self.assertNotIn(self.tmpdir, text)
|
||||
return json.loads(text)
|
||||
|
||||
async def preview(self, operation, status=200, **body):
|
||||
return await self.post(PREVIEW, {"operation": operation, **body}, status)
|
||||
|
||||
|
||||
class TestPreviewRoute(MaintenanceAPITestCase):
|
||||
async def test_preview_answers_a_plan_for_each_operation(self):
|
||||
prompt = self.save("Kept")
|
||||
self.image(prompt, "gone.png", exists=False)
|
||||
self.save("Unused")
|
||||
for operation in OPERATIONS:
|
||||
with self.subTest(operation=operation):
|
||||
plan = await self.preview(operation, options={})
|
||||
self.assertEqual(set(plan), PLAN_KEYS)
|
||||
self.assertTrue(plan["success"])
|
||||
self.assertEqual(plan["operation"], operation)
|
||||
self.assertIsInstance(plan["token"], str)
|
||||
self.assertEqual(set(plan["scope"]), {"count", "label"})
|
||||
self.assertEqual(set(plan["protected"]), {"count", "reasons"})
|
||||
self.assertEqual(
|
||||
plan["backup"],
|
||||
{
|
||||
"policy": "on",
|
||||
"destination_label": "prompts.db.backup_<date and time>",
|
||||
},
|
||||
)
|
||||
self.assertFalse(plan["media"]["deletes_files"])
|
||||
self.assertLessEqual(len(plan["details"]), 50)
|
||||
self.assertTrue(plan["expires_at"].endswith("+00:00"))
|
||||
self.assertNotIn("fingerprint", json.dumps(plan))
|
||||
missing = await self.preview("missing_links")
|
||||
self.assertEqual(
|
||||
missing["scope"], {"count": 1, "label": "1 link to a missing file"}
|
||||
)
|
||||
unused = await self.preview("unused_components")
|
||||
self.assertEqual(unused["scope"]["count"], 1)
|
||||
|
||||
async def test_preview_writes_nothing(self):
|
||||
prompt = self.save("Kept")
|
||||
self.image(prompt, "kept.png")
|
||||
self.image(prompt, "gone.png", exists=False)
|
||||
self.save("Unused")
|
||||
db_path = self.api.db.model.db_path
|
||||
self.api.db.get_statistics()
|
||||
files = [db_path, db_path + "-wal"]
|
||||
for operation in OPERATIONS:
|
||||
with self.subTest(operation=operation):
|
||||
before = [_digest(path) for path in files]
|
||||
shm = os.path.getsize(db_path + "-shm")
|
||||
media = {p.name: _digest(p) for p in self.output_dir.iterdir()}
|
||||
await self.preview(operation)
|
||||
self.assertEqual([_digest(path) for path in files], before)
|
||||
self.assertEqual(os.path.getsize(db_path + "-shm"), shm)
|
||||
self.assertEqual(
|
||||
{p.name: _digest(p) for p in self.output_dir.iterdir()}, media
|
||||
)
|
||||
self.assertEqual(
|
||||
[p["text"] for p in self.api.db.search_prompts()], ["Unused", "Kept"]
|
||||
)
|
||||
|
||||
async def test_unknown_operation_is_400(self):
|
||||
cases = [
|
||||
{"operation": "reboot"},
|
||||
{"operation": "duplicate_files"}, # S6 Task 17, not yet
|
||||
{"operation": 3},
|
||||
{},
|
||||
{"operation": "optimize", "options": {"all": True}},
|
||||
{"operation": "optimize", "options": []},
|
||||
{"operation": "optimize", "extra": 1},
|
||||
]
|
||||
for body in cases:
|
||||
with self.subTest(body=body):
|
||||
reply = await self.post(PREVIEW, body, 400)
|
||||
self.assertEqual(reply["success"], False)
|
||||
self.assertEqual(reply["code"], "invalid-request")
|
||||
response = await self.client.request(
|
||||
"POST", PREVIEW, data=b"[1]", headers={"Content-Type": "application/json"}
|
||||
)
|
||||
self.assertEqual(response.status, 400)
|
||||
self.assertEqual((await response.json())["code"], "invalid-request")
|
||||
big = json.dumps({"operation": "optimize", "pad": "x" * 8192})
|
||||
response = await self.client.request(
|
||||
"POST", PREVIEW, data=big, headers={"Content-Type": "application/json"}
|
||||
)
|
||||
self.assertEqual(response.status, 413)
|
||||
self.assertEqual((await response.json())["code"], "too-large")
|
||||
|
||||
async def test_preview_failure_is_coded_without_exception_text(self):
|
||||
with patch.object(
|
||||
self.api.db,
|
||||
"preview_maintenance",
|
||||
side_effect=sqlite3.OperationalError(f"boom {self.tmpdir}"),
|
||||
):
|
||||
with self.assertLogs("prompt_manager", "ERROR"):
|
||||
reply = await self.preview("optimize", status=500)
|
||||
self.assertEqual(
|
||||
reply,
|
||||
{"success": False, "code": None, "error": "Failed to review maintenance"},
|
||||
)
|
||||
|
||||
async def test_scope_too_large_is_413(self):
|
||||
from py.api.maintenance_plans import PlanStore
|
||||
|
||||
self.api._maintenance_plans = PlanStore(max_bytes=10)
|
||||
self.save("Unused")
|
||||
reply = await self.preview("unused_components", status=413)
|
||||
self.assertEqual(reply["code"], "too-large")
|
||||
|
||||
async def test_preview_during_a_restore_swap_is_storage_changed(self):
|
||||
real = self.api.db.preview_maintenance
|
||||
|
||||
def swapped(*args, **kwargs):
|
||||
draft = real(*args, **kwargs)
|
||||
self.api._db_generation += 1 # a restore replaced the database
|
||||
return draft
|
||||
|
||||
with patch.object(self.api.db, "preview_maintenance", side_effect=swapped):
|
||||
reply = await self.preview("optimize", status=409)
|
||||
self.assertEqual(reply["code"], "storage-changed")
|
||||
|
||||
async def test_missing_links_preview_uses_the_health_roots(self):
|
||||
unmounted = Path(self.tmpdir) / "unmounted"
|
||||
unmounted.mkdir()
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir), str(unmounted)]
|
||||
prompt = self.save("Links")
|
||||
self.image(prompt, "gone.png", exists=False)
|
||||
self.image(prompt, "away.png", folder=unmounted, exists=False)
|
||||
unmounted.rmdir()
|
||||
plan = await self.preview("missing_links")
|
||||
self.assertEqual(plan["scope"]["count"], 1)
|
||||
self.assertEqual(
|
||||
plan["protected"],
|
||||
{"count": 1, "reasons": ["1 link under an unavailable source"]},
|
||||
)
|
||||
|
||||
async def test_restart_loses_plans(self):
|
||||
plan = await self.preview("optimize")
|
||||
self.assertEqual(
|
||||
self.api._maintenance_plans.peek(plan["token"], "optimize").operation,
|
||||
"optimize",
|
||||
)
|
||||
restarted = PromptManagerAPI(db=PromptDatabase(self.api.db.model.db_path))
|
||||
try:
|
||||
with self.assertRaises(PlanExpired):
|
||||
restarted._maintenance_plans.peek(plan["token"], "optimize")
|
||||
finally:
|
||||
restarted.db.close_all()
|
||||
|
||||
|
||||
class TestLegacyMaintenance(MaintenanceAPITestCase):
|
||||
async def test_vacuum_failure_reported_not_success(self):
|
||||
with patch(
|
||||
"database.models.sqlite3.connect",
|
||||
side_effect=sqlite3.OperationalError("database is locked"),
|
||||
):
|
||||
with self.assertLogs("prompt_manager", "ERROR"):
|
||||
reply = await self.post(
|
||||
"/prompt_manager/maintenance", {"operations": ["vacuum"]}, 200
|
||||
)
|
||||
self.assertFalse(reply["all_successful"])
|
||||
self.assertFalse(reply["results"]["vacuum"]["success"])
|
||||
self.assertEqual(reply["results"]["vacuum"]["error"], "database is locked")
|
||||
|
||||
async def test_legacy_orphaned_image_cleanup_skips_unavailable_sources(self):
|
||||
unmounted = Path(self.tmpdir) / "unmounted"
|
||||
unmounted.mkdir()
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir), str(unmounted)]
|
||||
prompt = self.save("Links")
|
||||
self.image(prompt, "gone.png", exists=False)
|
||||
away = self.image(prompt, "away.png", folder=unmounted, exists=False)
|
||||
unmounted.rmdir()
|
||||
reply = await self.post(
|
||||
"/prompt_manager/maintenance",
|
||||
{"operations": ["cleanup_orphaned_images"]},
|
||||
200,
|
||||
)
|
||||
self.assertEqual(
|
||||
reply["results"]["cleanup_orphaned_images"],
|
||||
{
|
||||
"success": True,
|
||||
"removed_count": 1,
|
||||
"message": "Removed 1 orphaned image records",
|
||||
},
|
||||
)
|
||||
self.assertIsNotNone(self.api.db.get_image_by_id(away))
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
import unittest
|
||||
|
||||
unittest.main()
|
||||
@@ -1374,13 +1374,17 @@ class TestModelHousekeeping(DatabaseTestCase):
|
||||
):
|
||||
self.assertEqual(self.db.model.get_database_info(), {})
|
||||
|
||||
def test_vacuum_logs_errors_instead_of_raising(self):
|
||||
def test_vacuum_logs_and_raises_its_failure(self):
|
||||
# S6: a failed VACUUM is reported (legacy /maintenance success:false),
|
||||
# never logged as done.
|
||||
self._save("x")
|
||||
self.db.model.vacuum_database()
|
||||
with patch(
|
||||
"database.models.sqlite3.connect", side_effect=sqlite3.OperationalError
|
||||
):
|
||||
self.db.model.vacuum_database() # logged, not raised
|
||||
with self.assertLogs("prompt_manager", "ERROR"):
|
||||
with self.assertRaises(sqlite3.OperationalError):
|
||||
self.db.model.vacuum_database()
|
||||
self.assertEqual(len(self.db.search_prompts()), 1)
|
||||
|
||||
def test_close_tolerates_a_connection_that_fails_to_close(self):
|
||||
|
||||
@@ -0,0 +1,443 @@
|
||||
"""Maintenance plans (#249 System S6): the plan store and the read-only previews.
|
||||
|
||||
``PlanStore`` holds each reviewed plan server-side under a single-use token
|
||||
for ``ttl`` seconds, within a count and byte budget it never truncates to fit.
|
||||
``PromptDatabase.preview_maintenance`` builds a plan on its own ``mode=ro``
|
||||
connection: it writes nothing (the TESTING.md read-only standard) and reuses
|
||||
the legacy maintenance queries, so a preview counts what the legacy cleanup
|
||||
would change, except that a missing-file link is only counted under a source
|
||||
that can be read now.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from database import maintenance # noqa: E402
|
||||
from database.operations import PromptDatabase # noqa: E402
|
||||
from py.api.maintenance_plans import ( # noqa: E402
|
||||
PlanExpired,
|
||||
PlanStore,
|
||||
PlanUsed,
|
||||
ScopeTooLarge,
|
||||
)
|
||||
from utils.hashing import generate_prompt_hash # noqa: E402
|
||||
|
||||
try:
|
||||
from tests import test_execution_history_database as history_tests # noqa: E402
|
||||
from tests.open_handles import assert_closed # noqa: E402
|
||||
from tests.test_execution_context_database import bundle # noqa: E402
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
import test_execution_history_database as history_tests # noqa: E402
|
||||
from open_handles import assert_closed # noqa: E402
|
||||
from test_execution_context_database import bundle # noqa: E402
|
||||
|
||||
OPERATIONS = ("duplicate_prompts", "optimize", "unused_components", "missing_links")
|
||||
|
||||
|
||||
def _digest(path):
|
||||
if not os.path.exists(path):
|
||||
return None
|
||||
with open(path, "rb") as handle:
|
||||
return hashlib.sha256(handle.read()).hexdigest()
|
||||
|
||||
|
||||
class FakeClock:
|
||||
def __init__(self):
|
||||
self.now = 1000.0
|
||||
|
||||
def __call__(self):
|
||||
return self.now
|
||||
|
||||
|
||||
class TestPlanStore(unittest.TestCase):
|
||||
def test_plan_expiry_budget_and_single_use(self):
|
||||
clock = FakeClock()
|
||||
store = PlanStore(ttl=900, max_plans=2, max_bytes=200, clock=clock)
|
||||
first = store.issue("optimize", "sha256:a", {"targets": []}, ("db", 0))
|
||||
self.assertGreaterEqual(len(first), 40) # token_urlsafe(32)
|
||||
plan = store.peek(first, "optimize")
|
||||
self.assertEqual(
|
||||
(plan.operation, plan.fingerprint, plan.db_identity),
|
||||
("optimize", "sha256:a", ("db", 0)),
|
||||
)
|
||||
# Single use: the second consume names the job the first one started.
|
||||
self.assertIs(store.consume(first, "optimize", job_id="job-1"), plan)
|
||||
with self.assertRaises(PlanUsed) as used:
|
||||
store.consume(first, "optimize", job_id="job-2")
|
||||
self.assertEqual(used.exception.job_id, "job-1")
|
||||
with self.assertRaises(PlanUsed):
|
||||
store.peek(first, "optimize")
|
||||
# Another operation's token, an unknown token and a non-string are expired.
|
||||
second = store.issue("missing_links", "sha256:b", {"targets": [1]}, ("db", 0))
|
||||
for token, operation in (
|
||||
(second, "optimize"),
|
||||
("nope", "missing_links"),
|
||||
(None, "missing_links"),
|
||||
):
|
||||
with self.subTest(token=token), self.assertRaises(PlanExpired):
|
||||
store.peek(token, operation)
|
||||
# Expiry: valid up to ttl, gone after it.
|
||||
clock.now += 900
|
||||
self.assertEqual(store.peek(second, "missing_links").fingerprint, "sha256:b")
|
||||
clock.now += 0.001
|
||||
with self.assertRaises(PlanExpired):
|
||||
store.consume(second, "missing_links", job_id="job-3")
|
||||
# Budget: a plan over max_bytes is refused whole, never truncated.
|
||||
with self.assertRaises(ScopeTooLarge):
|
||||
store.issue("unused_components", "x", {"targets": list(range(100))}, ())
|
||||
# Count budget: the oldest plan makes room for a new one and is expired.
|
||||
a = store.issue("optimize", "a", {}, ())
|
||||
b = store.issue("optimize", "b", {}, ())
|
||||
c = store.issue("optimize", "c", {}, ())
|
||||
with self.assertRaises(PlanExpired):
|
||||
store.peek(a, "optimize")
|
||||
self.assertEqual(
|
||||
[store.peek(t, "optimize").fingerprint for t in (b, c)], ["b", "c"]
|
||||
)
|
||||
# Byte budget across plans: older plans are dropped until the new one fits.
|
||||
sized = PlanStore(max_plans=8, max_bytes=100, clock=clock)
|
||||
pad = {"pad": "x" * 40} # 50 bytes as compact JSON
|
||||
p1, p2, p3 = (sized.issue("optimize", str(n), pad, ()) for n in range(3))
|
||||
with self.assertRaises(PlanExpired):
|
||||
sized.peek(p1, "optimize")
|
||||
self.assertEqual([sized.peek(t).fingerprint for t in (p2, p3)], ["1", "2"])
|
||||
|
||||
def test_expires_at_is_wall_clock(self):
|
||||
store = PlanStore(ttl=60, clock=FakeClock(), wall_clock=lambda: 5000.0)
|
||||
token = store.issue("optimize", "f", {}, ())
|
||||
self.assertEqual(store.peek(token, "optimize").expires_at, 5060.0)
|
||||
|
||||
def test_discard_forgets_a_plan(self):
|
||||
store = PlanStore()
|
||||
token = store.issue("optimize", "f", {}, ())
|
||||
store.discard(token)
|
||||
store.discard("unknown")
|
||||
with self.assertRaises(PlanExpired):
|
||||
store.peek(token, "optimize")
|
||||
|
||||
def test_used_tokens_are_remembered_within_a_bound(self):
|
||||
store = PlanStore(max_plans=2)
|
||||
tokens = []
|
||||
for index in range(store.MAX_USED + 1):
|
||||
token = store.issue("optimize", str(index), {}, ())
|
||||
store.consume(token, "optimize", job_id=f"job-{index}")
|
||||
tokens.append(token)
|
||||
with self.assertRaises(PlanExpired): # the oldest record was dropped
|
||||
store.peek(tokens[0], "optimize")
|
||||
with self.assertRaises(PlanUsed):
|
||||
store.peek(tokens[-1], "optimize")
|
||||
|
||||
|
||||
class MaintenanceDatabaseCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.root = Path(os.path.realpath(self.temp.name))
|
||||
self.output = self.root / "output"
|
||||
self.output.mkdir()
|
||||
self.db_path = str(self.root / "prompts.db")
|
||||
self.db = PromptDatabase(self.db_path)
|
||||
|
||||
def tearDown(self):
|
||||
self.db.close_all()
|
||||
assert_closed(self, self.root)
|
||||
self.temp.cleanup()
|
||||
|
||||
def save(self, text, **metadata):
|
||||
return self.db.save_prompt(
|
||||
text, prompt_hash=generate_prompt_hash(text), **metadata
|
||||
)
|
||||
|
||||
def image(self, prompt_id, name, *, exists=True, folder=None):
|
||||
path = Path(folder or self.output) / name
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if exists:
|
||||
path.write_bytes(b"png")
|
||||
return self.db.link_image_to_prompt(prompt_id, str(path))
|
||||
|
||||
def duplicate(self, text):
|
||||
"""A second prompt row whose text differs only in case (legacy duplicate)."""
|
||||
with self.db.model.get_connection() as conn:
|
||||
cursor = conn.execute(
|
||||
"INSERT INTO prompts(text, created_at) VALUES(?, '2999-01-01')",
|
||||
(text,),
|
||||
)
|
||||
return cursor.lastrowid
|
||||
|
||||
def preview(self, operation, roots=None):
|
||||
roots = [str(self.output)] if roots is None else roots
|
||||
return self.db.preview_maintenance(operation, roots)
|
||||
|
||||
def ids(self, sql, *args):
|
||||
with self.db.model.get_connection() as conn:
|
||||
return [row[0] for row in conn.execute(sql, args)]
|
||||
|
||||
|
||||
class TestPreviews(MaintenanceDatabaseCase):
|
||||
def populate(self):
|
||||
kept = self.save("Kept prompt")
|
||||
self.image(kept, "kept.png")
|
||||
self.image(kept, "gone.png", exists=False)
|
||||
self.save("Unused prompt")
|
||||
self.duplicate("KEPT PROMPT")
|
||||
return kept
|
||||
|
||||
def test_preview_writes_nothing(self):
|
||||
self.populate()
|
||||
# The app's own connection stays open, as in the running server.
|
||||
self.db.get_statistics()
|
||||
files = [self.db_path, self.db_path + "-wal"]
|
||||
real_connect = sqlite3.connect
|
||||
|
||||
class Recording(sqlite3.Connection):
|
||||
changes_at_close = []
|
||||
|
||||
def close(self):
|
||||
Recording.changes_at_close.append(self.total_changes)
|
||||
super().close()
|
||||
|
||||
opened = []
|
||||
|
||||
def connect(*args, **kwargs):
|
||||
conn = real_connect(*args, factory=Recording, **kwargs)
|
||||
opened.append((args, kwargs, conn))
|
||||
return conn
|
||||
|
||||
for operation in OPERATIONS:
|
||||
with self.subTest(operation=operation):
|
||||
before = [_digest(path) for path in files]
|
||||
shm_size = os.path.getsize(self.db_path + "-shm")
|
||||
media = {
|
||||
name: _digest(self.output / name)
|
||||
for name in os.listdir(self.output)
|
||||
}
|
||||
prompts = self.ids("SELECT id FROM prompts ORDER BY id")
|
||||
images = self.ids("SELECT id FROM generated_images ORDER BY id")
|
||||
opened.clear()
|
||||
Recording.changes_at_close.clear()
|
||||
with patch.object(maintenance.sqlite3, "connect", side_effect=connect):
|
||||
draft = self.preview(operation)
|
||||
self.assertTrue(draft.fingerprint.startswith("sha256:"))
|
||||
self.assertEqual(len(opened), 1)
|
||||
args, kwargs, conn = opened[0]
|
||||
self.assertTrue(str(args[0]).endswith("?mode=ro"), args)
|
||||
self.assertTrue(kwargs.get("uri"))
|
||||
self.assertEqual(Recording.changes_at_close, [0])
|
||||
with self.assertRaises(sqlite3.ProgrammingError): # closed
|
||||
conn.execute("SELECT 1")
|
||||
self.assertEqual([_digest(path) for path in files], before)
|
||||
self.assertEqual(os.path.getsize(self.db_path + "-shm"), shm_size)
|
||||
self.assertEqual(
|
||||
{
|
||||
name: _digest(self.output / name)
|
||||
for name in os.listdir(self.output)
|
||||
},
|
||||
media,
|
||||
)
|
||||
self.assertEqual(
|
||||
self.ids("SELECT id FROM prompts ORDER BY id"), prompts
|
||||
)
|
||||
self.assertEqual(
|
||||
self.ids("SELECT id FROM generated_images ORDER BY id"), images
|
||||
)
|
||||
|
||||
def test_unknown_operation_raises(self):
|
||||
with self.assertRaises(ValueError):
|
||||
self.db.preview_maintenance("reboot", [])
|
||||
|
||||
def test_same_library_gives_the_same_fingerprint(self):
|
||||
self.populate()
|
||||
for operation in OPERATIONS:
|
||||
with self.subTest(operation=operation):
|
||||
self.assertEqual(
|
||||
self.preview(operation).fingerprint,
|
||||
self.preview(operation).fingerprint,
|
||||
)
|
||||
|
||||
def test_preview_details_are_capped_and_carry_no_server_path(self):
|
||||
prompt = self.save("Many")
|
||||
for index in range(60):
|
||||
self.image(prompt, f"sub/gone-{index}.png", exists=False)
|
||||
draft = self.preview("missing_links")
|
||||
self.assertEqual(draft.scope["count"], 60)
|
||||
self.assertEqual(len(draft.details), maintenance.MAX_DETAILS)
|
||||
self.assertEqual(draft.details[0]["label"], "sub/gone-0.png")
|
||||
self.assertEqual(draft.details[0]["detail"], "output · the file is missing")
|
||||
self.assertNotIn(str(self.root), str(draft.public()))
|
||||
self.assertEqual(len(draft.targets), 60)
|
||||
|
||||
def test_duplicate_prompts_plan_counts_the_legacy_groups(self):
|
||||
kept = self.save("A cat")
|
||||
self.duplicate("A CAT")
|
||||
self.duplicate("a cat")
|
||||
paired = self.db.get_prompt_pair(self.db.save_prompt_pair("a dog", "blur"))
|
||||
self.duplicate("A DOG")
|
||||
draft = self.preview("duplicate_prompts")
|
||||
self.assertEqual(draft.scope["count"], 2)
|
||||
self.assertEqual(draft.scope["label"], "2 duplicate prompts in 1 group")
|
||||
self.assertEqual(draft.protected["count"], 2)
|
||||
self.assertEqual(
|
||||
draft.protected["reasons"], ["1 group belongs to a saved pair"]
|
||||
)
|
||||
self.assertEqual(draft.targets[0][0], kept)
|
||||
self.assertEqual(len(draft.targets), 1)
|
||||
self.assertEqual(
|
||||
[row["detail"] for row in draft.details],
|
||||
["Keeps the oldest of 3 copies", "Kept: belongs to a saved pair"],
|
||||
)
|
||||
self.assertEqual(draft.details[0]["label"], "A cat")
|
||||
self.assertIsNotNone(paired)
|
||||
|
||||
def test_unused_components_plan_protects_pairs_history_and_titled_parents(self):
|
||||
orphan = self.save("Orphan")
|
||||
pair = self.db.get_prompt_pair(self.db.save_prompt_pair("paired", "neg"))
|
||||
titled = self.save("Titled parent")
|
||||
untitled = self.save("Explicit parent")
|
||||
protected = self.save("Protected", tags=["__protected__"])
|
||||
with self.db.model.get_connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO prompt_library_state(prompt_id,title) "
|
||||
"VALUES(?,?),(?,NULL)",
|
||||
(titled, "My recipe", untitled),
|
||||
)
|
||||
hinted = self.save("Recorded run")
|
||||
draft_before_hint = self.preview("unused_components")
|
||||
self.assertEqual(sorted(draft_before_hint.targets), sorted([orphan, hinted]))
|
||||
self.record_run_hint(hinted)
|
||||
draft = self.preview("unused_components")
|
||||
self.assertEqual(draft.targets, [orphan])
|
||||
self.assertEqual(draft.scope["count"], 1)
|
||||
self.assertEqual(draft.scope["label"], "1 unused prompt component")
|
||||
self.assertEqual(draft.protected["count"], 6)
|
||||
self.assertEqual(
|
||||
draft.protected["reasons"],
|
||||
[
|
||||
"2 in saved pairs",
|
||||
"1 tagged __protected__",
|
||||
"2 with Library state (a title or an explicit parent)",
|
||||
"1 used by a recorded run",
|
||||
],
|
||||
)
|
||||
self.assertEqual(
|
||||
draft.details,
|
||||
[
|
||||
{
|
||||
"label": "Orphan",
|
||||
"detail": maintenance.UNUSED_DETAIL,
|
||||
}
|
||||
],
|
||||
)
|
||||
# The plan counts exactly what the legacy prune removes.
|
||||
self.assertIsNotNone(pair)
|
||||
self.assertEqual(protected > 0, True)
|
||||
self.assertEqual(self.db.prune_orphaned_prompts(), 1)
|
||||
self.assertEqual(self.preview("unused_components").targets, [])
|
||||
|
||||
def record_run_hint(self, prompt_id, run=0):
|
||||
"""Make ``prompt_id`` the live positive hint of a recorded run."""
|
||||
text = self.db.get_prompt_by_id(prompt_id)["text"]
|
||||
history = history_tests.HistoryDatabaseTests
|
||||
start = history.start(self, self.db, run)
|
||||
self.db.write_execution_start(start)
|
||||
self.db.write_execution_finish(history.finish(self, start))
|
||||
self.assertTrue(
|
||||
self.db.write_execution_contexts(
|
||||
start.run_id, start.storage_epoch, bundle(positive=text, negative="")
|
||||
)
|
||||
)
|
||||
|
||||
def test_missing_links_plan_skips_unavailable_roots(self):
|
||||
prompt = self.save("Links")
|
||||
self.image(prompt, "here.png")
|
||||
gone = self.image(prompt, "gone.png", exists=False)
|
||||
unmounted = self.root / "unmounted"
|
||||
self.image(prompt, "away.png", folder=unmounted)
|
||||
os.remove(unmounted / "away.png")
|
||||
os.rmdir(unmounted) # the drive went away: its links look missing
|
||||
self.image(
|
||||
prompt, "elsewhere.png", folder=self.root / "elsewhere", exists=False
|
||||
)
|
||||
draft = self.preview("missing_links", [str(self.output), str(unmounted)])
|
||||
self.assertEqual(draft.targets, [gone])
|
||||
self.assertEqual(draft.scope["count"], 1)
|
||||
self.assertEqual(draft.scope["label"], "1 link to a missing file")
|
||||
self.assertEqual(draft.protected["count"], 2)
|
||||
self.assertEqual(
|
||||
draft.protected["reasons"],
|
||||
[
|
||||
"1 link under an unavailable source",
|
||||
"1 link outside the image sources",
|
||||
],
|
||||
)
|
||||
self.assertFalse(draft.media["deletes_files"])
|
||||
# No readable source: nothing is counted, every link is protected.
|
||||
none = self.preview("missing_links", [str(unmounted)])
|
||||
self.assertEqual((none.targets, none.scope["count"]), ([], 0))
|
||||
self.assertEqual(none.protected["count"], 4)
|
||||
|
||||
def test_missing_links_plan_matches_the_health_check(self):
|
||||
prompt = self.save("Links")
|
||||
self.image(prompt, "a.png", exists=False)
|
||||
self.image(prompt, "b.png", exists=False)
|
||||
self.image(prompt, "c.png")
|
||||
report = self.db.run_health_checks([str(self.output)])
|
||||
missing = next(c for c in report["checks"] if c["id"] == "missing_files")
|
||||
self.assertEqual(self.preview("missing_links").scope["count"], missing["count"])
|
||||
|
||||
def test_optimize_plan_reports_sizes_without_paths(self):
|
||||
self.populate()
|
||||
draft = self.preview("optimize")
|
||||
self.assertEqual(draft.targets, [])
|
||||
self.assertEqual(draft.scope, {"count": 1, "label": "The whole database"})
|
||||
self.assertEqual(
|
||||
[row["label"] for row in draft.details],
|
||||
["Database file", "Free pages", "Free space needed"],
|
||||
)
|
||||
self.assertNotIn(str(self.root), str(draft.public()))
|
||||
|
||||
def test_legacy_cleanup_missing_images_skips_unavailable_roots(self):
|
||||
prompt = self.save("Links")
|
||||
gone = self.image(prompt, "gone.png", exists=False)
|
||||
unmounted = self.root / "unmounted"
|
||||
away = self.image(prompt, "away.png", folder=unmounted, exists=False)
|
||||
outside = self.image(prompt, "x.png", folder=self.root / "other", exists=False)
|
||||
unmounted.rmdir()
|
||||
self.assertEqual(
|
||||
self.db.cleanup_missing_images(roots=[str(self.output), str(unmounted)]), 1
|
||||
)
|
||||
remaining = self.ids("SELECT id FROM generated_images ORDER BY id")
|
||||
self.assertEqual(remaining, sorted([away, outside]))
|
||||
self.assertNotIn(gone, remaining)
|
||||
# Without roots it keeps the legacy rule: every link whose file is gone.
|
||||
self.assertEqual(self.db.cleanup_missing_images(), 2)
|
||||
|
||||
def test_reader_closes_its_connection_when_a_read_raises(self):
|
||||
self.populate()
|
||||
with patch.object(
|
||||
maintenance, "_orphaned_ids", side_effect=sqlite3.OperationalError("boom")
|
||||
):
|
||||
with self.assertRaises(sqlite3.OperationalError):
|
||||
self.preview("unused_components")
|
||||
# The teardown's assert_closed proves the read-only handle was closed.
|
||||
|
||||
|
||||
class TestVacuum(MaintenanceDatabaseCase):
|
||||
def test_vacuum_failure_raises(self):
|
||||
with patch(
|
||||
"database.models.sqlite3.connect",
|
||||
side_effect=sqlite3.OperationalError("disk I/O error"),
|
||||
):
|
||||
with self.assertLogs("prompt_manager", "ERROR"):
|
||||
with self.assertRaises(sqlite3.OperationalError):
|
||||
self.db.model.vacuum_database()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user