Vito Sansevero
49f39ca911
Final review fixes I1, I2 and I3 for the workflow storage dedupe: - I1: the migration always takes a verified backup, whatever maintenance.safety_backup says; the plan reports backup.policy "required" and the disk check counts it. - I2: that backup is prompts.db.pre-workflow-storage_<stamp>, outside the pruned prompts.db.backup_* pool and kept until the user removes it; the job's last line and the preview name it, and the legacy restore accepts it. README's changelog warns that an older version opens and restores a rewritten database but shows no workflows, and says how to go back. - I3: tools/workflow_text_digests.py writes the sha256 of every record's stored texts (read-only; inline before the rewrite, through the accessor after), so a diff before and after proves byte identity. - A row the migration could not rewrite is listed in the job's errors and named in its last line, and Reclaim space still runs.
256 lines
10 KiB
Python
256 lines
10 KiB
Python
"""Store each workflow once: the migration and Reclaim space jobs (#249 S6).
|
|
|
|
Reviewed like every maintenance operation (``POST /prompt_manager/
|
|
maintenance/preview {operation:"workflow_storage"}``, a ``Plan`` with the
|
|
records to rewrite, the backup and the disk check) and applied with its token
|
|
(``POST /prompt_manager/maintenance/apply``). The apply starts:
|
|
|
|
1. **Store each workflow once**: kind ``maintenance``, admission ``records``,
|
|
cancellable, groups ``changed``/``protected``/``errors``. It always takes
|
|
a verified pre-migration backup, whatever the safety backup policy, named
|
|
``<db>.pre-workflow-storage_<stamp>`` outside the pruned ``<db>.backup_*``
|
|
pool and kept until the user removes it (``backup-failed`` ends the job
|
|
with nothing rewritten). It then rewrites old-form rows batch by batch
|
|
(``database/workflow_storage.py``): one transaction per batch of at most
|
|
``BATCH_SECONDS``, then ``BATCH_PAUSE`` with the write lock free, progress
|
|
after each, Cancel honoured between batches. A row that cannot be
|
|
rewritten stays inline and is listed in ``errors``; any other failure
|
|
rolls its batch back whole and fails the job. Every record still reads as
|
|
before, and the next apply continues from the first old-form row.
|
|
2. **Reclaim space**, once every record was looked at: kind ``maintenance``,
|
|
admission ``exclusive`` (it runs alone; VACUUM holds the write lock), not
|
|
cancellable. It checks the disk for VACUUM's copies of the live pages
|
|
(``disk_plan``), then sweeps, runs VACUUM and truncates the WAL. When
|
|
another job holds the registry, it does not start and the finished
|
|
migration says so and names Optimize database.
|
|
|
|
Only the database changes: no image or video file is opened.
|
|
"""
|
|
|
|
import asyncio
|
|
import sqlite3
|
|
|
|
from .jobs import MAINTENANCE_JOB_KIND, JobBusy
|
|
from .maintenance import (
|
|
BACKUP_FAILED,
|
|
MAINTENANCE_GROUPS,
|
|
PRE_MIGRATION_LABEL,
|
|
SafetyBackupFailed,
|
|
_fail_job,
|
|
_megabytes,
|
|
disk_plan,
|
|
short_space_message,
|
|
take_safety_backup,
|
|
)
|
|
|
|
try:
|
|
from ...database import workflow_storage
|
|
from ...database.health import readonly_uri
|
|
except ImportError:
|
|
from database import workflow_storage
|
|
from database.health import readonly_uri
|
|
|
|
OPERATION = workflow_storage.OPERATION
|
|
TITLE = "Store each workflow once"
|
|
RECLAIM_TITLE = "Reclaim space"
|
|
REWRITE_FAILED = (
|
|
"Storing workflows once stopped on an error. The batch in progress was "
|
|
"rolled back; every record reads as before. Run it again to continue."
|
|
)
|
|
RECLAIM_FAILED = "Reclaim space failed; the database is unchanged."
|
|
ROW_KEPT = "could not be rewritten; it keeps its inline text and reads as before"
|
|
|
|
|
|
def _backup_part(backup):
|
|
return f"Pre-migration backup {backup}"
|
|
|
|
|
|
def _plural(count, one, many=None):
|
|
return f"{count:,} {one if count == 1 else (many or one + 's')}"
|
|
|
|
|
|
def live_bytes(db_path):
|
|
"""The pages VACUUM copies: page_count minus the free list (blocking)."""
|
|
conn = sqlite3.connect(readonly_uri(db_path), uri=True)
|
|
try:
|
|
page_size = conn.execute("PRAGMA page_size").fetchone()[0]
|
|
pages = conn.execute("PRAGMA page_count").fetchone()[0]
|
|
free = conn.execute("PRAGMA freelist_count").fetchone()[0]
|
|
return page_size * (pages - free)
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
class WorkflowStorageRoutesMixin:
|
|
"""Mixin for the maintenance routes: the workflow storage jobs."""
|
|
|
|
def _start_workflow_storage(self, db, backup):
|
|
"""Start the migration job (raises JobBusy); the token is the caller's."""
|
|
return self._jobs.start(
|
|
MAINTENANCE_JOB_KIND,
|
|
TITLE,
|
|
admission="records",
|
|
cancellable=True,
|
|
scope={"operation": OPERATION},
|
|
groups=MAINTENANCE_GROUPS,
|
|
runner=lambda job: self._run_workflow_storage(job, db, backup),
|
|
)
|
|
|
|
async def _run_workflow_storage(self, job, db, _backup_on):
|
|
# Always backed up first, whatever the policy: the way back.
|
|
backup = await self._pre_migration_backup(job, db)
|
|
if backup is None:
|
|
return
|
|
if await self._rewrite_workflows(job, db, backup):
|
|
job.finish("finished")
|
|
self._start_reclaim_space(job, db)
|
|
|
|
async def _pre_migration_backup(self, job, db):
|
|
"""The verified ``<db>.pre-workflow-storage_<stamp>`` copy, or None.
|
|
|
|
Outside the pruned ``<db>.backup_*`` pool: it stays until the user
|
|
removes it. A failure ends the job ``backup-failed``, nothing changed.
|
|
"""
|
|
job.update(status_line="Taking the pre-migration backup")
|
|
job.log_line("Taking the pre-migration backup of the database")
|
|
try:
|
|
backup = await self._run_in_executor(
|
|
take_safety_backup, db.model, PRE_MIGRATION_LABEL
|
|
)
|
|
except SafetyBackupFailed:
|
|
self.logger.error("Pre-migration backup failed", exc_info=True)
|
|
_fail_job(job, "backup-failed", BACKUP_FAILED)
|
|
return None
|
|
job.log_line(
|
|
f"Pre-migration backup {backup} verified; it is kept until you remove it"
|
|
)
|
|
return backup
|
|
|
|
async def _rewrite_workflows(self, job, db, backup):
|
|
"""Rewrite every old-form row; True when every row was looked at."""
|
|
counts = await self._run_in_executor(db.workflow_storage_status)
|
|
job.update(total=counts["rows"], processed=0)
|
|
job.update(status_line="Rewriting image records")
|
|
job.log_line(f"{_plural(counts['rows'], 'image record')} to rewrite")
|
|
rows = protected = errors = 0
|
|
last_id = 0
|
|
while True:
|
|
if job.cancelled:
|
|
return self._stopped(job, rows, backup)
|
|
try:
|
|
batch = await self._run_in_executor(
|
|
db.migrate_workflow_batch,
|
|
last_id,
|
|
workflow_storage.BATCH_ROWS,
|
|
workflow_storage.BATCH_BYTES,
|
|
)
|
|
except Exception:
|
|
self.logger.exception("Storing workflows once failed")
|
|
_fail_job(job, None, REWRITE_FAILED)
|
|
return False
|
|
if batch["done"]:
|
|
break
|
|
last_id = batch["last_id"]
|
|
rows += batch["rows"]
|
|
protected += batch["protected"]
|
|
errors += len(batch["errors"])
|
|
job.record_counted("changed", batch["rows"])
|
|
job.record_counted("protected", batch["protected"])
|
|
for image_id in batch["errors"]:
|
|
job.record(
|
|
"errors",
|
|
{"path": "", "detail": f"Image record {image_id}: {ROW_KEPT}"},
|
|
f"! Image record {image_id}: {ROW_KEPT}",
|
|
)
|
|
total = max(job.total, rows)
|
|
job.update(
|
|
processed=rows,
|
|
total=total,
|
|
progress=100 * rows // total if total else 100,
|
|
status_line=f"Rewrote {_plural(rows, 'image record')}",
|
|
)
|
|
job.publish({"type": "progress", "processed": rows, "total": total})
|
|
# Leave the write lock free between batches (monitor, run counts).
|
|
await asyncio.sleep(workflow_storage.BATCH_PAUSE)
|
|
removed = await self._run_in_executor(db.sweep_workflow_documents)
|
|
line = f"Rewrote {_plural(rows, 'image record')}"
|
|
if protected:
|
|
line += f"; {_plural(protected, 'stored value')} not text, kept as is"
|
|
if errors:
|
|
kept = "keeps its" if errors == 1 else "keep their"
|
|
line += (
|
|
f"; {_plural(errors, 'image record')} could not be rewritten and "
|
|
f"{kept} inline text"
|
|
)
|
|
job.log_line(line)
|
|
if removed:
|
|
job.log_line(f"Removed {_plural(removed, 'unused workflow document')}")
|
|
job.update(progress=100, status_line=f"{line} · {_backup_part(backup)}")
|
|
return True
|
|
|
|
def _stopped(self, job, rows, backup):
|
|
line = (
|
|
f"Stopped after {_plural(rows, 'image record')}; "
|
|
"run it again to continue"
|
|
)
|
|
job.log_line(line)
|
|
job.update(status_line=f"{line} · {_backup_part(backup)}")
|
|
job.finish("cancelled")
|
|
return False
|
|
|
|
def _start_reclaim_space(self, finished, db):
|
|
"""Start Reclaim space after the migration ``finished``, or say why not."""
|
|
try:
|
|
self._jobs.start(
|
|
MAINTENANCE_JOB_KIND,
|
|
RECLAIM_TITLE,
|
|
admission="exclusive",
|
|
cancellable=False,
|
|
scope={"operation": OPERATION},
|
|
groups=MAINTENANCE_GROUPS,
|
|
runner=lambda job: self._run_reclaim_space(job, db),
|
|
)
|
|
except JobBusy as busy:
|
|
line = (
|
|
f"Reclaim space did not start: {busy.running.title} is running. "
|
|
"Run Optimize database to give the space back."
|
|
)
|
|
finished.log_line(line)
|
|
finished.update(status_line=f"{finished.status_line} · {line}")
|
|
return
|
|
finished.log_line("Reclaim space started")
|
|
|
|
async def _run_reclaim_space(self, job, db):
|
|
job.update(status_line="Checking free disk space")
|
|
try:
|
|
live = await self._run_in_executor(live_bytes, db.model.db_path)
|
|
_disk, short, _size = await self._run_in_executor(
|
|
disk_plan,
|
|
db.model.db_path,
|
|
backup=False,
|
|
vacuum=True,
|
|
vacuum_bytes=live,
|
|
)
|
|
except (OSError, sqlite3.Error):
|
|
self.logger.exception("Could not measure the disk space for Reclaim space")
|
|
return _fail_job(job, None, RECLAIM_FAILED)
|
|
if short is not None:
|
|
return _fail_job(job, "insufficient-space", short_space_message(short))
|
|
job.update(status_line="Reclaiming space (the database is locked)")
|
|
job.log_line("Rebuilding the database file (VACUUM)")
|
|
try:
|
|
result = await self._run_in_executor(db.reclaim_space)
|
|
except Exception:
|
|
self.logger.exception("Reclaim space failed")
|
|
return _fail_job(job, None, RECLAIM_FAILED)
|
|
saved = max(0, result["before_bytes"] - result["after_bytes"])
|
|
line = (
|
|
f"Reclaimed {_megabytes(saved)} · the database is now "
|
|
f"{_megabytes(result['after_bytes'])}"
|
|
)
|
|
if result["checkpoint_busy"]:
|
|
line += " (the rest after the next restart)"
|
|
job.record_counted("changed", 1)
|
|
job.log_line(line)
|
|
job.update(progress=100, status_line=line)
|