ComfyAssets-ComfyUI_PromptManager/py/api/maintenance.py

"""Maintenance routes: reviewed, read-only plans (#249 System S6).

``POST /prompt_manager/maintenance/preview {operation, options:{}}`` reviews
one operation (``duplicate_prompts``, ``optimize``, ``unused_components`` or
``missing_links``; ``database/maintenance.py``) off the event loop and
answers a ``Plan``: ``{success, token, operation, expires_at, scope:{count,
label}, protected:{count, reasons}, backup:{policy, destination_label},
media:{deletes_files, label}, details:[≤50 {label, detail}]}``. The preview
writes nothing. The plan's fingerprint, its targets and the sources it judged
stay in the server's ``PlanStore`` under the token, bound to the database the
preview read (a restore that replaces it invalidates the plan).

The operation enum is strict, ``options`` must be an empty object and the
body is capped at ``PREVIEW_BODY_MAX`` bytes. Errors are ``{success:false,
code, error}`` with ``invalid-request`` (400), ``too-large`` (413: the body,
or a plan over the store's budget), ``storage-changed`` (409: the database
was replaced while the preview read it) or a null code (500); none carries a
server path or exception text.

``POST /prompt_manager/maintenance/apply {token}`` applies a reviewed plan as
an Activity job (202 ``{job}``): kind ``maintenance``, titled after the
operation, admission ``records`` (refused while a scan, restore, LoRA import
or other record-changing job runs, and refusing them meanwhile), not
cancellable, with outcome groups ``changed``, ``protected`` and ``errors``.
The token is consumed when the job starts (on the same loop turn, before its
runner runs); a refused start keeps it. The runner:

1. takes a safety backup when the plan was reviewed with the policy on or the
   policy (``maintenance.safety_backup``, default on) is on now: SQLite's
   backup API into a temp file beside the database, ``verify_database_file``,
   fsync, ``0o600`` on POSIX, then ``os.replace`` onto a unique
   ``<db>.backup_<YYYYmmdd_HHMMSS_ffffff>[_n]`` name. Any failure removes
   what it wrote and ends the job ``backup-failed`` with nothing changed;
2. applies the plan, which first rechecks the reviewed fingerprint inside
   ``BEGIN IMMEDIATE`` (``plan-changed`` writes nothing);
3. counts the changed and protected records, and prunes old safety backups.

The job log names the backup by its file name only, and the job says "No
safety backup" when none was taken. Apply errors: ``invalid-request`` (400),
``apply-failed`` (500: nothing started), ``plan-expired`` (410: unknown,
expired or forgotten after a restart),
``plan-used`` (409, with the job that applied it), ``storage-changed`` (409:
a restore replaced the database since the review) and ``busy`` (409).
"""

import datetime
import json
import os
import shutil
import sqlite3

from aiohttp import web

from .admin import _prune_safety_backups, _scan_location, _scan_roots
from .job_routes import busy_response
from .jobs import MAINTENANCE_JOB_KIND, JobBusy
from .maintenance_plans import PlanExpired, PlanUsed, ScopeTooLarge

try:
    from ...database import safety_backup as _backups
    from ...database.safety_backup import (  # noqa: F401 - re-exported
        SAFETY_TEMP_PREFIX,
        SafetyBackupFailed,
        sweep_safety_temps,
    )
    from ...database.workflow_storage import PRE_MIGRATION_LABEL, pre_migration_backups
    from ...database.health import readonly_uri
    from ...database.maintenance import MAINTENANCE_OPERATIONS, PlanChanged
except ImportError:
    from database import safety_backup as _backups
    from database.safety_backup import (  # noqa: F401 - re-exported
        SAFETY_TEMP_PREFIX,
        SafetyBackupFailed,
        sweep_safety_temps,
    )
    from database.workflow_storage import PRE_MIGRATION_LABEL, pre_migration_backups
    from database.health import readonly_uri
    from database.maintenance import MAINTENANCE_OPERATIONS, PlanChanged

PREVIEW_BODY_MAX = 4096
MAINTENANCE_PREVIEW_LEASE = "Maintenance preview"
# Duplicate-file previews carry the reviewed keepers (py/api/duplicate_files.py).
DUPLICATE_PREVIEW_BODY_MAX = 1024 * 1024
DUPLICATE_FILES = "duplicate_files"
WORKFLOW_STORAGE = "workflow_storage"  # applied by py/api/workflow_storage.py
PREVIEW_OPERATIONS = MAINTENANCE_OPERATIONS + (DUPLICATE_FILES,)
# Room left over beyond what a step writes, so a disk is never filled to 0.
DISK_MARGIN_BYTES = 16 * 2**20
# The concept's Maintenance row titles, which name each operation's job.
MAINTENANCE_TITLES = {
    "duplicate_prompts": "Duplicate prompt text",
    "optimize": "Optimize database",
    "unused_components": "Unused prompt components",
    "missing_links": "Missing-file links",
    "workflow_storage": "Store each workflow once",
}
_OPERATION_ERROR = "operation must be one of: " + ", ".join(PREVIEW_OPERATIONS)
MAINTENANCE_GROUPS = ("changed", "protected", "errors")
MAX_TOKEN_LENGTH = 128
BACKUP_FAILED = (
    "The safety backup could not be taken and verified; nothing was changed."
)
PLAN_CHANGED = (
    "The library changed since the review; nothing was changed. "
    "Review the plan again."
)
APPLY_FAILED = "Maintenance failed; nothing was changed."
NOTHING_TO_DO = "Every image record already stores its workflow once; nothing to do."
NO_BACKUP = "No safety backup"


def backup_failed_message(error):
    """The job line for a failed backup, naming a damaged workflow document."""
    reason = getattr(error, "reason", None) or ""
    if reason.startswith("Workflow document "):
        return f"{BACKUP_FAILED} {reason}"
    return BACKUP_FAILED


def _free_bytes(path):
    return shutil.disk_usage(path).free


def _device(path):
    try:
        return os.stat(path).st_dev
    except OSError:
        return path


def sqlite_temp_dir(environ=None, *, windows=None, usable=None):
    """The folder SQLite builds its temporary files in (VACUUM's copy).

    Mirrors SQLite's own search, which differs from ``tempfile``'s. On Unix
    (``unixTempFileDir``): ``SQLITE_TMPDIR``, ``TMPDIR``, then ``/var/tmp``,
    ``/usr/tmp`` and ``/tmp``, the first that is a writable directory, else
    the current directory. On Windows (``GetTempPathW``): ``TMP``, ``TEMP``,
    then ``USERPROFILE``, else the current directory.
    """
    environ = os.environ if environ is None else environ
    if windows is None:
        windows = os.name == "nt"
    if windows:
        for name in ("TMP", "TEMP", "USERPROFILE"):
            if environ.get(name):
                return environ[name]
        return os.path.abspath(".")
    if usable is None:

        def usable(path):
            return os.path.isdir(path) and os.access(path, os.W_OK | os.X_OK)

    candidates = [environ.get("SQLITE_TMPDIR"), environ.get("TMPDIR")]
    candidates += ["/var/tmp", "/usr/tmp", "/tmp"]
    for candidate in candidates:
        if candidate and usable(candidate):
            return candidate
    return os.path.abspath(".")


def database_bytes(db_path):
    """The size a safety backup of the database will be (blocking).

    SQLite's backup API copies every page, committed WAL content included, so
    the copy is ``page_count * page_size`` read on one read-only snapshot.
    """
    conn = sqlite3.connect(readonly_uri(db_path), uri=True)
    try:
        page_size = conn.execute("PRAGMA page_size").fetchone()[0]
        return page_size * conn.execute("PRAGMA page_count").fetchone()[0]
    finally:
        conn.close()


def _megabytes(size):
    return f"{size / 2**20:,.1f} MB"


def disk_plan(db_path, *, backup, vacuum, vacuum_bytes=None):
    """Whether the disk has room for an apply's writes (blocking).

    ``backup``: the safety backup, one database-sized file beside the
    database. ``vacuum``: VACUUM writes the rebuilt database through the WAL
    beside it and builds it in a temporary copy in SQLite's temporary folder
    (``sqlite_temp_dir``; SQLite's documentation: up to twice the database
    in free space). Needs
    on one device add up. ``vacuum_bytes``, when given, is what VACUUM
    copies (the live pages, for Reclaim space) instead of the whole file.
    Returns ``(public, short)``: ``public`` is
    ``{free_bytes, required_bytes, sufficient}`` for the database folder,
    ``short`` None or ``(place, what, required, free)`` of the first place
    that is short, and ``size`` the database's size (0 when nothing is
    written).
    """
    db_path = os.path.abspath(db_path)
    folder = os.path.dirname(db_path)
    size = database_bytes(db_path) if backup or vacuum else 0
    needs = []  # (place, folder, bytes, what)
    if backup:
        needs.append(("database folder", folder, size, "the safety backup"))
    if vacuum:
        copy = size if vacuum_bytes is None else vacuum_bytes
        what = "Optimize" if vacuum_bytes is None else "Reclaim space"
        needs.append(("database folder", folder, copy, what))
        needs.append(("temporary folder", sqlite_temp_dir(), copy, what))
    devices = {}
    for place, path, needed, what in needs:
        entry = devices.setdefault(_device(path), {"place": place, "path": path})
        entry["bytes"] = entry.get("bytes", 0) + needed
        entry.setdefault("what", [])
        if what not in entry["what"]:
            entry["what"].append(what)
    short = None
    public = {"free_bytes": _free_bytes(folder), "required_bytes": 0}
    for device, entry in devices.items():
        required = entry["bytes"] + DISK_MARGIN_BYTES
        free = (
            public["free_bytes"]
            if entry["path"] == folder
            else _free_bytes(entry["path"])
        )
        if device == _device(folder):
            public["required_bytes"] = required
        if free < required and short is None:
            short = (entry["place"], " and ".join(entry["what"]), required, free)
    public["sufficient"] = short is None
    return public, short, size


# A restore upload's hard ceiling; below it, free disk space decides.
RESTORE_MAX_BYTES = 64 * 2**30


def size_label(size):
    """``1.6 GB`` or ``120.0 MB``: a backup's size for a message."""
    if size >= 2**30:
        return f"{size / 2**30:,.1f} GB"
    return _megabytes(size)


def restore_room(db_path):
    """The largest restore upload the disk has room for (blocking).

    Everything a restore writes is in the library's folder (#318): the
    upload, its staged copy (``database/restore_stage.py``) and the WAL
    growth while the stage is copied over the live database (3x the upload),
    plus the safety backup of the current database (1x its size), with
    ``DISK_MARGIN_BYTES`` to spare.
    """
    db_path = os.path.abspath(db_path)
    folder = os.path.dirname(db_path)
    current = database_bytes(db_path) if os.path.exists(db_path) else 0
    room = (_free_bytes(folder) - current - DISK_MARGIN_BYTES) // 3
    return max(0, room)


def restore_space_message(size, room, *, known):
    """The truthful refusal of a restore the disks have no room for."""
    if known:
        return (
            f"Not enough free disk space to restore a {size_label(size)} backup; "
            f"there is room for about {size_label(room)}. Nothing was changed."
        )
    return (
        "Not enough free disk space to restore this backup: it is larger than "
        f"the {size_label(room)} there is room for. Nothing was changed."
    )


def short_space_message(short):
    """The truthful refusal for ``disk_plan``'s ``short``."""
    place, what, required, free = short
    verb = "need" if " and " in what else "needs"
    return (
        f"Not enough free disk space: {what} {verb} about {_megabytes(required)} "
        f"in the {place} and {_megabytes(free)} is free. Nothing was changed."
    )


def _changed_line(operation, count):
    if operation == "optimize":
        return "Optimized the database"
    if operation == "duplicate_prompts":
        noun = "duplicate prompt" if count == 1 else "duplicate prompts"
        return f"Merged {count:,} {noun}"
    if operation == "unused_components":
        noun = "unused prompt component" if count == 1 else "unused prompt components"
        return f"Removed {count:,} {noun}"
    noun = "link to a missing file" if count == 1 else "links to missing files"
    return f"Removed {count:,} {noun}"


def take_safety_backup(model, label="backup"):
    """Write and verify a safety backup; return its file name (blocking).

    See ``database/safety_backup.py``: a backup file on disk is always a
    verified one. Raises SafetyBackupFailed, leaving no file.
    """
    return os.path.basename(_backups.take_safety_backup(model, label))


def _failure(message, status, code=None, **extra):
    return web.json_response(
        {"success": False, "code": code, "error": message, **extra}, status=status
    )


def _invalid(message):
    return _failure(message, 400, "invalid-request")


def _iso(timestamp):
    return datetime.datetime.fromtimestamp(timestamp, datetime.timezone.utc).isoformat()


class MaintenanceRoutesMixin:
    """Mixin providing the /prompt_manager/maintenance plan endpoints."""

    def _register_maintenance_routes(self, routes):
        @routes.post("/prompt_manager/maintenance/preview")
        async def maintenance_preview_route(request):
            return await self.preview_maintenance(request)

        @routes.post("/prompt_manager/maintenance/apply")
        async def maintenance_apply_route(request):
            return await self.apply_maintenance_plan(request)

    def _database_identity(self, db):
        """What binds a plan to one database: the swap generation and its file."""
        path = os.path.normcase(os.path.realpath(db.model.db_path))
        return (self._db_generation, path)

    @staticmethod
    def _safety_backup_policy():
        """``on`` or ``off``: whether an apply takes a safety backup first."""
        from ..config import PromptManagerConfig

        return "on" if PromptManagerConfig.SAFETY_BACKUP else "off"

    async def _read_maintenance_body(self, request, allowed, cap=None):
        """``(data, None)`` for a capped JSON object with only ``allowed`` keys.

        ``cap(data)``, when given, is the body's byte limit for what it asks
        (read up to ``DUPLICATE_PREVIEW_BODY_MAX``); else PREVIEW_BODY_MAX.
        """
        limit = PREVIEW_BODY_MAX if cap is None else DUPLICATE_PREVIEW_BODY_MAX
        data, error = await self._read_json_body(request, max_bytes=limit)
        if error is not None:
            if error.status == 413:
                return None, _failure("Request body too large", 413, "too-large")
            return None, _invalid("Request body must be a JSON object")
        if cap is not None:
            size = len(json.dumps(data, separators=(",", ":")).encode("utf-8"))
            if size > cap(data):
                return None, _failure("Request body too large", 413, "too-large")
        unknown = sorted(set(data) - set(allowed))
        if unknown:
            return None, _invalid("Unknown field: " + ", ".join(unknown)[:200])
        return data, None

    async def preview_maintenance(self, request):
        """Review one operation under a registry lease (#249 S8).

        The preview's read-only connections must not be open while a restore
        switches the database: it is refused (409 ``busy``) while an
        ``exclusive`` job runs, and a restore waits for running previews.
        """
        try:
            async with self._jobs.lease(MAINTENANCE_PREVIEW_LEASE):
                return await self._preview_maintenance(request)
        except JobBusy as busy:
            return busy_response(busy)

    async def _preview_maintenance(self, request):
        data, error = await self._read_maintenance_body(
            request,
            ("operation", "options"),
            cap=lambda body: (
                DUPLICATE_PREVIEW_BODY_MAX
                if body.get("operation") == DUPLICATE_FILES
                else PREVIEW_BODY_MAX
            ),
        )
        if error is not None:
            return error
        operation = data.get("operation")
        options = data.get("options", {})
        if not isinstance(operation, str) or operation not in PREVIEW_OPERATIONS:
            return _invalid(_OPERATION_ERROR)
        if operation == DUPLICATE_FILES:
            return await self.preview_duplicate_files(options)
        if not isinstance(options, dict) or options:
            return _invalid(f"{operation} takes no options")
        db = self.db
        identity = self._database_identity(db)
        try:
            roots = (
                await self._run_in_executor(self._health_roots)
                if operation == "missing_links"
                else []
            )
            draft = await self._run_in_executor(
                db.preview_maintenance, operation, roots
            )
        except Exception:
            self.logger.exception(f"Maintenance preview of {operation} failed")
            return _failure("Failed to review maintenance", 500)
        return await self._plan_response(
            db,
            identity,
            draft.operation,
            draft.fingerprint,
            draft.public(),
            {
                "targets": draft.targets,
                "roots": roots,
                "protected": draft.protected["count"],
                "rows": draft.scope["count"],
            },
        )

    async def _plan_response(
        self, db, identity, operation, fingerprint, public, payload
    ):
        """Issue the reviewed plan's token and answer the ``Plan``.

        Refused with ``storage-changed`` when the database was replaced while
        it was reviewed. The plan reports the safety backup's expected size
        and the disk space the apply needs (``disk_plan``).
        """
        if identity != self._database_identity(self.db):
            return _failure(
                "The database was replaced while it was reviewed. Review again.",
                409,
                "storage-changed",
            )
        policy = self._safety_backup_policy()
        destination = "backup"
        if operation == WORKFLOW_STORAGE:
            # Always backed up first, whatever the policy (the way back);
            # one kept pre-migration backup ever, pooled ones after it.
            policy = "required"
            kept = await self._run_in_executor(pre_migration_backups, db.model.db_path)
            destination = "backup" if kept else PRE_MIGRATION_LABEL
        try:
            disk, _short, size = await self._run_in_executor(
                disk_plan,
                db.model.db_path,
                backup=policy != "off",
                vacuum=operation == "optimize",
            )
        except (OSError, sqlite3.Error):
            self.logger.exception("Could not measure the disk space for a plan")
            return _failure("Failed to review maintenance", 500)
        store = self._maintenance_plans
        try:
            token = store.issue(
                operation, fingerprint, {**payload, "backup": policy}, identity
            )
        except ScopeTooLarge:
            return _failure(
                "This plan is too large to review at once.", 413, "too-large"
            )
        plan = store.peek(token, operation)
        name = os.path.basename(db.model.db_path)
        return web.json_response(
            {
                "success": True,
                "token": token,
                "expires_at": _iso(plan.expires_at),
                **public,
                "backup": {
                    "policy": policy,
                    "destination_label": f"{name}.{destination}_<date and time>",
                    "expected_bytes": size if policy != "off" else None,
                },
                "disk": disk,
            },
            headers={"Cache-Control": "no-store"},
        )

    # ── apply ─────────────────────────────────────────────────────────

    async def apply_maintenance_plan(self, request):
        data, error = await self._read_maintenance_body(request, ("token",))
        if error is not None:
            return error
        token = data.get("token")
        if not isinstance(token, str) or not 0 < len(token) <= MAX_TOKEN_LENGTH:
            return _invalid("token must be the plan's token")
        plan, refused = self._peek_plan(token)
        if refused is not None:
            return refused
        store = self._maintenance_plans
        db = self.db
        if plan.db_identity != self._database_identity(db):
            store.discard(token)
            return _failure(
                "The database was replaced since the review. Review again.",
                409,
                "storage-changed",
            )
        if plan.operation == WORKFLOW_STORAGE and not plan.payload.get("rows"):
            store.discard(token)  # no job, no backup: nothing to rewrite
            return _failure(NOTHING_TO_DO, 409, "nothing-to-do")
        backup = self._backup_promised(plan)
        try:
            _disk, short, _size = await self._run_in_executor(
                disk_plan,
                db.model.db_path,
                backup=backup,
                vacuum=plan.operation == "optimize",
            )
        except (OSError, sqlite3.Error):
            self.logger.exception("Could not measure the disk space for an apply")
            # Coded: the workspace tells the server's own refusal (nothing
            # started) from a proxy's 5xx (the job may have started).
            return _failure(APPLY_FAILED, 500, "apply-failed")
        if short is not None:
            # Refused before any job starts: nothing fails halfway for space.
            return _failure(short_space_message(short), 507, "insufficient-space")
        # The token may have been used, expired or dropped while the disk was
        # measured: look again with no await before the job starts.
        plan, refused = self._peek_plan(token)
        if refused is not None:
            return refused
        try:
            if plan.operation == DUPLICATE_FILES:
                job = self._start_duplicate_cleanup(db, plan, backup)
            elif plan.operation == WORKFLOW_STORAGE:
                job = self._start_workflow_storage(db, backup)
            else:
                job = self._jobs.start(
                    MAINTENANCE_JOB_KIND,
                    MAINTENANCE_TITLES[plan.operation],
                    # VACUUM rewrites the whole file under the write lock: it
                    # runs alone, beside no other job of any class.
                    admission=(
                        "exclusive" if plan.operation == "optimize" else "records"
                    ),
                    cancellable=False,
                    scope={"operation": plan.operation},
                    groups=MAINTENANCE_GROUPS,
                    runner=lambda job: self._run_maintenance_plan(
                        job, db, plan, backup
                    ),
                )
        except JobBusy as busy:
            return busy_response(busy)  # the token stays usable
        # Same loop turn as the peek and the start: the runner has not run.
        store.consume(token, plan.operation, job_id=job.id)
        return web.json_response({"success": True, "job": job.summary()}, status=202)

    def _peek_plan(self, token):
        """``(plan, None)`` for a usable token, else ``(None, refusal)``."""
        try:
            return self._maintenance_plans.peek(token), None
        except PlanUsed as used:
            job = self._jobs.get(used.job_id)
            return None, _failure(
                "This plan was already applied.",
                409,
                "plan-used",
                job={"id": used.job_id, "title": job.title if job else None},
            )
        except PlanExpired:
            return None, _failure(
                "This review expired. Review the plan again.", 410, "plan-expired"
            )

    def _backup_promised(self, plan):
        """A plan reviewed with the policy on keeps that promise; so does on now.

        Decided once, when the apply starts (the disk check measured it), and
        handed to the job.
        """
        if plan.operation == WORKFLOW_STORAGE:
            return True  # never without its pre-migration backup
        return plan.payload["backup"] == "on" or self._safety_backup_policy() == "on"

    async def _safety_backup_step(self, job, db, backup_on):
        """``(ok, name)``: take the promised safety backup, or report none.

        ``name`` is the backup's file name, or None when no backup was
        promised. A failed backup ends the job ``backup-failed`` (``ok``
        False).
        """
        if not backup_on:
            job.log_line("No safety backup (the policy is off)")
            return True, None
        job.update(status_line="Taking a safety backup")
        job.log_line("Taking a safety backup of the database")
        try:
            backup = await self._run_in_executor(take_safety_backup, db.model)
        except SafetyBackupFailed as error:
            self.logger.error("Safety backup failed", exc_info=True)
            _fail_job(job, "backup-failed", backup_failed_message(error))
            return False, None
        job.log_line(f"Safety backup {backup} verified")
        return True, backup

    async def _prune_backup_pool(self, job, db):
        """Keep the one ``<db>.backup_*`` pool (restore's too) at its cap."""
        removed = await self._run_in_executor(_prune_safety_backups, db.model.db_path)
        if removed:
            job.log_line(f"Removed {len(removed)} older safety backups")

    async def _run_maintenance_plan(self, job, db, plan, backup_on):
        ok, backup = await self._safety_backup_step(job, db, backup_on)
        if not ok:
            return
        try:
            await self._apply_maintenance_plan(job, db, plan, backup)
        finally:
            if backup:
                await self._prune_backup_pool(job, db)

    async def _apply_maintenance_plan(self, job, db, plan, backup):
        payload = plan.payload
        job.update(status_line=f"Applying: {job.title}")
        try:
            result = await self._run_in_executor(
                db.apply_maintenance,
                plan.operation,
                plan.fingerprint,
                payload["targets"],
                payload["roots"],
            )
        except PlanChanged:
            return _fail_job(job, "plan-changed", PLAN_CHANGED)
        except Exception:
            self.logger.exception(f"Maintenance {plan.operation} failed")
            return _fail_job(job, None, APPLY_FAILED)
        rows = await self._run_in_executor(
            self._maintenance_rows, plan.operation, result["rows"]
        )
        job.record_counted("changed", result["changed"], rows)
        job.record_counted("protected", payload["protected"])
        line = _changed_line(plan.operation, result["changed"])
        job.log_line(line)
        job.update(progress=100, status_line=f"{line} · {backup_note(backup)}")

    def _maintenance_rows(self, operation, described):
        """Outcome rows for the changed records (root-relative paths only)."""
        if operation == "missing_links":
            roots = _scan_roots(self._get_all_output_dirs())
            rows = []
            for item in described:
                path, label, index = _scan_location(item["image_path"], roots)
                rows.append(
                    {
                        "path": path,
                        "root": label,
                        "root_index": index,
                        "detail": "Link removed; the file was already missing",
                    }
                )
            return rows
        if operation == "duplicate_prompts":
            return [
                {
                    "path": "",
                    "detail": (
                        f"Prompt {item['prompt_id']} merged into "
                        f"prompt {item['kept_id']}"
                    ),
                }
                for item in described
            ]
        return [
            {"path": "", "detail": f"Removed: {item.get('text') or item['prompt_id']}"}
            for item in described
        ]


def backup_note(backup):
    """The status line's backup part: the file name, or that there is none."""
    return f"Safety backup {backup}" if backup else NO_BACKUP


def _fail_job(job, code, message):
    """End ``job`` failed with ``message`` (one error row) and its code.

    The code is also the job's ``scope["failure"]``, so Activity can offer
    Review again for a plan the library overtook.
    """
    if code:
        job.scope["failure"] = code
    job.record("errors", {"path": "", "detail": message}, f"! {message}")
    job.update(status_line=message)
    job.publish({"type": "error", "code": code, "message": message})
    job.finish("failed")