fix(maintenance): never remove the kept copy; move records before unlink
Final review of #313: - C1: the duplicate find tells files apart by (st_dev, st_ino), so a source listed twice, a symlinked source inside another and a hardlink are one file. The preview and the job refuse a removal that os.path.samestat matches with its kept copy. - I1: each file's records move to its verified kept copy in one short transaction before that file is unlinked; a failed move deletes nothing and a failed unlink leaves the records on the identical kept copy. The status line counts files removed and not removed. - I2: links are looked up under every spelling of the source (the real path, the configured and monitored folders, ComfyUI's output). - I5: image tags join the kept copy as their union and the removed copy's pending, undecided Needs review items are settled; the plan says so. - I3: the VACUUM disk check measures SQLite's real temporary folder. - M1: the token is looked at again with no await before the job starts. - M2: the backup decision is made once and handed to the job. - M3: backups are linked into place; startup sweeps interrupted temps. - M4: Optimize runs alone (new "exclusive" admission) and its plan says the database is locked while VACUUM runs. - M5: the reviewed duplicate merge is strict: collisions and Library titles are protected, and errors roll back. The legacy cleanup is unchanged. - M7: 2,000 keep entries within a 1 MiB body. M8: event waits in tests. - M9: the preview binds files by size, mtime and inode without hashing; the job hashes right before each unlink.
This commit is contained in:
Vito Sansevero
@@ -360,6 +360,16 @@ class PromptManagerAPI(
|
||||
# forgets them (py/api/maintenance.py).
|
||||
self._maintenance_plans = PlanStore()
|
||||
|
||||
# A safety backup interrupted by a crash leaves its temp copy behind.
|
||||
try:
|
||||
from .maintenance import sweep_safety_temps
|
||||
|
||||
swept = sweep_safety_temps(self.db.model.db_path)
|
||||
if swept:
|
||||
self.logger.info(f"Removed {len(swept)} interrupted backup copies")
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Could not sweep interrupted backup copies: {e}")
|
||||
|
||||
# Run cleanup on initialization to remove any existing duplicates
|
||||
try:
|
||||
removed = self.db.cleanup_duplicates()
|
||||
|
||||
+30
-7
@@ -135,6 +135,17 @@ def _collect_media_files(output_dirs, extensions):
|
||||
return found
|
||||
|
||||
|
||||
def _physical_file_key(path, stat):
|
||||
"""What tells two physical files apart: ``(st_dev, st_ino)``.
|
||||
|
||||
A filesystem that reports no inode (``st_ino`` 0, some network or FAT
|
||||
volumes on Windows) falls back to the resolved, case-folded path.
|
||||
"""
|
||||
if stat.st_ino:
|
||||
return ("inode", stat.st_dev, stat.st_ino)
|
||||
return ("path", os.path.normcase(os.path.realpath(str(path))))
|
||||
|
||||
|
||||
def _thumbnail_rel_path(rel_path, thumbnail_ext):
|
||||
"""Relative path of the thumbnail the gallery would generate for ``rel_path``."""
|
||||
return f"thumbnails/{rel_path.with_suffix('').as_posix()}_thumb{thumbnail_ext}"
|
||||
@@ -743,13 +754,18 @@ class AdminRoutesMixin:
|
||||
"""Content-identical media files under ``roots`` (blocking).
|
||||
|
||||
Each root is walked as ``_collect_media_files`` walks (no symlinks,
|
||||
no ``thumbnails``, bounded); a file is listed once, under the first
|
||||
root that holds it, and at most MAX_SCAN_FILES files are read. Files
|
||||
no ``thumbnails``, bounded). A physical file is listed once, under
|
||||
the first root that holds it: files are told apart by ``(st_dev,
|
||||
st_ino)``, so a source listed twice under two spellings (a symlink, a
|
||||
bind mount, a case variant), a source inside another one or a
|
||||
hardlink is one file, never a copy of itself. At most MAX_SCAN_FILES
|
||||
files are read. Files
|
||||
are hashed on ``PromptManagerConfig.WORKER_THREADS`` threads;
|
||||
``progress(done, total)`` is called from those threads about every
|
||||
percent and once at the end, and ``stop()`` true skips the files not
|
||||
yet read. Returns ``[{hash, size, files:[{slot, abs, rel, size,
|
||||
mtime}]}]`` for every hash shared by two or more files, ``slot``
|
||||
mtime, mtime_ns, inode}]}]`` for every hash shared by two or more
|
||||
files (``inode`` is ``[st_dev, st_ino]``), ``slot``
|
||||
being the root's place in ``roots`` and the files oldest first.
|
||||
"""
|
||||
from ..config import PromptManagerConfig
|
||||
@@ -759,10 +775,16 @@ class AdminRoutesMixin:
|
||||
seen = set()
|
||||
for slot, root in enumerate(roots):
|
||||
for path in _collect_media_files([root], extensions):
|
||||
key = os.path.normcase(str(path))
|
||||
try:
|
||||
stat = path.stat()
|
||||
except OSError:
|
||||
continue
|
||||
key = _physical_file_key(path, stat)
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
media_files.append((slot, root, path))
|
||||
media_files.append((slot, root, path, stat))
|
||||
if len(media_files) >= MAX_SCAN_FILES:
|
||||
break
|
||||
media_files = media_files[:MAX_SCAN_FILES]
|
||||
total = len(media_files)
|
||||
self.logger.info(f"Found {total} media files to analyze")
|
||||
@@ -771,17 +793,18 @@ class AdminRoutesMixin:
|
||||
lock = threading.Lock()
|
||||
|
||||
def describe(item):
|
||||
slot, root, media_path = item
|
||||
slot, root, media_path, stat = item
|
||||
info = None
|
||||
if stop is None or not stop():
|
||||
try:
|
||||
stat = media_path.stat()
|
||||
info = {
|
||||
"slot": slot,
|
||||
"abs": str(media_path),
|
||||
"rel": Path(os.path.relpath(str(media_path), root)).as_posix(),
|
||||
"size": stat.st_size,
|
||||
"mtime": stat.st_mtime,
|
||||
"mtime_ns": stat.st_mtime_ns,
|
||||
"inode": [stat.st_dev, stat.st_ino],
|
||||
"hash": self._calculate_file_hash(media_path),
|
||||
}
|
||||
except Exception as e:
|
||||
|
||||
+205
-94
@@ -57,11 +57,12 @@ from .duplicate_review import ( # noqa: F401 (CHANGED, OUTSIDE_SOURCE: public)
|
||||
KEEPER,
|
||||
KEEPER_CHANGED,
|
||||
KEPT_COPY,
|
||||
LINKS_FAILED,
|
||||
LOCKED_FILE,
|
||||
NOT_ATTEMPTED,
|
||||
OUTSIDE_SOURCE,
|
||||
RECORDS_FAILED,
|
||||
ROW_DETAILS,
|
||||
SAME_FILE,
|
||||
SOURCES_CHANGED,
|
||||
SOURCES_CHANGED_REVIEW,
|
||||
HeldFind,
|
||||
@@ -88,9 +89,9 @@ from .jobs import (
|
||||
from .maintenance import DUPLICATE_FILES, _fail_job, _failure, _invalid, backup_note
|
||||
|
||||
try:
|
||||
from ...database.maintenance import FILE_LINK_SQL
|
||||
from ...database.duplicate_copies import FILE_LINK_SQL
|
||||
except ImportError:
|
||||
from database.maintenance import FILE_LINK_SQL
|
||||
from database.duplicate_copies import FILE_LINK_SQL
|
||||
|
||||
DUPLICATE_FIND_TITLE = "Find duplicate image files"
|
||||
DUPLICATE_CLEANUP_TITLE = "Remove duplicate image files"
|
||||
@@ -432,10 +433,14 @@ class DuplicateFilesMixin:
|
||||
)
|
||||
|
||||
def _review_duplicate_files(self, db, selection):
|
||||
"""Check every keeper and removal now (blocking); the plan's parts."""
|
||||
"""Check every keeper and removal now (blocking); the plan's parts.
|
||||
|
||||
No file is read: each copy is bound to what the search saw by its
|
||||
size, mtime and inode, and never to its kept copy itself. The removal
|
||||
job hashes each file again right before it unlinks it.
|
||||
"""
|
||||
listed = self._listed_sources()
|
||||
targets, kept, protected = [], [], {}
|
||||
hasher = self._calculate_file_hash
|
||||
for group, keepers in selection.values():
|
||||
removals = [r for r in group["files"] if r not in keepers]
|
||||
for record in removals:
|
||||
@@ -444,15 +449,7 @@ class DuplicateFilesMixin:
|
||||
):
|
||||
raise SourcesChanged(record["root"])
|
||||
keeper = next(
|
||||
(
|
||||
k
|
||||
for k in keepers
|
||||
if file_problem(
|
||||
k["abs"], k["root"], group["hash"], k["size"], hasher
|
||||
)
|
||||
is None
|
||||
),
|
||||
None,
|
||||
(k for k in keepers if _bound_problem(k, group) is None), None
|
||||
)
|
||||
reviewed = []
|
||||
for record in removals:
|
||||
@@ -461,13 +458,7 @@ class DuplicateFilesMixin:
|
||||
elif record["root_index"] is None:
|
||||
problem = OUTSIDE_SOURCE
|
||||
else:
|
||||
problem = file_problem(
|
||||
record["abs"],
|
||||
record["root"],
|
||||
group["hash"],
|
||||
record["size"],
|
||||
hasher,
|
||||
)
|
||||
problem = _bound_problem(record, group, keeper["abs"])
|
||||
if problem:
|
||||
protected[problem] = protected.get(problem, 0) + 1
|
||||
continue
|
||||
@@ -475,14 +466,55 @@ class DuplicateFilesMixin:
|
||||
if reviewed:
|
||||
targets.extend(reviewed)
|
||||
kept.append(self._copy_row(keeper, KEPT_COPY))
|
||||
links = db.count_file_links([t["abs"] for t in targets]) if targets else {}
|
||||
records = {}
|
||||
if targets:
|
||||
spellings = self._source_spellings()
|
||||
records = db.count_copy_records(
|
||||
{
|
||||
t["abs"]: {
|
||||
"spellings": _file_spellings(spellings, t["root"], t["rel"]),
|
||||
"locator": t["locator"],
|
||||
"sha256": t["hash"],
|
||||
}
|
||||
for t in targets
|
||||
}
|
||||
)
|
||||
return {
|
||||
"targets": targets,
|
||||
"kept": kept,
|
||||
"fingerprint": _fingerprint(targets),
|
||||
"public": _public_plan(targets, protected, links),
|
||||
"public": _public_plan(targets, protected, records),
|
||||
}
|
||||
|
||||
def _source_spellings(self):
|
||||
"""``{source identity: [spellings]}`` the monitor and config use (blocking).
|
||||
|
||||
The search walks each source's real path, while the image monitor
|
||||
links a file under the folder it watches: the configured spelling,
|
||||
ComfyUI's output folder or a symlink to it. Each absolute candidate
|
||||
is listed under the identity (``normcase(realpath)``) it resolves to.
|
||||
"""
|
||||
from ..config import GalleryConfig
|
||||
|
||||
candidates = list(self._monitored_directories())
|
||||
candidates += list(GalleryConfig.MONITORING_DIRECTORIES)
|
||||
try:
|
||||
import folder_paths
|
||||
|
||||
candidates.append(folder_paths.get_output_directory())
|
||||
except Exception: # nosec B110 - outside ComfyUI there is none
|
||||
pass
|
||||
candidates.append(self._find_comfyui_output_dir())
|
||||
spellings = {}
|
||||
for candidate in candidates:
|
||||
if not candidate or not os.path.isabs(str(candidate)):
|
||||
continue
|
||||
spelled = os.path.abspath(str(candidate))
|
||||
listed = spellings.setdefault(_identity(spelled), [])
|
||||
if spelled not in listed:
|
||||
listed.append(spelled)
|
||||
return spellings
|
||||
|
||||
def _target(self, group, record, keeper):
|
||||
return {
|
||||
"group_id": group["group_id"],
|
||||
@@ -497,6 +529,8 @@ class DuplicateFilesMixin:
|
||||
"keeper_rel": keeper["rel"],
|
||||
"keeper_root": keeper["root"],
|
||||
"keeper_size": keeper["size"],
|
||||
"locator": os.path.normcase(record["abs"]),
|
||||
"keeper_locator": os.path.normcase(keeper["abs"]),
|
||||
}
|
||||
|
||||
def _copy_row(self, record, detail):
|
||||
@@ -509,7 +543,7 @@ class DuplicateFilesMixin:
|
||||
|
||||
# ── removal ───────────────────────────────────────────────────────
|
||||
|
||||
def _start_duplicate_cleanup(self, db, plan):
|
||||
def _start_duplicate_cleanup(self, db, plan, backup_on):
|
||||
"""Register the removal job for a reviewed plan (on the loop)."""
|
||||
find = self._jobs.active(DUPLICATES_JOB_KIND)
|
||||
if find is not None:
|
||||
@@ -521,7 +555,7 @@ class DuplicateFilesMixin:
|
||||
cancellable=False,
|
||||
scope={"operation": DUPLICATE_FILES},
|
||||
groups=DUPLICATE_CLEANUP_GROUP_NAMES,
|
||||
runner=lambda job: self._run_duplicate_cleanup(job, db, plan),
|
||||
runner=lambda job: self._run_duplicate_cleanup(job, db, plan, backup_on),
|
||||
)
|
||||
|
||||
def _sources_unchanged(self, targets):
|
||||
@@ -530,12 +564,12 @@ class DuplicateFilesMixin:
|
||||
self._source_listed(listed, t["root_index"], t["root"]) for t in targets
|
||||
)
|
||||
|
||||
async def _run_duplicate_cleanup(self, job, db, plan):
|
||||
async def _run_duplicate_cleanup(self, job, db, plan, backup_on):
|
||||
targets = plan.payload["targets"]
|
||||
unchanged = await self._run_in_executor(self._sources_unchanged, targets)
|
||||
if not unchanged:
|
||||
return _fail_job(job, "sources-changed", SOURCES_CHANGED)
|
||||
ok, backup = await self._safety_backup_step(job, db, plan)
|
||||
ok, backup = await self._safety_backup_step(job, db, backup_on)
|
||||
if not ok:
|
||||
return
|
||||
try:
|
||||
@@ -549,36 +583,60 @@ class DuplicateFilesMixin:
|
||||
for row in plan.payload["kept"]:
|
||||
job.record("kept", row, f"= {row['root']}/{row['path']} · {KEPT_COPY}")
|
||||
job.update(total=len(targets), status_line="Removing duplicate files")
|
||||
removed, failure, keepers = [], None, {}
|
||||
spellings = await self._run_in_executor(self._source_spellings)
|
||||
removed, failure, keepers = 0, None, {}
|
||||
totals = dict.fromkeys(("moved", "merged", "favorites", "tags", "review"), 0)
|
||||
for done, target in enumerate(targets, start=1):
|
||||
if failure is not None:
|
||||
self._record_target(job, "kept", target, NOT_ATTEMPTED)
|
||||
continue
|
||||
outcome, code = await self._run_in_executor(
|
||||
self._remove_one_duplicate, target, keepers
|
||||
outcome, detail, counts = await self._run_in_executor(
|
||||
self._remove_one_duplicate, db, target, keepers, spellings
|
||||
)
|
||||
for key, value in (counts or {}).items():
|
||||
totals[key] += value
|
||||
if outcome == "removed":
|
||||
removed.append(target)
|
||||
detail = f"A copy of {target['keeper_rel']}"
|
||||
self._record_target(job, "removed", target, detail, "-")
|
||||
removed += 1
|
||||
text = f"A copy of {target['keeper_rel']}"
|
||||
self._record_target(job, "removed", target, text, "-")
|
||||
elif outcome == "kept":
|
||||
self._record_target(job, "kept", target, ROW_DETAILS[code])
|
||||
self._record_target(job, "kept", target, ROW_DETAILS[detail])
|
||||
else:
|
||||
failure = code
|
||||
self._record_target(job, "errors", target, code, "!")
|
||||
failure = detail
|
||||
self._record_target(job, "errors", target, detail, "!")
|
||||
job.update(processed=done, progress=int(done / len(targets) * 100))
|
||||
links_ok = await self._move_links(job, db, removed)
|
||||
if removed:
|
||||
self.invalidate_gallery_cache()
|
||||
line = f"Removed {_plural(len(removed), 'duplicate file')}"
|
||||
self._log_moved_records(job, totals)
|
||||
line = f"Removed {_plural(removed, 'duplicate file')}"
|
||||
if removed < len(targets):
|
||||
line += f", {len(targets) - removed:,} not removed"
|
||||
job.log_line(line)
|
||||
if failure is not None or not links_ok:
|
||||
line = "Stopped after an error · " + line if failure else line
|
||||
job.update(progress=100, status_line=f"{line} · {backup_note(backup)}")
|
||||
if failure is not None:
|
||||
line = f"Stopped after an error · {line}"
|
||||
job.update(progress=100, status_line=f"{line} · {backup_note(backup)}")
|
||||
if failure is not None:
|
||||
job.publish({"type": "error", "code": "delete-failed", "message": line})
|
||||
job.finish("failed")
|
||||
return
|
||||
job.update(progress=100, status_line=f"{line} · {backup_note(backup)}")
|
||||
|
||||
@staticmethod
|
||||
def _log_moved_records(job, totals):
|
||||
links = totals["moved"] + totals["merged"]
|
||||
if links:
|
||||
job.log_line(f"Moved {_plural(links, 'database link')} to the kept copies")
|
||||
if totals["favorites"]:
|
||||
job.log_line(
|
||||
f"Moved {_plural(totals['favorites'], 'favorite')} to the kept copies"
|
||||
)
|
||||
if totals["tags"]:
|
||||
job.log_line(
|
||||
f"Added {_plural(totals['tags'], 'image tag')} to the kept copies"
|
||||
)
|
||||
if totals["review"]:
|
||||
job.log_line(
|
||||
f"Settled {_plural(totals['review'], 'Needs review item')} "
|
||||
"of removed copies"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _record_target(job, group, target, detail, mark="="):
|
||||
@@ -592,73 +650,126 @@ class DuplicateFilesMixin:
|
||||
group, row, f"{mark} {target['root_label']}/{target['rel']} · {detail}"
|
||||
)
|
||||
|
||||
def _remove_one_duplicate(self, target, keepers):
|
||||
"""``(outcome, detail or code)`` for one reviewed removal (blocking).
|
||||
def _remove_one_duplicate(self, db, target, keepers, spellings):
|
||||
"""``(outcome, detail, counts)`` for one reviewed removal (blocking).
|
||||
|
||||
``keepers`` caches each group's kept-copy check for this job.
|
||||
The kept copy (once per group; cached in ``keepers``) and the file
|
||||
are checked again, hashes included, and the file must not be its kept
|
||||
copy. Then the file's records move to the kept copy in one short
|
||||
transaction, and only then is the file unlinked: a crash or a failed
|
||||
unlink leaves its records on the identical kept copy, never on a
|
||||
deleted file. A failed move leaves the file and its records alone.
|
||||
"""
|
||||
hasher = self._calculate_file_hash
|
||||
group_id = target["group_id"]
|
||||
if group_id not in keepers:
|
||||
keepers[group_id] = (
|
||||
file_problem(
|
||||
target["keeper_abs"],
|
||||
target["keeper_root"],
|
||||
target["hash"],
|
||||
target["keeper_size"],
|
||||
hasher,
|
||||
)
|
||||
is None
|
||||
)
|
||||
if not keepers[group_id]:
|
||||
return "kept", KEEPER
|
||||
keepers[group_id] = _kept_copy_state(target, hasher)
|
||||
if keepers[group_id] is None:
|
||||
return "kept", KEEPER, None
|
||||
problem = file_problem(
|
||||
target["abs"], target["root"], target["hash"], target["size"], hasher
|
||||
target["abs"],
|
||||
target["root"],
|
||||
target["hash"],
|
||||
target["size"],
|
||||
hasher,
|
||||
keeper=target["keeper_abs"],
|
||||
)
|
||||
if problem:
|
||||
return "kept", problem
|
||||
return "kept", problem, None
|
||||
try:
|
||||
counts = db.move_copy_records(
|
||||
self._copy_move(target, keepers[group_id], spellings)
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception(f"Could not move the records of {target['rel']}")
|
||||
return "error", RECORDS_FAILED, None
|
||||
try:
|
||||
_unlink(target["abs"])
|
||||
except FileNotFoundError:
|
||||
return "kept", GONE
|
||||
return "removed", None, counts
|
||||
except PermissionError:
|
||||
self.logger.warning(f"Duplicate file is locked: {target['rel']}")
|
||||
return "error", LOCKED_FILE
|
||||
self.logger.warning(
|
||||
f"Duplicate file is locked: {target['rel']}; "
|
||||
"its records now name the kept copy"
|
||||
)
|
||||
return "error", LOCKED_FILE, counts
|
||||
except OSError:
|
||||
self.logger.warning(
|
||||
f"Could not delete duplicate file {target['rel']}", exc_info=True
|
||||
f"Could not delete duplicate file {target['rel']}; "
|
||||
"its records now name the kept copy",
|
||||
exc_info=True,
|
||||
)
|
||||
return "error", DELETE_FAILED
|
||||
return "error", DELETE_FAILED, counts
|
||||
self.logger.info(f"Deleted duplicate file: {target['rel']}")
|
||||
_remove_thumbnail(Path(target["root"]), Path(target["rel"]), self.logger)
|
||||
return "removed", None
|
||||
return "removed", None, counts
|
||||
|
||||
async def _move_links(self, job, db, removed):
|
||||
"""Move the removed files' links to their kept copy; False on failure."""
|
||||
if not removed:
|
||||
return True
|
||||
moves = [(t["abs"], t["keeper_abs"]) for t in removed]
|
||||
favorites = [
|
||||
(
|
||||
_favorite_root_key(t["root"]),
|
||||
_favorite_relative_key(t["rel"]),
|
||||
_favorite_root_key(t["keeper_root"]),
|
||||
_favorite_relative_key(t["keeper_rel"]),
|
||||
@staticmethod
|
||||
def _copy_move(target, kept_stat, spellings):
|
||||
"""What ``move_copy_records`` moves for one removal."""
|
||||
removed = _file_spellings(spellings, target["root"], target["rel"])
|
||||
kept = _file_spellings(spellings, target["keeper_root"], target["keeper_rel"])
|
||||
same_source = _identity(target["root"]) == _identity(target["keeper_root"])
|
||||
native_rel = target["rel"].replace("/", os.sep)
|
||||
native_kept = target["keeper_rel"].replace("/", os.sep)
|
||||
targets = {}
|
||||
for spelled in removed:
|
||||
prefix = spelled[: len(spelled) - len(native_rel)]
|
||||
targets[spelled] = (
|
||||
prefix + native_kept if same_source else target["keeper_abs"]
|
||||
)
|
||||
for t in removed
|
||||
]
|
||||
try:
|
||||
counts = await self._run_in_executor(db.move_file_links, moves, favorites)
|
||||
except Exception:
|
||||
self.logger.exception("Moving duplicate file links failed")
|
||||
job.record(
|
||||
"errors", {"path": "", "detail": LINKS_FAILED}, f"! {LINKS_FAILED}"
|
||||
)
|
||||
return False
|
||||
links = counts["moved"] + counts["merged"]
|
||||
job.log_line(f"Moved {_plural(links, 'database link')} to the kept copies")
|
||||
if counts["favorites"]:
|
||||
job.log_line(
|
||||
f"Moved {_plural(counts['favorites'], 'favorite')} to the kept copies"
|
||||
)
|
||||
return True
|
||||
return {
|
||||
"removed": removed,
|
||||
"kept": kept,
|
||||
"targets": targets,
|
||||
"kept_path": target["keeper_abs"],
|
||||
"favorite": (
|
||||
_favorite_root_key(target["root"]),
|
||||
_favorite_relative_key(target["rel"]),
|
||||
_favorite_root_key(target["keeper_root"]),
|
||||
_favorite_relative_key(target["keeper_rel"]),
|
||||
),
|
||||
"removed_locator": target["locator"],
|
||||
"kept_locator": target["keeper_locator"],
|
||||
"sha256": target["hash"],
|
||||
"kept_size": kept_stat.st_size,
|
||||
"kept_mtime_ns": kept_stat.st_mtime_ns,
|
||||
}
|
||||
|
||||
|
||||
def _bound_problem(record, group, keeper=None):
|
||||
"""``file_problem`` without reading the file: size, mtime and inode."""
|
||||
return file_problem(
|
||||
record["abs"],
|
||||
record["root"],
|
||||
group["hash"],
|
||||
record["size"],
|
||||
hash_check=False,
|
||||
mtime_ns=record.get("mtime_ns"),
|
||||
inode=record.get("inode"),
|
||||
keeper=keeper,
|
||||
)
|
||||
|
||||
|
||||
def _kept_copy_state(target, hasher):
|
||||
"""The kept copy's ``os.stat`` when it is still the reviewed bytes, else None."""
|
||||
problem = file_problem(
|
||||
target["keeper_abs"],
|
||||
target["keeper_root"],
|
||||
target["hash"],
|
||||
target["keeper_size"],
|
||||
hasher,
|
||||
)
|
||||
if problem:
|
||||
return None
|
||||
try:
|
||||
return os.stat(target["keeper_abs"])
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def _file_spellings(spellings, root, rel):
|
||||
"""Every path one file is linked by: its real path, then each spelling."""
|
||||
native = rel.replace("/", os.sep)
|
||||
roots = [root] + spellings.get(_identity(root), [])
|
||||
return list(dict.fromkeys(os.path.join(spelled, native) for spelled in roots))
|
||||
|
||||
+87
-32
@@ -16,7 +16,9 @@ from pathlib import Path
|
||||
from .admin import DELETABLE_MEDIA_EXTENSIONS
|
||||
from .maintenance import DUPLICATE_FILES
|
||||
|
||||
MAX_KEEP = 5000 # keep entries one review takes
|
||||
# Keep entries one review takes; with paths of a few hundred characters the
|
||||
# list stays inside the preview body cap (DUPLICATE_PREVIEW_BODY_MAX, 1 MiB).
|
||||
MAX_KEEP = 2000
|
||||
MAX_DETAILS = 50
|
||||
MAX_JOB_ID = 64
|
||||
HASH_CHUNK = 1024 * 1024
|
||||
@@ -28,7 +30,12 @@ CHANGED_CONTENT = "changed"
|
||||
GONE = "gone"
|
||||
UNREADABLE = "unreadable"
|
||||
KEEPER = "keeper"
|
||||
SAME = "same"
|
||||
PROTECTED_REASONS = {
|
||||
SAME: (
|
||||
"file is the same file as its kept copy",
|
||||
"files are the same file as their kept copy",
|
||||
),
|
||||
OUTSIDE_SOURCE: (
|
||||
"file is not inside a configured source",
|
||||
"files are not inside a configured source",
|
||||
@@ -48,7 +55,9 @@ PROTECTED_REASONS = {
|
||||
# Job outcome rows (never an OS error text).
|
||||
CHANGED = "The file changed since the review; it was kept."
|
||||
KEEPER_CHANGED = "The kept copy changed or is gone; this file was kept."
|
||||
SAME_FILE = "This is the same file as its kept copy; it was kept."
|
||||
ROW_DETAILS = {
|
||||
SAME: SAME_FILE,
|
||||
OUTSIDE_SOURCE: "The file is no longer inside its source; it was kept.",
|
||||
LINK: "The file is now a link, not a regular file; it was kept.",
|
||||
CHANGED_CONTENT: CHANGED,
|
||||
@@ -63,9 +72,9 @@ KEPT_COPY = "Kept copy"
|
||||
SOURCES_CHANGED = (
|
||||
"The sources changed since the review; nothing was deleted. Review again."
|
||||
)
|
||||
LINKS_FAILED = (
|
||||
"The files were removed, but their database links could not be moved. "
|
||||
"Review Missing-file links to tidy them."
|
||||
RECORDS_FAILED = (
|
||||
"The database could not be updated, so the file was not deleted. "
|
||||
"Its links and tags are unchanged."
|
||||
)
|
||||
FIND_FAILED = "The duplicate search failed."
|
||||
FIND_ERROR_EVENT = "An internal error occurred. Check server logs for details."
|
||||
@@ -99,15 +108,30 @@ def _hash_file(path):
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def file_problem(path, root, digest, size, hasher=None):
|
||||
def file_problem(
|
||||
path,
|
||||
root,
|
||||
digest,
|
||||
size,
|
||||
hasher=None,
|
||||
*,
|
||||
hash_check=True,
|
||||
mtime_ns=None,
|
||||
inode=None,
|
||||
keeper=None,
|
||||
):
|
||||
"""Why ``path`` may not be treated as a copy in ``root``, or None (blocking).
|
||||
|
||||
Containment is decided as text first, with no filesystem access: the
|
||||
normalized path must lie below the normalized root, outside
|
||||
``thumbnails``, with a media extension. Only then is the file looked at:
|
||||
a regular file, not a link, whose resolved path is still inside the
|
||||
resolved root, of ``size`` bytes and with content hash ``digest``. Every
|
||||
handle is closed when this returns.
|
||||
resolved root and that is not ``keeper`` itself (``os.path.samestat``: a
|
||||
hardlink, or one file reached through two spellings), of ``size`` bytes.
|
||||
``mtime_ns`` and ``inode`` (``[st_dev, st_ino]``), when given, bind it to
|
||||
what the search saw; ``hash_check`` compares its content hash with
|
||||
``digest`` (the removal job, right before an unlink). Every handle is
|
||||
closed when this returns.
|
||||
"""
|
||||
# py.config imports ComfyUI's server module: only when it is in use.
|
||||
from ..config import _canonical_path, _lexical_path, _lexically_inside
|
||||
@@ -134,9 +158,15 @@ def file_problem(path, root, digest, size, hasher=None):
|
||||
try:
|
||||
if not _lexically_inside(_canonical_path(path), [_canonical_path(root)]):
|
||||
return OUTSIDE_SOURCE # a linked folder on the way
|
||||
if keeper is not None and os.path.samestat(info, os.stat(keeper)):
|
||||
return SAME
|
||||
if info.st_size != size:
|
||||
return CHANGED_CONTENT
|
||||
if (hasher or _hash_file)(path) != digest:
|
||||
if mtime_ns is not None and info.st_mtime_ns != mtime_ns:
|
||||
return CHANGED_CONTENT
|
||||
if inode is not None and [info.st_dev, info.st_ino] != list(inode):
|
||||
return CHANGED_CONTENT
|
||||
if hash_check and (hasher or _hash_file)(path) != digest:
|
||||
return CHANGED_CONTENT
|
||||
except FileNotFoundError:
|
||||
return GONE
|
||||
@@ -197,7 +227,7 @@ def _parse_options(options):
|
||||
if not isinstance(job_id, str) or not 0 < len(job_id) <= MAX_JOB_ID:
|
||||
return None, None, "job_id must be a duplicate search's job id"
|
||||
if not isinstance(keep, list) or not 0 < len(keep) <= MAX_KEEP:
|
||||
return None, None, f"keep must list 1 to {MAX_KEEP} files to keep"
|
||||
return None, None, f"keep must list 1 to {MAX_KEEP:,} files to keep"
|
||||
parsed = []
|
||||
for entry in keep:
|
||||
key, message = _keep_entry(entry)
|
||||
@@ -246,7 +276,29 @@ def _fingerprint(targets):
|
||||
return "sha256:" + digest.hexdigest()
|
||||
|
||||
|
||||
def _public_plan(targets, protected, links):
|
||||
def _record_parts(records, target):
|
||||
"""What moves with one removal: its links, tags and Needs review items."""
|
||||
found = records.get(target["abs"], {})
|
||||
links, tags, review = (found.get(key, 0) for key in ("links", "tags", "review"))
|
||||
parts = [
|
||||
(
|
||||
(
|
||||
f"{_plural(links, 'database link')} "
|
||||
f"{'moves' if links == 1 else 'move'} to the kept copy"
|
||||
)
|
||||
if links
|
||||
else "no database links"
|
||||
)
|
||||
]
|
||||
if tags:
|
||||
parts.append("its image tags join the kept copy's")
|
||||
if review:
|
||||
item = "Needs review item is" if review == 1 else "Needs review items are"
|
||||
parts.append(f"{review:,} {item} settled")
|
||||
return parts
|
||||
|
||||
|
||||
def _public_plan(targets, protected, records):
|
||||
count = len(targets)
|
||||
groups = len({t["group_id"] for t in targets})
|
||||
size = sum(t["size"] for t in targets)
|
||||
@@ -254,32 +306,35 @@ def _public_plan(targets, protected, links):
|
||||
f"{n:,} {PROTECTED_REASONS[code][0 if n == 1 else 1]}"
|
||||
for code, n in sorted(protected.items())
|
||||
]
|
||||
details = []
|
||||
for target in targets[:MAX_DETAILS]:
|
||||
n = links.get(target["abs"], 0)
|
||||
moves = (
|
||||
f"{_plural(n, 'database link')} "
|
||||
f"{'moves' if n == 1 else 'move'} to the kept copy"
|
||||
if n
|
||||
else "no database links"
|
||||
)
|
||||
details.append(
|
||||
{
|
||||
"label": target["rel"],
|
||||
"detail": f"{target['root_label']} · a copy of {target['keeper_rel']}"
|
||||
f" · {moves}",
|
||||
}
|
||||
)
|
||||
details = [
|
||||
{
|
||||
"label": target["rel"],
|
||||
"detail": f"{target['root_label']} · a copy of {target['keeper_rel']}"
|
||||
+ "".join(f" · {part}" for part in _record_parts(records, target)),
|
||||
}
|
||||
for target in targets[:MAX_DETAILS]
|
||||
]
|
||||
label = f"{_plural(count, 'duplicate file')} in {_plural(groups, 'group')}"
|
||||
media = (
|
||||
f"Deletes {_plural(count, 'file')} ({human_bytes(size)}) from disk; the "
|
||||
"kept copies stay. The safety backup covers the database, not these files."
|
||||
)
|
||||
tagged = sum(1 for t in targets if records.get(t["abs"], {}).get("tags"))
|
||||
review = sum(records.get(t["abs"], {}).get("review", 0) for t in targets)
|
||||
if tagged:
|
||||
media += (
|
||||
f" The image tags of {_plural(tagged, 'removed copy', 'removed copies')}"
|
||||
" join their kept copy's."
|
||||
)
|
||||
if review:
|
||||
media += (
|
||||
f" {_plural(review, 'pending Needs review item')} of the removed "
|
||||
f"{'copy is' if review == 1 else 'copies are'} settled."
|
||||
)
|
||||
return {
|
||||
"operation": DUPLICATE_FILES,
|
||||
"scope": {"count": count, "label": label},
|
||||
"protected": {"count": sum(protected.values()), "reasons": reasons},
|
||||
"media": {
|
||||
"deletes_files": True,
|
||||
"label": f"Deletes {_plural(count, 'file')} ({human_bytes(size)}) from "
|
||||
"disk; the kept copies stay. The safety backup covers the database, "
|
||||
"not these files.",
|
||||
},
|
||||
"media": {"deletes_files": True, "label": media},
|
||||
"details": details,
|
||||
}
|
||||
|
||||
+11
-5
@@ -19,7 +19,11 @@ ones. One job of a kind runs at a time, and each job has an admission class
|
||||
- ``guarded`` must not overlap record changes but may overlap other guarded
|
||||
work (thumbnail generation and clearing, later backup). It is refused while
|
||||
a ``records`` job runs.
|
||||
- ``read`` only reads; it is refused only by a running job of its own kind.
|
||||
- ``read`` only reads; it is refused only by a running job of its own kind
|
||||
or an ``exclusive`` one.
|
||||
- ``exclusive`` runs alone (Optimize's VACUUM, which rewrites the whole
|
||||
database under the write lock): it is refused while any job runs, and every
|
||||
job is refused while it runs.
|
||||
|
||||
A short legacy-page operation takes a ``hold``: a running, non-cancellable
|
||||
job that lasts as long as the request, so the Activity page lists it and a
|
||||
@@ -75,12 +79,14 @@ IMAGE_TAG_GROUP_NAMES = ("tagged", "unchanged", "skipped", "errors")
|
||||
# Thumbnail generation (#249 System S5): previews written, already fresh, failed.
|
||||
THUMBNAIL_GROUP_NAMES = ("generated", "skipped", "errors")
|
||||
|
||||
ADMISSIONS = ("read", "guarded", "records")
|
||||
ADMISSIONS = ("read", "guarded", "records", "exclusive")
|
||||
# What each admission class is refused beside (besides its own kind).
|
||||
# ``exclusive`` (Optimize's VACUUM) runs alone: beside no job of any class.
|
||||
_REFUSED_BESIDE = {
|
||||
"read": frozenset(),
|
||||
"guarded": frozenset({"records"}),
|
||||
"records": frozenset({"records", "guarded"}),
|
||||
"read": frozenset({"exclusive"}),
|
||||
"guarded": frozenset({"records", "exclusive"}),
|
||||
"records": frozenset({"records", "guarded", "exclusive"}),
|
||||
"exclusive": frozenset(ADMISSIONS),
|
||||
}
|
||||
|
||||
# Kinds of the legacy-page operations that hold the registry (D80; titles in
|
||||
|
||||
+145
-35
@@ -66,7 +66,7 @@ except ImportError:
|
||||
|
||||
PREVIEW_BODY_MAX = 4096
|
||||
# Duplicate-file previews carry the reviewed keepers (py/api/duplicate_files.py).
|
||||
DUPLICATE_PREVIEW_BODY_MAX = 512 * 1024
|
||||
DUPLICATE_PREVIEW_BODY_MAX = 1024 * 1024
|
||||
DUPLICATE_FILES = "duplicate_files"
|
||||
PREVIEW_OPERATIONS = MAINTENANCE_OPERATIONS + (DUPLICATE_FILES,)
|
||||
# Room left over beyond what a step writes, so a disk is never filled to 0.
|
||||
@@ -112,6 +112,36 @@ def _device(path):
|
||||
return path
|
||||
|
||||
|
||||
def sqlite_temp_dir(environ=None, *, windows=None, usable=None):
|
||||
"""The folder SQLite builds its temporary files in (VACUUM's copy).
|
||||
|
||||
Mirrors SQLite's own search, which differs from ``tempfile``'s. On Unix
|
||||
(``unixTempFileDir``): ``SQLITE_TMPDIR``, ``TMPDIR``, then ``/var/tmp``,
|
||||
``/usr/tmp`` and ``/tmp``, the first that is a writable directory, else
|
||||
the current directory. On Windows (``GetTempPathW``): ``TMP``, ``TEMP``,
|
||||
then ``USERPROFILE``, else the current directory.
|
||||
"""
|
||||
environ = os.environ if environ is None else environ
|
||||
if windows is None:
|
||||
windows = os.name == "nt"
|
||||
if windows:
|
||||
for name in ("TMP", "TEMP", "USERPROFILE"):
|
||||
if environ.get(name):
|
||||
return environ[name]
|
||||
return os.path.abspath(".")
|
||||
if usable is None:
|
||||
|
||||
def usable(path):
|
||||
return os.path.isdir(path) and os.access(path, os.W_OK | os.X_OK)
|
||||
|
||||
candidates = [environ.get("SQLITE_TMPDIR"), environ.get("TMPDIR")]
|
||||
candidates += ["/var/tmp", "/usr/tmp", "/tmp"]
|
||||
for candidate in candidates:
|
||||
if candidate and usable(candidate):
|
||||
return candidate
|
||||
return os.path.abspath(".")
|
||||
|
||||
|
||||
def database_bytes(db_path):
|
||||
"""The size a safety backup of the database will be (blocking).
|
||||
|
||||
@@ -135,8 +165,9 @@ def disk_plan(db_path, *, backup, vacuum):
|
||||
|
||||
``backup``: the safety backup, one database-sized file beside the
|
||||
database. ``vacuum``: VACUUM writes the rebuilt database through the WAL
|
||||
beside it and builds it in a temporary copy in the temporary folder
|
||||
(SQLite's documentation: up to twice the database in free space). Needs
|
||||
beside it and builds it in a temporary copy in SQLite's temporary folder
|
||||
(``sqlite_temp_dir``; SQLite's documentation: up to twice the database
|
||||
in free space). Needs
|
||||
on one device add up. Returns ``(public, short)``: ``public`` is
|
||||
``{free_bytes, required_bytes, sufficient}`` for the database folder,
|
||||
``short`` None or ``(place, what, required, free)`` of the first place
|
||||
@@ -151,7 +182,7 @@ def disk_plan(db_path, *, backup, vacuum):
|
||||
needs.append(("database folder", folder, size, "the safety backup"))
|
||||
if vacuum:
|
||||
needs.append(("database folder", folder, size, "Optimize"))
|
||||
needs.append(("temporary folder", tempfile.gettempdir(), size, "Optimize"))
|
||||
needs.append(("temporary folder", sqlite_temp_dir(), size, "Optimize"))
|
||||
devices = {}
|
||||
for place, path, needed, what in needs:
|
||||
entry = devices.setdefault(_device(path), {"place": place, "path": path})
|
||||
@@ -210,9 +241,7 @@ def _remove_quietly(*paths):
|
||||
|
||||
def _reserve_backup_name(db_path):
|
||||
"""Create and return an unused ``<db>.backup_<stamp>[_n]`` (O_EXCL)."""
|
||||
base = f"{db_path}.backup_{_now().strftime('%Y%m%d_%H%M%S_%f')}"
|
||||
for attempt in range(_NAME_ATTEMPTS):
|
||||
candidate = base if attempt == 0 else f"{base}_{attempt}"
|
||||
for candidate in _backup_names(db_path):
|
||||
try:
|
||||
fd = os.open(candidate, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
|
||||
except FileExistsError:
|
||||
@@ -222,6 +251,66 @@ def _reserve_backup_name(db_path):
|
||||
raise SafetyBackupFailed("No free backup name")
|
||||
|
||||
|
||||
def _backup_names(db_path):
|
||||
base = f"{db_path}.backup_{_now().strftime('%Y%m%d_%H%M%S_%f')}"
|
||||
for attempt in range(_NAME_ATTEMPTS):
|
||||
yield base if attempt == 0 else f"{base}_{attempt}"
|
||||
|
||||
|
||||
def _place_backup(temp, db_path):
|
||||
"""Give the verified ``temp`` its unique backup name; return that path.
|
||||
|
||||
``os.link`` is atomic and refuses an existing name, so no empty or half
|
||||
file ever carries a backup name, even across a crash. A filesystem
|
||||
without hard links falls back to reserving the name (``O_EXCL``) and
|
||||
``os.replace``. The temp file is removed by the caller.
|
||||
"""
|
||||
for candidate in _backup_names(db_path):
|
||||
try:
|
||||
os.link(temp, candidate)
|
||||
except FileExistsError:
|
||||
continue
|
||||
except (OSError, NotImplementedError):
|
||||
break # no hard links here
|
||||
else:
|
||||
return candidate
|
||||
else:
|
||||
raise SafetyBackupFailed("No free backup name")
|
||||
final = _reserve_backup_name(db_path)
|
||||
try:
|
||||
os.replace(temp, final)
|
||||
except OSError:
|
||||
_remove_quietly(final)
|
||||
raise
|
||||
return final
|
||||
|
||||
|
||||
SAFETY_TEMP_PREFIX = ".pm-safety-"
|
||||
|
||||
|
||||
def sweep_safety_temps(db_path):
|
||||
"""Remove the temp copies an interrupted safety backup left (blocking).
|
||||
|
||||
Called once at startup, before any job can take a backup. Returns the
|
||||
names removed.
|
||||
"""
|
||||
folder = os.path.dirname(os.path.abspath(db_path))
|
||||
removed = []
|
||||
try:
|
||||
names = os.listdir(folder)
|
||||
except OSError:
|
||||
return removed
|
||||
for name in names:
|
||||
if not (name.startswith(SAFETY_TEMP_PREFIX) and ".tmp" in name):
|
||||
continue
|
||||
try:
|
||||
os.remove(os.path.join(folder, name))
|
||||
removed.append(name)
|
||||
except OSError:
|
||||
continue
|
||||
return removed
|
||||
|
||||
|
||||
def _standalone(path):
|
||||
"""Make the copy one self-contained file (rollback journal, not WAL).
|
||||
|
||||
@@ -250,7 +339,7 @@ def take_safety_backup(model):
|
||||
"""
|
||||
db_path = os.path.abspath(model.db_path)
|
||||
fd, temp = tempfile.mkstemp(
|
||||
dir=os.path.dirname(db_path), prefix=".pm-safety-", suffix=".tmp"
|
||||
dir=os.path.dirname(db_path), prefix=SAFETY_TEMP_PREFIX, suffix=".tmp"
|
||||
)
|
||||
os.close(fd)
|
||||
final = None
|
||||
@@ -264,8 +353,7 @@ def take_safety_backup(model):
|
||||
_fsync(temp)
|
||||
if os.name != "nt":
|
||||
os.chmod(temp, 0o600)
|
||||
final = _reserve_backup_name(db_path)
|
||||
os.replace(temp, final)
|
||||
final = _place_backup(temp, db_path)
|
||||
return os.path.basename(final)
|
||||
except OSError as exc:
|
||||
_remove_quietly(final)
|
||||
@@ -445,21 +533,10 @@ class MaintenanceRoutesMixin:
|
||||
token = data.get("token")
|
||||
if not isinstance(token, str) or not 0 < len(token) <= MAX_TOKEN_LENGTH:
|
||||
return _invalid("token must be the plan's token")
|
||||
plan, refused = self._peek_plan(token)
|
||||
if refused is not None:
|
||||
return refused
|
||||
store = self._maintenance_plans
|
||||
try:
|
||||
plan = store.peek(token)
|
||||
except PlanUsed as used:
|
||||
job = self._jobs.get(used.job_id)
|
||||
return _failure(
|
||||
"This plan was already applied.",
|
||||
409,
|
||||
"plan-used",
|
||||
job={"id": used.job_id, "title": job.title if job else None},
|
||||
)
|
||||
except PlanExpired:
|
||||
return _failure(
|
||||
"This review expired. Review the plan again.", 410, "plan-expired"
|
||||
)
|
||||
db = self.db
|
||||
if plan.db_identity != self._database_identity(db):
|
||||
store.discard(token)
|
||||
@@ -482,36 +559,69 @@ class MaintenanceRoutesMixin:
|
||||
if short is not None:
|
||||
# Refused before any job starts: nothing fails halfway for space.
|
||||
return _failure(short_space_message(short), 507, "insufficient-space")
|
||||
# The token may have been used, expired or dropped while the disk was
|
||||
# measured: look again with no await before the job starts.
|
||||
plan, refused = self._peek_plan(token)
|
||||
if refused is not None:
|
||||
return refused
|
||||
try:
|
||||
if plan.operation == DUPLICATE_FILES:
|
||||
job = self._start_duplicate_cleanup(db, plan)
|
||||
job = self._start_duplicate_cleanup(db, plan, backup)
|
||||
else:
|
||||
job = self._jobs.start(
|
||||
MAINTENANCE_JOB_KIND,
|
||||
MAINTENANCE_TITLES[plan.operation],
|
||||
admission="records",
|
||||
# VACUUM rewrites the whole file under the write lock: it
|
||||
# runs alone, beside no other job of any class.
|
||||
admission=(
|
||||
"exclusive" if plan.operation == "optimize" else "records"
|
||||
),
|
||||
cancellable=False,
|
||||
scope={"operation": plan.operation},
|
||||
groups=MAINTENANCE_GROUPS,
|
||||
runner=lambda job: self._run_maintenance_plan(job, db, plan),
|
||||
runner=lambda job: self._run_maintenance_plan(
|
||||
job, db, plan, backup
|
||||
),
|
||||
)
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy) # the token stays usable
|
||||
# Same loop turn as the start: the runner has not run yet.
|
||||
# Same loop turn as the peek and the start: the runner has not run.
|
||||
store.consume(token, plan.operation, job_id=job.id)
|
||||
return web.json_response({"success": True, "job": job.summary()}, status=202)
|
||||
|
||||
def _peek_plan(self, token):
|
||||
"""``(plan, None)`` for a usable token, else ``(None, refusal)``."""
|
||||
try:
|
||||
return self._maintenance_plans.peek(token), None
|
||||
except PlanUsed as used:
|
||||
job = self._jobs.get(used.job_id)
|
||||
return None, _failure(
|
||||
"This plan was already applied.",
|
||||
409,
|
||||
"plan-used",
|
||||
job={"id": used.job_id, "title": job.title if job else None},
|
||||
)
|
||||
except PlanExpired:
|
||||
return None, _failure(
|
||||
"This review expired. Review the plan again.", 410, "plan-expired"
|
||||
)
|
||||
|
||||
def _backup_promised(self, plan):
|
||||
"""A plan reviewed with the policy on keeps that promise; so does on now."""
|
||||
"""A plan reviewed with the policy on keeps that promise; so does on now.
|
||||
|
||||
Decided once, when the apply starts (the disk check measured it), and
|
||||
handed to the job.
|
||||
"""
|
||||
return plan.payload["backup"] == "on" or self._safety_backup_policy() == "on"
|
||||
|
||||
async def _safety_backup_step(self, job, db, plan):
|
||||
async def _safety_backup_step(self, job, db, backup_on):
|
||||
"""``(ok, name)``: take the promised safety backup, or report none.
|
||||
|
||||
``name`` is the backup's file name, or None when the policy is off.
|
||||
A failed backup ends the job ``backup-failed`` (``ok`` False).
|
||||
``name`` is the backup's file name, or None when no backup was
|
||||
promised. A failed backup ends the job ``backup-failed`` (``ok``
|
||||
False).
|
||||
"""
|
||||
if not self._backup_promised(plan):
|
||||
if not backup_on:
|
||||
job.log_line("No safety backup (the policy is off)")
|
||||
return True, None
|
||||
job.update(status_line="Taking a safety backup")
|
||||
@@ -531,8 +641,8 @@ class MaintenanceRoutesMixin:
|
||||
if removed:
|
||||
job.log_line(f"Removed {len(removed)} older safety backups")
|
||||
|
||||
async def _run_maintenance_plan(self, job, db, plan):
|
||||
ok, backup = await self._safety_backup_step(job, db, plan)
|
||||
async def _run_maintenance_plan(self, job, db, plan, backup_on):
|
||||
ok, backup = await self._safety_backup_step(job, db, backup_on)
|
||||
if not ok:
|
||||
return
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user