Vito Sansevero
58ededde6d
Final review of #313: - C1: the duplicate find tells files apart by (st_dev, st_ino), so a source listed twice, a symlinked source inside another and a hardlink are one file. The preview and the job refuse a removal that os.path.samestat matches with its kept copy. - I1: each file's records move to its verified kept copy in one short transaction before that file is unlinked; a failed move deletes nothing and a failed unlink leaves the records on the identical kept copy. The status line counts files removed and not removed. - I2: links are looked up under every spelling of the source (the real path, the configured and monitored folders, ComfyUI's output). - I5: image tags join the kept copy as their union and the removed copy's pending, undecided Needs review items are settled; the plan says so. - I3: the VACUUM disk check measures SQLite's real temporary folder. - M1: the token is looked at again with no await before the job starts. - M2: the backup decision is made once and handed to the job. - M3: backups are linked into place; startup sweeps interrupted temps. - M4: Optimize runs alone (new "exclusive" admission) and its plan says the database is locked while VACUUM runs. - M5: the reviewed duplicate merge is strict: collisions and Library titles are protected, and errors roll back. The legacy cleanup is unchanged. - M7: 2,000 keep entries within a 1 MiB body. M8: event waits in tests. - M9: the preview binds files by size, mtime and inode without hashing; the job hashes right before each unlink.
776 lines
30 KiB
Python
776 lines
30 KiB
Python
"""Duplicate image files on the job registry (#249 System S6, Task 17).
|
|
|
|
- ``POST /prompt_manager/jobs/duplicates`` starts the find (202 ``{job}``):
|
|
kind ``duplicates``, "Find duplicate image files", admission ``read``,
|
|
cancellable, outcome group ``groups_found``. It hashes the media files of
|
|
every effective source (``_hash_media_groups``, shared with the legacy
|
|
scan) and the server keeps the groups it found under the job id: content
|
|
hash, root-relative paths and each file's source index. A find is refused
|
|
while a duplicate removal runs, and one find runs at a time.
|
|
- ``GET /prompt_manager/maintenance/duplicates/{job_id}?offset&limit`` pages
|
|
those groups, at most ``MAX_GROUP_PAGE`` at a time: ``{success, job:{id,
|
|
state}, total, offset, limit, groups:[{group_id, size, count, files}]}``.
|
|
Each file names its root-relative ``path``, its source's public label and
|
|
index and the Images serve URLs; never an absolute path.
|
|
- ``POST /prompt_manager/maintenance/preview {operation:"duplicate_files",
|
|
options:{job_id, keep:[{group_id, root_index, path}]}}`` reviews a removal:
|
|
in each group named, the files kept are the ones listed and every other
|
|
copy is removed. Each keeper and each removal is checked against the held
|
|
group, with containment decided as text before any filesystem access
|
|
(``file_problem``): inside its source, a media file outside
|
|
``thumbnails``, then a regular file (not a link) whose resolved path is
|
|
still inside the source, with the same size and content hash. A removal
|
|
that fails, or whose kept copy fails, is protected and listed with the
|
|
reason. The plan carries the files' database links, which move to the
|
|
kept copy.
|
|
- ``POST /prompt_manager/maintenance/apply {token}`` runs the reviewed plan
|
|
as the ``duplicate_cleanup`` job: admission ``records``, not cancellable,
|
|
groups ``removed``, ``kept`` and ``errors``. It checks the sources are
|
|
unchanged, takes the safety backup the policy promises (the database only;
|
|
removed files are not in it), then for each removal rechecks its kept copy
|
|
and the file itself (``file_problem``) and removes it with every handle
|
|
closed (``_unlink``), together with its thumbnail. A file that changed is
|
|
kept; the first unlink that fails stops the job, a locked file with the
|
|
coded ``LOCKED_FILE`` message (never the OS text), and the files after it
|
|
are not attempted. The database links of the files actually removed then
|
|
move to their kept copy (``PromptDatabase.move_file_links``).
|
|
|
|
The legacy gallery routes (``/scan_duplicates``, its stream and status, and
|
|
``/delete_duplicate_images``) keep their shapes on the same registry kinds;
|
|
see ``py/api/admin.py``.
|
|
"""
|
|
|
|
import asyncio
|
|
import os
|
|
from pathlib import Path
|
|
|
|
from aiohttp import web
|
|
|
|
from .admin import DUPLICATE_FIND_LEGACY_TITLE, _remove_thumbnail
|
|
from .duplicate_review import ( # noqa: F401 (CHANGED, OUTSIDE_SOURCE: public)
|
|
CHANGED,
|
|
DELETE_FAILED,
|
|
FIND_CANCELLED,
|
|
FIND_ERROR_EVENT,
|
|
FIND_FAILED,
|
|
GONE,
|
|
KEEPER,
|
|
KEEPER_CHANGED,
|
|
KEPT_COPY,
|
|
LOCKED_FILE,
|
|
NOT_ATTEMPTED,
|
|
OUTSIDE_SOURCE,
|
|
RECORDS_FAILED,
|
|
ROW_DETAILS,
|
|
SAME_FILE,
|
|
SOURCES_CHANGED,
|
|
SOURCES_CHANGED_REVIEW,
|
|
HeldFind,
|
|
SourcesChanged,
|
|
_fingerprint,
|
|
_group_order,
|
|
_identity,
|
|
_parse_options,
|
|
_plural,
|
|
_public_plan,
|
|
_select,
|
|
file_problem,
|
|
human_bytes,
|
|
)
|
|
from .images import _favorite_relative_key, _favorite_root_key, _output_image_entry
|
|
from .job_routes import busy_response
|
|
from .jobs import (
|
|
DUPLICATE_CLEANUP_GROUP_NAMES,
|
|
DUPLICATE_CLEANUP_JOB_KIND,
|
|
DUPLICATE_GROUP_NAMES,
|
|
DUPLICATES_JOB_KIND,
|
|
JobBusy,
|
|
)
|
|
from .maintenance import DUPLICATE_FILES, _fail_job, _failure, _invalid, backup_note
|
|
|
|
try:
|
|
from ...database.duplicate_copies import FILE_LINK_SQL
|
|
except ImportError:
|
|
from database.duplicate_copies import FILE_LINK_SQL
|
|
|
|
DUPLICATE_FIND_TITLE = "Find duplicate image files"
|
|
DUPLICATE_CLEANUP_TITLE = "Remove duplicate image files"
|
|
MAX_GROUP_PAGE = 10 # groups a page lists (D69: ten at a time)
|
|
MAX_HELD_FINDS = 3 # finds whose groups the server keeps for review
|
|
favorite_root_key = _favorite_root_key
|
|
|
|
|
|
def query_plan_sql():
|
|
"""The link reconciliation's statements (EXPLAIN QUERY PLAN checks)."""
|
|
return list(FILE_LINK_SQL)
|
|
|
|
|
|
def _unlink(path):
|
|
"""Remove one file (the caller closed every handle on it)."""
|
|
os.remove(path)
|
|
|
|
|
|
class DuplicateFilesMixin:
|
|
"""Mixin providing the duplicate-file find, its groups and its removal."""
|
|
|
|
def _register_duplicate_routes(self, routes):
|
|
@routes.post("/prompt_manager/jobs/duplicates")
|
|
async def start_duplicate_find_route(request):
|
|
return await self.start_duplicate_find(request)
|
|
|
|
@routes.get("/prompt_manager/maintenance/duplicates/{job_id}")
|
|
async def duplicate_groups_route(request):
|
|
return await self.duplicate_groups(request)
|
|
|
|
# ── find ──────────────────────────────────────────────────────────
|
|
|
|
async def start_duplicate_find(self, request):
|
|
"""Start the duplicate find (202), or 409 naming the job in the way."""
|
|
try:
|
|
job = self._start_duplicate_find(origin="workspace")
|
|
except JobBusy as busy:
|
|
return busy_response(busy)
|
|
return web.json_response({"success": True, "job": job.summary()}, status=202)
|
|
|
|
def _start_duplicate_find(self, origin):
|
|
"""Register a find job (on the loop); raises JobBusy.
|
|
|
|
A workspace find walks every effective source; a gallery one the
|
|
gallery's output folder, as the legacy scan did. Both resolve their
|
|
folders when the job begins, so a source change later cannot move it.
|
|
"""
|
|
cleanup = self._jobs.active(DUPLICATE_CLEANUP_JOB_KIND)
|
|
if cleanup is not None:
|
|
raise JobBusy(cleanup)
|
|
held = HeldFind(origin)
|
|
if origin == "workspace":
|
|
title = DUPLICATE_FIND_TITLE
|
|
else:
|
|
title = DUPLICATE_FIND_LEGACY_TITLE
|
|
job = self._jobs.start(
|
|
DUPLICATES_JOB_KIND,
|
|
title,
|
|
admission="read",
|
|
cancellable=True,
|
|
groups=DUPLICATE_GROUP_NAMES,
|
|
runner=lambda job: self._run_duplicate_find(job, held),
|
|
)
|
|
held.job_id = job.id
|
|
self._remember_find(held)
|
|
return job
|
|
|
|
def _remember_find(self, held):
|
|
finds = self._duplicate_finds
|
|
finds[held.job_id] = held
|
|
for job_id in list(finds):
|
|
if self._jobs.get(job_id) is None:
|
|
del finds[job_id]
|
|
while len(finds) > MAX_HELD_FINDS:
|
|
del finds[next(iter(finds))]
|
|
|
|
def _find_covers(self, job, legacy_dir):
|
|
held = self._duplicate_finds.get(job.id)
|
|
return held is not None and held.covers(legacy_dir)
|
|
|
|
def _latest_duplicate_find(self):
|
|
"""The running find, else the newest finished one this session."""
|
|
for job in self._jobs.list():
|
|
if job.kind == DUPLICATES_JOB_KIND and job.id in self._duplicate_finds:
|
|
return job
|
|
return None
|
|
|
|
def _resolve_find_roots(self, held):
|
|
"""Fill in where a find looks (blocking): sources and the gallery folder."""
|
|
listed = [os.path.abspath(str(d)) for d in self._get_all_output_dirs()]
|
|
held.legacy_dir = self._find_comfyui_output_dir()
|
|
if held.origin == "workspace":
|
|
held.roots = list(enumerate(listed))
|
|
return
|
|
if not held.legacy_dir or not os.path.isdir(held.legacy_dir):
|
|
self.logger.warning("ComfyUI output directory not found")
|
|
held.roots = []
|
|
return
|
|
legacy = os.path.abspath(held.legacy_dir)
|
|
index = next(
|
|
(i for i, d in enumerate(listed) if _identity(d) == _identity(legacy)),
|
|
None,
|
|
)
|
|
held.roots = [(index, legacy)]
|
|
|
|
def _find_progress(self, job, done, total):
|
|
if done < job.processed:
|
|
return # hashing threads report out of order; keep the count rising
|
|
pct = int(done / total * 100) if total else 100
|
|
job.update(
|
|
processed=done,
|
|
total=total,
|
|
status_line=f"Hashing file {done:,} of {total:,}",
|
|
)
|
|
job.publish(
|
|
{
|
|
"type": "progress",
|
|
"progress": pct,
|
|
"status": f"Hashing file {done}/{total}...",
|
|
"processed": done,
|
|
"found": 0,
|
|
}
|
|
)
|
|
|
|
async def _run_duplicate_find(self, job, held):
|
|
loop = asyncio.get_running_loop()
|
|
|
|
def report(done, total): # from the hashing threads
|
|
loop.call_soon_threadsafe(self._find_progress, job, done, total)
|
|
|
|
try:
|
|
job.update(status_line="Looking for media files")
|
|
job.publish(
|
|
{
|
|
"type": "progress",
|
|
"progress": 0,
|
|
"status": "Collecting media files...",
|
|
"processed": 0,
|
|
"found": 0,
|
|
}
|
|
)
|
|
await self._run_in_executor(self._resolve_find_roots, held)
|
|
job.scope["output_folders"] = len(held.roots)
|
|
job.log_line(f"Searching {_plural(len(held.roots), 'source')}")
|
|
groups = await self._run_in_executor(
|
|
self._hash_media_groups,
|
|
[root for _index, root in held.roots],
|
|
report,
|
|
lambda: job.cancelled,
|
|
)
|
|
if job.cancelled:
|
|
job.update(status_line="Cancelled; nothing was changed.")
|
|
job.publish({"type": "error", "message": FIND_CANCELLED})
|
|
job.finish("cancelled")
|
|
return
|
|
await self._finish_find(job, held, groups)
|
|
except Exception:
|
|
self.logger.exception("Duplicate search failed")
|
|
job.update(status_line=FIND_FAILED)
|
|
job.publish({"type": "error", "message": FIND_ERROR_EVENT})
|
|
job.finish("failed")
|
|
|
|
async def _finish_find(self, job, held, groups):
|
|
for group in groups:
|
|
group["group_id"] = group["hash"]
|
|
for record in group["files"]:
|
|
index, root = held.roots[record.pop("slot")]
|
|
record["root_index"] = index
|
|
record["root"] = root
|
|
groups.sort(key=_group_order)
|
|
held.groups = groups
|
|
labels = {root: self._public_path(root) for _index, root in held.roots}
|
|
rows = [
|
|
{
|
|
"path": group["files"][0]["rel"],
|
|
"root": labels[group["files"][0]["root"]],
|
|
"root_index": group["files"][0]["root_index"],
|
|
"detail": f"{len(group['files'])} identical files · "
|
|
f"{human_bytes(group['size'])} each",
|
|
}
|
|
for group in groups
|
|
]
|
|
job.record_counted("groups_found", len(groups), rows)
|
|
line = f"Found {_plural(len(groups), 'group')} of duplicate files"
|
|
job.log_line(line)
|
|
legacy = []
|
|
if held.covers(held.legacy_dir) and held.legacy_dir:
|
|
legacy = await self._run_in_executor(
|
|
self._legacy_duplicate_groups, groups, held.legacy_dir
|
|
)
|
|
# Every file was read by now (the last reports may still be queued).
|
|
job.update(progress=100, status_line=line, processed=job.total)
|
|
job.publish(
|
|
{
|
|
"type": "complete",
|
|
"processed": job.processed,
|
|
"found": len(legacy),
|
|
"duplicates": legacy,
|
|
"message": f"Found {len(legacy)} groups of duplicate images",
|
|
}
|
|
)
|
|
|
|
# ── groups ────────────────────────────────────────────────────────
|
|
|
|
def _held_result(self, job_id):
|
|
"""``(held, None)`` for a finished find's groups, else an error reply."""
|
|
held = self._duplicate_finds.get(job_id)
|
|
job = self._jobs.get(job_id)
|
|
if held is None or job is None or job.kind != DUPLICATES_JOB_KIND:
|
|
return None, _failure(
|
|
"No duplicate search with this id. Search again.", 404, "not-found"
|
|
)
|
|
if job.running:
|
|
return None, _failure(
|
|
"The duplicate search is still running.", 409, "job-running"
|
|
)
|
|
if held.groups is None:
|
|
return None, _failure(
|
|
"This search did not finish. Search again.", 409, "no-groups"
|
|
)
|
|
return held, None
|
|
|
|
async def duplicate_groups(self, request):
|
|
"""One page of a finished find's groups (at most MAX_GROUP_PAGE)."""
|
|
try:
|
|
offset = int(request.query.get("offset", "0"))
|
|
limit = int(request.query.get("limit", str(MAX_GROUP_PAGE)))
|
|
except ValueError:
|
|
return _invalid("offset and limit must be whole numbers")
|
|
if offset < 0 or not 0 < limit <= MAX_GROUP_PAGE:
|
|
return _invalid(f"limit must be 1 to {MAX_GROUP_PAGE}; offset 0 or more")
|
|
job_id = request.match_info["job_id"]
|
|
held, error = self._held_result(job_id)
|
|
if error is not None:
|
|
return error
|
|
page = held.groups[offset : offset + limit]
|
|
groups = await self._run_in_executor(self._describe_groups, page)
|
|
job = self._jobs.get(job_id)
|
|
return web.json_response(
|
|
{
|
|
"success": True,
|
|
"job": {"id": job.id, "state": job.state},
|
|
"total": len(held.groups),
|
|
"offset": offset,
|
|
"limit": limit,
|
|
"groups": groups,
|
|
},
|
|
headers={"Cache-Control": "no-store"},
|
|
)
|
|
|
|
def _describe_groups(self, groups):
|
|
"""Public group rows (blocking: one stat per file)."""
|
|
return [
|
|
{
|
|
"group_id": group["group_id"],
|
|
"size": group["size"],
|
|
"count": len(group["files"]),
|
|
"files": [self._describe_copy(record) for record in group["files"]],
|
|
}
|
|
for group in groups
|
|
]
|
|
|
|
def _describe_copy(self, record):
|
|
label = self._public_path(record["root"])
|
|
base = {
|
|
"filename": os.path.basename(record["rel"]),
|
|
"path": record["rel"],
|
|
"root": label,
|
|
"root_index": record["root_index"],
|
|
"url": None,
|
|
"thumbnail_url": None,
|
|
"size": record["size"],
|
|
"modified_time": record["mtime"],
|
|
"media_type": "image",
|
|
"missing": False,
|
|
}
|
|
if record["root_index"] is None:
|
|
return base
|
|
try:
|
|
entry = _output_image_entry(
|
|
Path(record["abs"]), Path(record["root"]), record["root_index"], label
|
|
)
|
|
except (OSError, ValueError):
|
|
return {**base, "missing": True}
|
|
return {
|
|
**base,
|
|
"url": entry["url"],
|
|
"thumbnail_url": entry["thumbnail_url"],
|
|
"size": entry["size"],
|
|
"modified_time": entry["modified_time"],
|
|
"media_type": entry["media_type"],
|
|
}
|
|
|
|
# ── review ────────────────────────────────────────────────────────
|
|
|
|
def _listed_sources(self):
|
|
"""``{identity: index}`` of the effective sources now (blocking)."""
|
|
return {
|
|
_identity(d): index for index, d in enumerate(self._get_all_output_dirs())
|
|
}
|
|
|
|
@staticmethod
|
|
def _source_listed(listed, index, root):
|
|
return index is not None and listed.get(_identity(root)) == index
|
|
|
|
async def preview_duplicate_files(self, options):
|
|
"""The ``duplicate_files`` plan for one finished find and its keepers."""
|
|
job_id, keep, message = _parse_options(options)
|
|
if message:
|
|
return _invalid(message)
|
|
held, error = self._held_result(job_id)
|
|
if error is not None:
|
|
return error
|
|
selection, message = _select(held, keep)
|
|
if message:
|
|
return _invalid(message)
|
|
db = self.db
|
|
identity = self._database_identity(db)
|
|
try:
|
|
review = await self._run_in_executor(
|
|
self._review_duplicate_files, db, selection
|
|
)
|
|
except SourcesChanged:
|
|
return _failure(SOURCES_CHANGED_REVIEW, 409, "sources-changed")
|
|
except Exception:
|
|
self.logger.exception("Duplicate file review failed")
|
|
return _failure("Failed to review maintenance", 500)
|
|
return await self._plan_response(
|
|
db,
|
|
identity,
|
|
DUPLICATE_FILES,
|
|
review["fingerprint"],
|
|
review["public"],
|
|
{
|
|
"targets": review["targets"],
|
|
"kept": review["kept"],
|
|
"protected": review["public"]["protected"]["count"],
|
|
},
|
|
)
|
|
|
|
def _review_duplicate_files(self, db, selection):
|
|
"""Check every keeper and removal now (blocking); the plan's parts.
|
|
|
|
No file is read: each copy is bound to what the search saw by its
|
|
size, mtime and inode, and never to its kept copy itself. The removal
|
|
job hashes each file again right before it unlinks it.
|
|
"""
|
|
listed = self._listed_sources()
|
|
targets, kept, protected = [], [], {}
|
|
for group, keepers in selection.values():
|
|
removals = [r for r in group["files"] if r not in keepers]
|
|
for record in removals:
|
|
if record["root_index"] is not None and not self._source_listed(
|
|
listed, record["root_index"], record["root"]
|
|
):
|
|
raise SourcesChanged(record["root"])
|
|
keeper = next(
|
|
(k for k in keepers if _bound_problem(k, group) is None), None
|
|
)
|
|
reviewed = []
|
|
for record in removals:
|
|
if keeper is None:
|
|
problem = KEEPER
|
|
elif record["root_index"] is None:
|
|
problem = OUTSIDE_SOURCE
|
|
else:
|
|
problem = _bound_problem(record, group, keeper["abs"])
|
|
if problem:
|
|
protected[problem] = protected.get(problem, 0) + 1
|
|
continue
|
|
reviewed.append(self._target(group, record, keeper))
|
|
if reviewed:
|
|
targets.extend(reviewed)
|
|
kept.append(self._copy_row(keeper, KEPT_COPY))
|
|
records = {}
|
|
if targets:
|
|
spellings = self._source_spellings()
|
|
records = db.count_copy_records(
|
|
{
|
|
t["abs"]: {
|
|
"spellings": _file_spellings(spellings, t["root"], t["rel"]),
|
|
"locator": t["locator"],
|
|
"sha256": t["hash"],
|
|
}
|
|
for t in targets
|
|
}
|
|
)
|
|
return {
|
|
"targets": targets,
|
|
"kept": kept,
|
|
"fingerprint": _fingerprint(targets),
|
|
"public": _public_plan(targets, protected, records),
|
|
}
|
|
|
|
def _source_spellings(self):
|
|
"""``{source identity: [spellings]}`` the monitor and config use (blocking).
|
|
|
|
The search walks each source's real path, while the image monitor
|
|
links a file under the folder it watches: the configured spelling,
|
|
ComfyUI's output folder or a symlink to it. Each absolute candidate
|
|
is listed under the identity (``normcase(realpath)``) it resolves to.
|
|
"""
|
|
from ..config import GalleryConfig
|
|
|
|
candidates = list(self._monitored_directories())
|
|
candidates += list(GalleryConfig.MONITORING_DIRECTORIES)
|
|
try:
|
|
import folder_paths
|
|
|
|
candidates.append(folder_paths.get_output_directory())
|
|
except Exception: # nosec B110 - outside ComfyUI there is none
|
|
pass
|
|
candidates.append(self._find_comfyui_output_dir())
|
|
spellings = {}
|
|
for candidate in candidates:
|
|
if not candidate or not os.path.isabs(str(candidate)):
|
|
continue
|
|
spelled = os.path.abspath(str(candidate))
|
|
listed = spellings.setdefault(_identity(spelled), [])
|
|
if spelled not in listed:
|
|
listed.append(spelled)
|
|
return spellings
|
|
|
|
def _target(self, group, record, keeper):
|
|
return {
|
|
"group_id": group["group_id"],
|
|
"hash": group["hash"],
|
|
"size": record["size"],
|
|
"root_index": record["root_index"],
|
|
"root": record["root"],
|
|
"root_label": self._public_path(record["root"]),
|
|
"abs": record["abs"],
|
|
"rel": record["rel"],
|
|
"keeper_abs": keeper["abs"],
|
|
"keeper_rel": keeper["rel"],
|
|
"keeper_root": keeper["root"],
|
|
"keeper_size": keeper["size"],
|
|
"locator": os.path.normcase(record["abs"]),
|
|
"keeper_locator": os.path.normcase(keeper["abs"]),
|
|
}
|
|
|
|
def _copy_row(self, record, detail):
|
|
return {
|
|
"path": record["rel"],
|
|
"root": self._public_path(record["root"]),
|
|
"root_index": record["root_index"],
|
|
"detail": detail,
|
|
}
|
|
|
|
# ── removal ───────────────────────────────────────────────────────
|
|
|
|
def _start_duplicate_cleanup(self, db, plan, backup_on):
|
|
"""Register the removal job for a reviewed plan (on the loop)."""
|
|
find = self._jobs.active(DUPLICATES_JOB_KIND)
|
|
if find is not None:
|
|
raise JobBusy(find)
|
|
return self._jobs.start(
|
|
DUPLICATE_CLEANUP_JOB_KIND,
|
|
DUPLICATE_CLEANUP_TITLE,
|
|
admission="records",
|
|
cancellable=False,
|
|
scope={"operation": DUPLICATE_FILES},
|
|
groups=DUPLICATE_CLEANUP_GROUP_NAMES,
|
|
runner=lambda job: self._run_duplicate_cleanup(job, db, plan, backup_on),
|
|
)
|
|
|
|
def _sources_unchanged(self, targets):
|
|
listed = self._listed_sources()
|
|
return all(
|
|
self._source_listed(listed, t["root_index"], t["root"]) for t in targets
|
|
)
|
|
|
|
async def _run_duplicate_cleanup(self, job, db, plan, backup_on):
|
|
targets = plan.payload["targets"]
|
|
unchanged = await self._run_in_executor(self._sources_unchanged, targets)
|
|
if not unchanged:
|
|
return _fail_job(job, "sources-changed", SOURCES_CHANGED)
|
|
ok, backup = await self._safety_backup_step(job, db, backup_on)
|
|
if not ok:
|
|
return
|
|
try:
|
|
await self._remove_duplicates(job, db, plan, backup)
|
|
finally:
|
|
if backup:
|
|
await self._prune_backup_pool(job, db)
|
|
|
|
async def _remove_duplicates(self, job, db, plan, backup):
|
|
targets = plan.payload["targets"]
|
|
for row in plan.payload["kept"]:
|
|
job.record("kept", row, f"= {row['root']}/{row['path']} · {KEPT_COPY}")
|
|
job.update(total=len(targets), status_line="Removing duplicate files")
|
|
spellings = await self._run_in_executor(self._source_spellings)
|
|
removed, failure, keepers = 0, None, {}
|
|
totals = dict.fromkeys(("moved", "merged", "favorites", "tags", "review"), 0)
|
|
for done, target in enumerate(targets, start=1):
|
|
if failure is not None:
|
|
self._record_target(job, "kept", target, NOT_ATTEMPTED)
|
|
continue
|
|
outcome, detail, counts = await self._run_in_executor(
|
|
self._remove_one_duplicate, db, target, keepers, spellings
|
|
)
|
|
for key, value in (counts or {}).items():
|
|
totals[key] += value
|
|
if outcome == "removed":
|
|
removed += 1
|
|
text = f"A copy of {target['keeper_rel']}"
|
|
self._record_target(job, "removed", target, text, "-")
|
|
elif outcome == "kept":
|
|
self._record_target(job, "kept", target, ROW_DETAILS[detail])
|
|
else:
|
|
failure = detail
|
|
self._record_target(job, "errors", target, detail, "!")
|
|
job.update(processed=done, progress=int(done / len(targets) * 100))
|
|
if removed:
|
|
self.invalidate_gallery_cache()
|
|
self._log_moved_records(job, totals)
|
|
line = f"Removed {_plural(removed, 'duplicate file')}"
|
|
if removed < len(targets):
|
|
line += f", {len(targets) - removed:,} not removed"
|
|
job.log_line(line)
|
|
if failure is not None:
|
|
line = f"Stopped after an error · {line}"
|
|
job.update(progress=100, status_line=f"{line} · {backup_note(backup)}")
|
|
if failure is not None:
|
|
job.publish({"type": "error", "code": "delete-failed", "message": line})
|
|
job.finish("failed")
|
|
|
|
@staticmethod
|
|
def _log_moved_records(job, totals):
|
|
links = totals["moved"] + totals["merged"]
|
|
if links:
|
|
job.log_line(f"Moved {_plural(links, 'database link')} to the kept copies")
|
|
if totals["favorites"]:
|
|
job.log_line(
|
|
f"Moved {_plural(totals['favorites'], 'favorite')} to the kept copies"
|
|
)
|
|
if totals["tags"]:
|
|
job.log_line(
|
|
f"Added {_plural(totals['tags'], 'image tag')} to the kept copies"
|
|
)
|
|
if totals["review"]:
|
|
job.log_line(
|
|
f"Settled {_plural(totals['review'], 'Needs review item')} "
|
|
"of removed copies"
|
|
)
|
|
|
|
@staticmethod
|
|
def _record_target(job, group, target, detail, mark="="):
|
|
row = {
|
|
"path": target["rel"],
|
|
"root": target["root_label"],
|
|
"root_index": target["root_index"],
|
|
"detail": detail,
|
|
}
|
|
job.record(
|
|
group, row, f"{mark} {target['root_label']}/{target['rel']} · {detail}"
|
|
)
|
|
|
|
def _remove_one_duplicate(self, db, target, keepers, spellings):
|
|
"""``(outcome, detail, counts)`` for one reviewed removal (blocking).
|
|
|
|
The kept copy (once per group; cached in ``keepers``) and the file
|
|
are checked again, hashes included, and the file must not be its kept
|
|
copy. Then the file's records move to the kept copy in one short
|
|
transaction, and only then is the file unlinked: a crash or a failed
|
|
unlink leaves its records on the identical kept copy, never on a
|
|
deleted file. A failed move leaves the file and its records alone.
|
|
"""
|
|
hasher = self._calculate_file_hash
|
|
group_id = target["group_id"]
|
|
if group_id not in keepers:
|
|
keepers[group_id] = _kept_copy_state(target, hasher)
|
|
if keepers[group_id] is None:
|
|
return "kept", KEEPER, None
|
|
problem = file_problem(
|
|
target["abs"],
|
|
target["root"],
|
|
target["hash"],
|
|
target["size"],
|
|
hasher,
|
|
keeper=target["keeper_abs"],
|
|
)
|
|
if problem:
|
|
return "kept", problem, None
|
|
try:
|
|
counts = db.move_copy_records(
|
|
self._copy_move(target, keepers[group_id], spellings)
|
|
)
|
|
except Exception:
|
|
self.logger.exception(f"Could not move the records of {target['rel']}")
|
|
return "error", RECORDS_FAILED, None
|
|
try:
|
|
_unlink(target["abs"])
|
|
except FileNotFoundError:
|
|
return "removed", None, counts
|
|
except PermissionError:
|
|
self.logger.warning(
|
|
f"Duplicate file is locked: {target['rel']}; "
|
|
"its records now name the kept copy"
|
|
)
|
|
return "error", LOCKED_FILE, counts
|
|
except OSError:
|
|
self.logger.warning(
|
|
f"Could not delete duplicate file {target['rel']}; "
|
|
"its records now name the kept copy",
|
|
exc_info=True,
|
|
)
|
|
return "error", DELETE_FAILED, counts
|
|
self.logger.info(f"Deleted duplicate file: {target['rel']}")
|
|
_remove_thumbnail(Path(target["root"]), Path(target["rel"]), self.logger)
|
|
return "removed", None, counts
|
|
|
|
@staticmethod
|
|
def _copy_move(target, kept_stat, spellings):
|
|
"""What ``move_copy_records`` moves for one removal."""
|
|
removed = _file_spellings(spellings, target["root"], target["rel"])
|
|
kept = _file_spellings(spellings, target["keeper_root"], target["keeper_rel"])
|
|
same_source = _identity(target["root"]) == _identity(target["keeper_root"])
|
|
native_rel = target["rel"].replace("/", os.sep)
|
|
native_kept = target["keeper_rel"].replace("/", os.sep)
|
|
targets = {}
|
|
for spelled in removed:
|
|
prefix = spelled[: len(spelled) - len(native_rel)]
|
|
targets[spelled] = (
|
|
prefix + native_kept if same_source else target["keeper_abs"]
|
|
)
|
|
return {
|
|
"removed": removed,
|
|
"kept": kept,
|
|
"targets": targets,
|
|
"kept_path": target["keeper_abs"],
|
|
"favorite": (
|
|
_favorite_root_key(target["root"]),
|
|
_favorite_relative_key(target["rel"]),
|
|
_favorite_root_key(target["keeper_root"]),
|
|
_favorite_relative_key(target["keeper_rel"]),
|
|
),
|
|
"removed_locator": target["locator"],
|
|
"kept_locator": target["keeper_locator"],
|
|
"sha256": target["hash"],
|
|
"kept_size": kept_stat.st_size,
|
|
"kept_mtime_ns": kept_stat.st_mtime_ns,
|
|
}
|
|
|
|
|
|
def _bound_problem(record, group, keeper=None):
|
|
"""``file_problem`` without reading the file: size, mtime and inode."""
|
|
return file_problem(
|
|
record["abs"],
|
|
record["root"],
|
|
group["hash"],
|
|
record["size"],
|
|
hash_check=False,
|
|
mtime_ns=record.get("mtime_ns"),
|
|
inode=record.get("inode"),
|
|
keeper=keeper,
|
|
)
|
|
|
|
|
|
def _kept_copy_state(target, hasher):
|
|
"""The kept copy's ``os.stat`` when it is still the reviewed bytes, else None."""
|
|
problem = file_problem(
|
|
target["keeper_abs"],
|
|
target["keeper_root"],
|
|
target["hash"],
|
|
target["keeper_size"],
|
|
hasher,
|
|
)
|
|
if problem:
|
|
return None
|
|
try:
|
|
return os.stat(target["keeper_abs"])
|
|
except OSError:
|
|
return None
|
|
|
|
|
|
def _file_spellings(spellings, root, rel):
|
|
"""Every path one file is linked by: its real path, then each spelling."""
|
|
native = rel.replace("/", os.sep)
|
|
roots = [root] + spellings.get(_identity(root), [])
|
|
return list(dict.fromkeys(os.path.join(spelled, native) for spelled in roots))
|