Vito Sansevero
ad392a99a7
POST /prompt_manager/images/delete/preview reviews both modes; the
maintenance apply takes {token, mode}. Library mode removes the records and
writes the removal mark; trash mode moves the file first and removes its
records only after a successful move. The workspace listing, the output scan,
the duplicate find and Tag output images skip marked files.
466 lines
18 KiB
Python
466 lines
18 KiB
Python
"""Delete selected images: remove them from the library, or trash them (#480).
|
||
|
||
``POST /prompt_manager/images/delete/preview {items | filter}`` reviews the
|
||
selection for both modes at once and answers a reviewed plan (the System ›
|
||
Maintenance contract, ``py/api/maintenance.py``): the token, the counts of
|
||
the records each file loses, the refused files with a coded reason (both
|
||
modes, and the trash-only ones), the prompts that lose their last image and
|
||
the safety backup. It writes nothing. ``items`` are Images listing identities
|
||
(at most 500); ``filter`` is "Select all N matching". Each target is matched
|
||
to its entry in the current listing (``not-listed`` otherwise), so every
|
||
later step uses the walked spelling, never the request's text.
|
||
|
||
``POST /prompt_manager/maintenance/apply {token, mode}`` runs the plan as the
|
||
``image_delete`` job (admission ``records``, not cancellable): the sources
|
||
are checked, the promised safety backup is taken, then each file is checked
|
||
again (containment and fingerprint) and
|
||
|
||
- ``mode="library"``: its records are deleted and its removal mark written
|
||
in one transaction; the file stays on disk, hidden;
|
||
- ``mode="trash"``: it is moved to the trash first (``utils/trash.py``); a
|
||
refused move leaves the file and its records alone; after a move its
|
||
records are deleted.
|
||
|
||
No path here unlinks a media file. A failed records write stops the job; the
|
||
files after it are not attempted.
|
||
"""
|
||
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import stat as stat_module
|
||
from pathlib import Path
|
||
|
||
from .admin import DELETABLE_MEDIA_EXTENSIONS, _remove_thumbnail
|
||
from .duplicate_review import (
|
||
CHANGED_CONTENT,
|
||
GONE,
|
||
LINK,
|
||
OUTSIDE_SOURCE,
|
||
UNREADABLE,
|
||
file_problem,
|
||
)
|
||
from .images import _favorite_relative_key, _favorite_root_key
|
||
from .jobs import IMAGE_DELETE_GROUP_NAMES, IMAGE_DELETE_JOB_KIND, JobBusy
|
||
from .job_routes import busy_response
|
||
from .maintenance import MAINTENANCE_PREVIEW_LEASE, _fail_job, _failure, backup_note
|
||
|
||
try:
|
||
from ...database.annotation_schema import MAX_LOCATOR_LENGTH
|
||
from ...utils import trash
|
||
except ImportError:
|
||
from database.annotation_schema import MAX_LOCATOR_LENGTH
|
||
from utils import trash
|
||
|
||
IMAGE_DELETE = "delete_images"
|
||
MODES = ("library", "trash")
|
||
TITLES = {
|
||
"library": "Delete images (remove from the library)",
|
||
"trash": "Delete images (move to the trash)",
|
||
}
|
||
MAX_DETAIL_ROWS = 50
|
||
TRASH_FOLDERS = frozenset({".trash", ".trashes", "$recycle.bin"})
|
||
|
||
# Reason codes (design §4.4) and the sentence each shows.
|
||
REASONS = {
|
||
"not-listed": "No longer listed in Images.",
|
||
"outside": "Outside the output folders.",
|
||
"link": "A link, not a file.",
|
||
"gone": "The file no longer exists.",
|
||
"unreadable": "The file can't be read.",
|
||
"thumbnail": "A generated preview, not an image.",
|
||
"not-media": "Not a media file.",
|
||
"trash-folder": "Inside a trash folder.",
|
||
"path-too-long": "The path is too long.",
|
||
"changed": "The file changed after the review.",
|
||
trash.NO_TRASH: "This drive has no trash.",
|
||
trash.NETWORK: "Network drives can't use the trash.",
|
||
trash.TRASH_IN_ROOT: "This drive's trash is inside an output folder.",
|
||
trash.LOCKED: "The file is open in another program. Close it and try again.",
|
||
trash.FAILED: "The file could not be moved to the trash.",
|
||
"not-attempted": "Not attempted: the job stopped after an error.",
|
||
}
|
||
_PROBLEM_CODES = {
|
||
OUTSIDE_SOURCE: "outside",
|
||
LINK: "link",
|
||
GONE: "gone",
|
||
UNREADABLE: "unreadable",
|
||
CHANGED_CONTENT: "changed",
|
||
}
|
||
SOURCES_CHANGED = "The sources changed since the review; nothing was changed."
|
||
RECORDS_FAILED = "The image records could not be removed; the job stopped."
|
||
|
||
|
||
def _is_trash_folder(name: str) -> bool:
|
||
folded = name.casefold()
|
||
return folded in TRASH_FOLDERS or (
|
||
folded.startswith(".trash-") and folded[7:].isdigit()
|
||
)
|
||
|
||
|
||
def lexical_problem(relative: str) -> str | None:
|
||
"""The refusal code a root-relative POSIX path earns before any file access."""
|
||
parts = relative.split("/")
|
||
if "thumbnails" in parts[:-1]:
|
||
return "thumbnail"
|
||
if any(_is_trash_folder(part) for part in parts[:-1]):
|
||
return "trash-folder"
|
||
if os.path.splitext(parts[-1])[1].lower() not in DELETABLE_MEDIA_EXTENSIONS:
|
||
return "not-media"
|
||
return None
|
||
|
||
|
||
def file_check(path, root, relative, *, expected=None):
|
||
"""``(code, lstat)``: why *path* may not be deleted, or ``(None, lstat)``.
|
||
|
||
Text first (inside the root, not a preview or trash folder, a media
|
||
file), then ``duplicate_review.file_problem``: a regular file, not a
|
||
link, whose resolved path is still inside the resolved root. With
|
||
*expected* (``{size, mtime_ns, inode}`` from the review) the file must
|
||
still be those bytes. Every handle is closed when this returns.
|
||
"""
|
||
code = lexical_problem(relative)
|
||
if code:
|
||
return code, None
|
||
try:
|
||
info = os.lstat(path)
|
||
except FileNotFoundError:
|
||
return "gone", None
|
||
except OSError:
|
||
return "unreadable", None
|
||
if getattr(info, "st_file_attributes", 0) & getattr(
|
||
stat_module, "FILE_ATTRIBUTE_REPARSE_POINT", 0
|
||
):
|
||
return "link", info
|
||
if expected is None:
|
||
problem = file_problem(path, root, None, info.st_size, hash_check=False)
|
||
else:
|
||
problem = file_problem(
|
||
path,
|
||
root,
|
||
None,
|
||
expected["size"],
|
||
hash_check=False,
|
||
mtime_ns=expected["mtime_ns"],
|
||
inode=expected["inode"],
|
||
)
|
||
if problem:
|
||
return _PROBLEM_CODES.get(problem, "outside"), info
|
||
if len(os.path.normcase(os.path.realpath(path))) > MAX_LOCATOR_LENGTH:
|
||
return "path-too-long", info
|
||
return None, info
|
||
|
||
|
||
def _row(target, code):
|
||
return {
|
||
"path": target["rel"],
|
||
"root_label": target["root_label"],
|
||
"code": code,
|
||
"detail": REASONS[code],
|
||
}
|
||
|
||
|
||
def _summary(rows):
|
||
counts = {}
|
||
for row in rows:
|
||
counts[row["code"]] = counts.get(row["code"], 0) + 1
|
||
return {"count": len(rows), "counts": counts, "rows": rows[:MAX_DETAIL_ROWS]}
|
||
|
||
|
||
def _fingerprint(targets) -> str:
|
||
material = [
|
||
(t["root_index"], t["rel"], t["size"], t["mtime_ns"], t["inode"])
|
||
for t in targets
|
||
]
|
||
return hashlib.sha256(json.dumps(material).encode("utf-8")).hexdigest()
|
||
|
||
|
||
class ImageDeleteMixin:
|
||
"""The Images Delete… review and its apply job (#480)."""
|
||
|
||
def _register_image_delete_routes(self, routes):
|
||
# A fixed path beside the dynamic /images/{image_id} routes: aiohttp
|
||
# (3.10+) looks fixed paths up first (see image_categories.py).
|
||
@routes.post("/prompt_manager/images/delete/preview")
|
||
async def preview_image_delete_route(request):
|
||
return await self.preview_image_delete(request)
|
||
|
||
# ── review ────────────────────────────────────────────────────────
|
||
|
||
async def preview_image_delete(self, request):
|
||
"""Review the selection under the maintenance preview lease."""
|
||
try:
|
||
async with self._jobs.lease(MAINTENANCE_PREVIEW_LEASE):
|
||
return await self._preview_image_delete(request)
|
||
except JobBusy as busy:
|
||
return busy_response(busy)
|
||
|
||
async def _preview_image_delete(self, request):
|
||
_data, targets, roots, error = await self._category_request(
|
||
request, frozenset()
|
||
)
|
||
if error is not None:
|
||
return error
|
||
db = self.db
|
||
identity = self._database_identity(db)
|
||
try:
|
||
files = {}
|
||
for index in dict.fromkeys(t[0] for t in targets):
|
||
files[index] = await self._get_gallery_files(roots[index])
|
||
review = await self._run_in_executor(
|
||
self._review_image_delete, db, roots, targets, files
|
||
)
|
||
except Exception:
|
||
self.logger.exception("Image delete review failed")
|
||
return _failure("The review failed. Check server logs for details.", 500)
|
||
return await self._plan_response(
|
||
db,
|
||
identity,
|
||
IMAGE_DELETE,
|
||
_fingerprint(review["targets"]),
|
||
review["public"],
|
||
{"targets": review["targets"]},
|
||
)
|
||
|
||
def _review_image_delete(self, db, roots, targets, files):
|
||
"""Every target checked now (blocking); ``{targets, public}``."""
|
||
listings, seen = {}, set()
|
||
acted, refused, trash_refused = [], [], []
|
||
spellings = self._source_spellings()
|
||
root_texts = [str(root) for root in roots]
|
||
for index, _scope, relative in targets:
|
||
root = roots[index]
|
||
if index not in listings:
|
||
listings[index] = self._favorite_listing(root, files[index])
|
||
spelled = listings[index].get(_favorite_relative_key(relative))
|
||
target = {
|
||
"root_index": index,
|
||
"root_scope": _scope,
|
||
"root": str(root),
|
||
"root_label": self._public_path(root),
|
||
"rel": spelled or relative,
|
||
}
|
||
if spelled is None:
|
||
refused.append(_row(target, "not-listed"))
|
||
continue
|
||
absolute = os.path.join(str(root), *spelled.split("/"))
|
||
code, info = file_check(absolute, str(root), spelled)
|
||
if code:
|
||
refused.append(_row(target, code))
|
||
continue
|
||
identity = (info.st_dev, info.st_ino)
|
||
if identity in seen:
|
||
continue # one physical file named twice (nested roots, hardlinks)
|
||
seen.add(identity)
|
||
target.update(self._removal_of(absolute, info, spellings))
|
||
target["trash"] = trash.trash_available(absolute, root_texts)
|
||
if target["trash"]:
|
||
trash_refused.append(_row(target, target["trash"]))
|
||
acted.append(target)
|
||
counted = db.count_removal_records([t["removal"] for t in acted])
|
||
library_count = len(acted)
|
||
public = {
|
||
"operation": IMAGE_DELETE,
|
||
"scope": {"count": library_count, "label": "images"},
|
||
"library": {"count": library_count},
|
||
"trash": {"count": library_count - len(trash_refused)},
|
||
"protected": {"count": len(refused), "reasons": []},
|
||
"refused": _summary(refused),
|
||
"trash_refused": _summary(trash_refused),
|
||
"records": counted["records"],
|
||
"orphaned_prompts": counted["orphaned_prompts"],
|
||
"media": {"deletes_files": False, "label": "Chosen in the review"},
|
||
"details": [],
|
||
}
|
||
return {"targets": acted, "public": public}
|
||
|
||
@staticmethod
|
||
def _removal_of(absolute, info, spellings):
|
||
favorites = [
|
||
(_favorite_root_key(root), _favorite_relative_key(relative))
|
||
for root, relative in spellings.favorites(absolute).items()
|
||
]
|
||
return {
|
||
"abs": absolute,
|
||
"size": info.st_size,
|
||
"mtime_ns": info.st_mtime_ns,
|
||
"inode": [info.st_dev, info.st_ino],
|
||
"removal": {
|
||
"spellings": list(spellings.of(absolute).values()),
|
||
"locator": os.path.normcase(os.path.realpath(absolute)),
|
||
"favorites": favorites,
|
||
"category": spellings.category_key(absolute),
|
||
},
|
||
}
|
||
|
||
# ── apply ─────────────────────────────────────────────────────────
|
||
|
||
def _start_image_delete(self, db, plan, mode, backup_on):
|
||
"""Register the delete job for a reviewed plan (on the loop)."""
|
||
return self._jobs.start(
|
||
IMAGE_DELETE_JOB_KIND,
|
||
TITLES[mode],
|
||
admission="records",
|
||
cancellable=False,
|
||
scope={"operation": IMAGE_DELETE, "mode": mode},
|
||
groups=IMAGE_DELETE_GROUP_NAMES,
|
||
runner=lambda job: self._run_image_delete(job, db, plan, mode, backup_on),
|
||
)
|
||
|
||
async def _run_image_delete(self, job, db, plan, mode, backup_on):
|
||
targets = plan.payload["targets"]
|
||
unchanged = await self._run_in_executor(self._sources_unchanged, targets)
|
||
if not unchanged:
|
||
return _fail_job(job, "sources-changed", SOURCES_CHANGED)
|
||
ok, backup = await self._safety_backup_step(job, db, backup_on)
|
||
if not ok:
|
||
return
|
||
try:
|
||
await self._delete_images(job, db, targets, mode, backup)
|
||
finally:
|
||
if backup:
|
||
await self._prune_backup_pool(job, db)
|
||
|
||
async def _delete_images(self, job, db, targets, mode, backup):
|
||
job.update(total=len(targets), status_line=TITLES[mode])
|
||
roots = [str(root) for root in self._get_all_output_dirs()]
|
||
done_group = "trashed" if mode == "trash" else "removed"
|
||
changed, failed = 0, False
|
||
for done, target in enumerate(targets, start=1):
|
||
if failed:
|
||
self._record_delete(job, "not_changed", target, "not-attempted")
|
||
continue
|
||
group, code, label = await self._run_in_executor(
|
||
self._delete_one_image, db, target, mode, roots
|
||
)
|
||
if group == "errors":
|
||
failed = True
|
||
job.record(
|
||
"errors",
|
||
{
|
||
"path": target["rel"],
|
||
"root": target["root_label"],
|
||
"root_index": target["root_index"],
|
||
"root_scope": target["root_scope"],
|
||
"detail": RECORDS_FAILED,
|
||
},
|
||
f"! {target['root_label']}/{target['rel']} · {RECORDS_FAILED}",
|
||
)
|
||
elif group == done_group:
|
||
changed += 1
|
||
detail = (
|
||
f"Moved to the {label}" if label else "Removed from the library"
|
||
)
|
||
self._record_delete(job, group, target, None, detail)
|
||
else:
|
||
self._record_delete(job, group, target, code)
|
||
job.update(processed=done, progress=int(done / len(targets) * 100))
|
||
if changed:
|
||
self.invalidate_gallery_cache()
|
||
verb = "Moved" if mode == "trash" else "Removed"
|
||
where = "to the trash" if mode == "trash" else "from the library"
|
||
line = f"{verb} {changed:,} of {len(targets):,} images {where}"
|
||
job.log_line(line)
|
||
job.update(progress=100, status_line=f"{line} · {backup_note(backup)}")
|
||
if failed:
|
||
job.scope["failure"] = "records-failed"
|
||
job.publish({"type": "error", "code": "records-failed", "message": line})
|
||
job.finish("failed")
|
||
|
||
@staticmethod
|
||
def _record_delete(job, group, target, code, detail=None):
|
||
text = detail or REASONS[code]
|
||
row = {
|
||
"path": target["rel"],
|
||
"root": target["root_label"],
|
||
"root_index": target["root_index"],
|
||
"root_scope": target["root_scope"],
|
||
"code": code,
|
||
"detail": text,
|
||
}
|
||
mark = {"removed": "-", "trashed": "-"}.get(group, "=")
|
||
job.record(
|
||
group, row, f"{mark} {target['root_label']}/{target['rel']} · {text}"
|
||
)
|
||
|
||
def _delete_one_image(self, db, target, mode, roots):
|
||
"""``(group, code, trash label)`` for one reviewed file (blocking)."""
|
||
code, _info = file_check(
|
||
target["abs"],
|
||
target["root"],
|
||
target["rel"],
|
||
expected={
|
||
"size": target["size"],
|
||
"mtime_ns": target["mtime_ns"],
|
||
"inode": target["inode"],
|
||
},
|
||
)
|
||
if code:
|
||
return "not_changed", code, None
|
||
label = None
|
||
if mode == "trash":
|
||
try:
|
||
label = trash.move_to_trash(target["abs"], roots)
|
||
except trash.TrashRefused as refused:
|
||
code = refused.code if refused.code in REASONS else trash.FAILED
|
||
return "not_changed", code, None
|
||
mark = None
|
||
else:
|
||
mark = (target["size"], target["mtime_ns"])
|
||
try:
|
||
db.remove_image_records(target["removal"], mark=mark)
|
||
except Exception:
|
||
self.logger.exception("Removing an image's records failed")
|
||
return "errors", None, label
|
||
_remove_thumbnail(Path(target["root"]), Path(target["rel"]), self.logger)
|
||
return ("trashed" if mode == "trash" else "removed"), None, label
|
||
|
||
# ── the workspace listing ─────────────────────────────────────────
|
||
|
||
def _hidden_entries(self, listings) -> set:
|
||
"""Normalized paths of listed files a removal mark hides (blocking).
|
||
|
||
Only marked locators are looked at, and only those files are
|
||
``lstat``'ed: a mark hides a file while its size and ``mtime_ns`` are
|
||
still the ones it was removed with.
|
||
"""
|
||
marks = self.db.removal_marks()
|
||
if not marks:
|
||
return set()
|
||
located = self._workspace_locators(listings)
|
||
hidden = set()
|
||
for locator, (size, mtime_ns) in marks.items():
|
||
entry = located.get(locator)
|
||
if entry is None:
|
||
continue
|
||
try:
|
||
info = os.lstat(entry.path)
|
||
except OSError:
|
||
continue
|
||
if (info.st_size, info.st_mtime_ns) == (size, mtime_ns):
|
||
hidden.add(entry.normalized)
|
||
return hidden
|
||
|
||
def _removed_paths(self, paths) -> set:
|
||
"""The absolute paths among *paths* a removal mark hides (blocking)."""
|
||
marks = self.db.removal_marks()
|
||
if not marks:
|
||
return set()
|
||
hidden = set()
|
||
for path in paths:
|
||
mark = marks.get(os.path.normcase(os.path.realpath(str(path))))
|
||
if mark is None:
|
||
continue
|
||
try:
|
||
info = os.lstat(path)
|
||
except OSError:
|
||
continue
|
||
if (info.st_size, info.st_mtime_ns) == tuple(mark):
|
||
hidden.add(str(path))
|
||
return hidden
|
||
|
||
|
||
def parse_mode(value):
|
||
"""The apply's ``mode``, or None when it is not one of MODES."""
|
||
return value if isinstance(value, str) and value in MODES else None
|