Vito
d6371b7fad
* feat(images): move deleted image files to the macOS Trash (#499) Trash mode on macOS now calls -[NSFileManager trashItemAtURL:resultingItemURL:error:] through ctypes (no dependency), in utils/trash_macos.py. Before any move it refuses what it cannot show to be recoverable, already in the review dialog: network volumes, read-only volumes, iCloud Drive and File Provider folders (cloud-sync), a trash folder inside an output root (by file identity), and files macOS would refuse for permission (no-permission). A refused move keeps the file and logs the NSError; a move macOS reports done but whose place cannot be shown goes under errors as trash-unconfirmed, keeps its records and stops the job. The native proof runs in portability.yml's new native-trash job on macos-latest and macos-15-intel (PM_NATIVE_TRASH=macos). Windows still reports no-trash (#500). * test(trash): resolve the temp folder and wait for a busy image to be released (#499) The permission tests keyed their lstat fakes by the unresolved temp path, but the backend resolves paths (/var -> /private/var on macOS, 8.3 short names on Windows), so on those runners the fakes never matched (portability run 40954618922; reproduced on Linux with a symlinked TMPDIR). The native test's detach first proves this process holds nothing under the mount, then retries the plain detach until macOS releases a volume it has just written a trash folder to (macos-15-intel), and still fails after a forced detach. * test(trash): count a volume that unmounted after a busy detach as detached (#499) On macos-15-intel a detach that reported "Resource busy" finished afterwards, so the retry found no such volume (portability run 40957282805). The loop now checks os.path.ismount before each attempt.
522 lines
21 KiB
Python
522 lines
21 KiB
Python
"""Delete selected images: remove them from the library, or trash them (#480).
|
||
|
||
``POST /prompt_manager/images/delete/preview {items | filter}`` reviews the
|
||
selection for both modes at once and answers a reviewed plan (the System ›
|
||
Maintenance contract, ``py/api/maintenance.py``): the token, the counts of
|
||
the records each file loses, the refused files with a coded reason (both
|
||
modes, and the trash-only ones), the prompts that lose their last image and
|
||
the safety backup. It writes nothing. ``items`` are Images listing identities
|
||
(at most 500); ``filter`` is "Select all N matching". Each target is matched
|
||
to its entry in the current listing (``not-listed`` otherwise), so every
|
||
later step uses the walked spelling, never the request's text.
|
||
|
||
``POST /prompt_manager/maintenance/apply {token, mode}`` runs the plan as the
|
||
``image_delete`` job (admission ``records``, not cancellable): the sources
|
||
are checked, the promised safety backup is taken, then each file is checked
|
||
again (containment and fingerprint) and
|
||
|
||
- ``mode="library"``: its records are deleted and its removal mark written
|
||
in one transaction; the file stays on disk, hidden;
|
||
- ``mode="trash"``: it is moved to the trash first (``utils/trash.py``); a
|
||
refused move leaves the file and its records alone; after a move its
|
||
records are deleted.
|
||
|
||
No path here unlinks a media file. A failed records write stops the job; the
|
||
files after it are not attempted.
|
||
"""
|
||
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import stat as stat_module
|
||
from pathlib import Path
|
||
|
||
from .admin import DELETABLE_MEDIA_EXTENSIONS, _remove_thumbnail
|
||
from .duplicate_review import (
|
||
CHANGED_CONTENT,
|
||
GONE,
|
||
LINK,
|
||
OUTSIDE_SOURCE,
|
||
UNREADABLE,
|
||
file_problem,
|
||
)
|
||
from .images import _favorite_relative_key, _favorite_root_key
|
||
from .jobs import IMAGE_DELETE_GROUP_NAMES, IMAGE_DELETE_JOB_KIND, JobBusy
|
||
from .job_routes import busy_response
|
||
from .maintenance import MAINTENANCE_PREVIEW_LEASE, _fail_job, _failure, backup_note
|
||
|
||
try:
|
||
from ...database import image_removal_schema
|
||
from ...database.image_removal import mark_hides
|
||
from ...database.annotation_schema import MAX_LOCATOR_LENGTH
|
||
from ...utils import trash
|
||
from ...utils.media_folders import is_trash_folder
|
||
except ImportError:
|
||
from database import image_removal_schema
|
||
from database.image_removal import mark_hides
|
||
from database.annotation_schema import MAX_LOCATOR_LENGTH
|
||
from utils import trash
|
||
from utils.media_folders import is_trash_folder
|
||
|
||
IMAGE_DELETE = "delete_images"
|
||
MODES = ("library", "trash")
|
||
TITLES = {
|
||
"library": "Delete images (remove from the library)",
|
||
"trash": "Delete images (move to the trash)",
|
||
}
|
||
MAX_DETAIL_ROWS = 50
|
||
# The system moved the file but its place in a trash could not be shown (#499).
|
||
TRASH_UNCONFIRMED = "trash-unconfirmed"
|
||
RECORDS_FAILED_CODE = "records-failed"
|
||
|
||
# Reason codes (design §4.4) and the sentence each shows.
|
||
REASONS = {
|
||
"not-listed": "No longer listed in Images.",
|
||
"outside": "Outside the output folders.",
|
||
"link": "A link, not a file.",
|
||
"gone": "The file no longer exists.",
|
||
"unreadable": "The file can't be read.",
|
||
"thumbnail": "A generated preview, not an image.",
|
||
"not-media": "Not a media file.",
|
||
"trash-folder": "Inside a trash folder.",
|
||
"path-too-long": "The path is too long.",
|
||
"changed": "The file changed after the review.",
|
||
trash.NO_TRASH: "This drive has no trash.",
|
||
trash.NETWORK: "Network drives can't use the trash.",
|
||
trash.TRASH_IN_ROOT: "This drive's trash is inside an output folder.",
|
||
trash.LOCKED: "The file is open in another program. Close it and try again.",
|
||
trash.FAILED: "The file could not be moved to the trash.",
|
||
trash.NO_PERMISSION: (
|
||
"PromptManager isn't allowed to move this file (no permission, or it is "
|
||
"locked in Finder)."
|
||
),
|
||
trash.CLOUD_SYNC: (
|
||
"Files in iCloud Drive or a cloud-synced folder can't be moved to the "
|
||
"trash here."
|
||
),
|
||
TRASH_UNCONFIRMED: (
|
||
"Moved by macOS, but its place in the trash could not be confirmed."
|
||
),
|
||
trash.WINDOWS_RECYCLE: (
|
||
"Moving files to the Recycle Bin isn't available on Windows: files larger "
|
||
"than the bin's capacity would be deleted permanently."
|
||
),
|
||
"not-attempted": "Not attempted: the job stopped after an error.",
|
||
}
|
||
_PROBLEM_CODES = {
|
||
OUTSIDE_SOURCE: "outside",
|
||
LINK: "link",
|
||
GONE: "gone",
|
||
UNREADABLE: "unreadable",
|
||
CHANGED_CONTENT: "changed",
|
||
}
|
||
SOURCES_CHANGED = "The sources changed since the review; nothing was changed."
|
||
RECORDS_FAILED = "The image records could not be removed; the job stopped."
|
||
|
||
|
||
def lexical_problem(relative: str) -> str | None:
|
||
"""The refusal code a root-relative POSIX path earns before any file access."""
|
||
parts = relative.split("/")
|
||
if "thumbnails" in parts[:-1]:
|
||
return "thumbnail"
|
||
if any(is_trash_folder(part) for part in parts[:-1]):
|
||
return "trash-folder"
|
||
if os.path.splitext(parts[-1])[1].lower() not in DELETABLE_MEDIA_EXTENSIONS:
|
||
return "not-media"
|
||
return None
|
||
|
||
|
||
def file_check(path, root, relative, *, expected=None):
|
||
"""``(code, lstat)``: why *path* may not be deleted, or ``(None, lstat)``.
|
||
|
||
Text first (inside the root, not a preview or trash folder, a media
|
||
file), then ``duplicate_review.file_problem``: a regular file, not a
|
||
link, whose resolved path is still inside the resolved root. With
|
||
*expected* (``{size, mtime_ns, inode}`` from the review) the file must
|
||
still be those bytes. Every handle is closed when this returns.
|
||
"""
|
||
code = lexical_problem(relative)
|
||
if code:
|
||
return code, None
|
||
try:
|
||
info = os.lstat(path)
|
||
except FileNotFoundError:
|
||
return "gone", None
|
||
except OSError:
|
||
return "unreadable", None
|
||
if getattr(info, "st_file_attributes", 0) & getattr(
|
||
stat_module, "FILE_ATTRIBUTE_REPARSE_POINT", 0
|
||
):
|
||
return "link", info
|
||
if expected is None:
|
||
problem = file_problem(path, root, None, info.st_size, hash_check=False)
|
||
else:
|
||
problem = file_problem(
|
||
path,
|
||
root,
|
||
None,
|
||
expected["size"],
|
||
hash_check=False,
|
||
mtime_ns=expected["mtime_ns"],
|
||
inode=expected["inode"],
|
||
)
|
||
if problem:
|
||
return _PROBLEM_CODES.get(problem, "outside"), info
|
||
if len(os.path.normcase(os.path.realpath(path))) > MAX_LOCATOR_LENGTH:
|
||
return "path-too-long", info
|
||
return None, info
|
||
|
||
|
||
def _row(target, code):
|
||
return {
|
||
"path": target["rel"],
|
||
"root_label": target["root_label"],
|
||
"code": code,
|
||
"detail": REASONS[code],
|
||
}
|
||
|
||
|
||
def _summary(rows):
|
||
counts = {}
|
||
for row in rows:
|
||
counts[row["code"]] = counts.get(row["code"], 0) + 1
|
||
return {"count": len(rows), "counts": counts, "rows": rows[:MAX_DETAIL_ROWS]}
|
||
|
||
|
||
def _fingerprint(targets) -> str:
|
||
material = [
|
||
(t["root_index"], t["rel"], t["size"], t["mtime_ns"], t["inode"])
|
||
for t in targets
|
||
]
|
||
return hashlib.sha256(json.dumps(material).encode("utf-8")).hexdigest()
|
||
|
||
|
||
class ImageDeleteMixin:
|
||
"""The Images Delete… review and its apply job (#480)."""
|
||
|
||
def _register_image_delete_routes(self, routes):
|
||
# A fixed path beside the dynamic /images/{image_id} routes: aiohttp
|
||
# (3.10+) looks fixed paths up first (see image_categories.py).
|
||
@routes.post("/prompt_manager/images/delete/preview")
|
||
async def preview_image_delete_route(request):
|
||
return await self.preview_image_delete(request)
|
||
|
||
# ── review ────────────────────────────────────────────────────────
|
||
|
||
async def preview_image_delete(self, request):
|
||
"""Review the selection under the maintenance preview lease."""
|
||
try:
|
||
async with self._jobs.lease(MAINTENANCE_PREVIEW_LEASE):
|
||
return await self._preview_image_delete(request)
|
||
except JobBusy as busy:
|
||
return busy_response(busy)
|
||
|
||
async def _preview_image_delete(self, request):
|
||
_data, targets, roots, error = await self._category_request(
|
||
request, frozenset()
|
||
)
|
||
if error is not None:
|
||
return error
|
||
db = self.db
|
||
identity = self._database_identity(db)
|
||
try:
|
||
files = {}
|
||
for index in dict.fromkeys(t[0] for t in targets):
|
||
files[index] = await self._get_gallery_files(roots[index])
|
||
review = await self._run_in_executor(
|
||
self._review_image_delete, db, roots, targets, files
|
||
)
|
||
except Exception:
|
||
self.logger.exception("Image delete review failed")
|
||
return _failure("The review failed. Check server logs for details.", 500)
|
||
return await self._plan_response(
|
||
db,
|
||
identity,
|
||
IMAGE_DELETE,
|
||
_fingerprint(review["targets"]),
|
||
review["public"],
|
||
{"targets": review["targets"]},
|
||
)
|
||
|
||
def _review_image_delete(self, db, roots, targets, files):
|
||
"""Every target checked now (blocking); ``{targets, public}``."""
|
||
listings, seen = {}, set()
|
||
acted, refused, trash_refused = [], [], []
|
||
spellings = self._source_spellings()
|
||
root_texts = [str(root) for root in roots]
|
||
for index, _scope, relative in targets:
|
||
root = roots[index]
|
||
if index not in listings:
|
||
listings[index] = self._favorite_listing(root, files[index])
|
||
spelled = listings[index].get(_favorite_relative_key(relative))
|
||
target = {
|
||
"root_index": index,
|
||
"root_scope": _scope,
|
||
"root": str(root),
|
||
"root_label": self._public_path(root),
|
||
"rel": spelled or relative,
|
||
}
|
||
if spelled is None:
|
||
refused.append(_row(target, "not-listed"))
|
||
continue
|
||
absolute = os.path.join(str(root), *spelled.split("/"))
|
||
code, info = file_check(absolute, str(root), spelled)
|
||
if code:
|
||
refused.append(_row(target, code))
|
||
continue
|
||
identity = (info.st_dev, info.st_ino)
|
||
if identity in seen:
|
||
continue # one physical file named twice (nested roots, hardlinks)
|
||
seen.add(identity)
|
||
target.update(self._removal_of(absolute, info, spellings))
|
||
target["trash"] = trash.trash_available(absolute, root_texts)
|
||
if target["trash"]:
|
||
trash_refused.append(_row(target, target["trash"]))
|
||
acted.append(target)
|
||
counted = db.count_removal_records([t["removal"] for t in acted])
|
||
library_count = len(acted)
|
||
public = {
|
||
"operation": IMAGE_DELETE,
|
||
"scope": {"count": library_count, "label": "images"},
|
||
"library": {"count": library_count},
|
||
"trash": {"count": library_count - len(trash_refused)},
|
||
"protected": {"count": len(refused), "reasons": []},
|
||
"refused": _summary(refused),
|
||
"trash_refused": _summary(trash_refused),
|
||
"records": counted["records"],
|
||
"orphaned_prompts": counted["orphaned_prompts"],
|
||
"media": {"deletes_files": False, "label": "Chosen in the review"},
|
||
"details": [],
|
||
}
|
||
return {"targets": acted, "public": public}
|
||
|
||
@staticmethod
|
||
def _removal_of(absolute, info, spellings):
|
||
favorites = [
|
||
(_favorite_root_key(root), _favorite_relative_key(relative))
|
||
for root, relative in spellings.favorites(absolute).items()
|
||
]
|
||
return {
|
||
"abs": absolute,
|
||
"size": info.st_size,
|
||
"mtime_ns": info.st_mtime_ns,
|
||
"inode": [info.st_dev, info.st_ino],
|
||
"removal": {
|
||
"spellings": list(spellings.of(absolute).values()),
|
||
"locator": os.path.normcase(os.path.realpath(absolute)),
|
||
"favorites": favorites,
|
||
"category": spellings.category_key(absolute),
|
||
},
|
||
}
|
||
|
||
# ── apply ─────────────────────────────────────────────────────────
|
||
|
||
def _start_image_delete(self, db, plan, mode, backup_on):
|
||
"""Register the delete job for a reviewed plan (on the loop)."""
|
||
return self._jobs.start(
|
||
IMAGE_DELETE_JOB_KIND,
|
||
TITLES[mode],
|
||
admission="records",
|
||
cancellable=False,
|
||
scope={"operation": IMAGE_DELETE, "mode": mode},
|
||
groups=IMAGE_DELETE_GROUP_NAMES,
|
||
runner=lambda job: self._run_image_delete(job, db, plan, mode, backup_on),
|
||
)
|
||
|
||
async def _run_image_delete(self, job, db, plan, mode, backup_on):
|
||
targets = plan.payload["targets"]
|
||
unchanged = await self._run_in_executor(self._sources_unchanged, targets)
|
||
if not unchanged:
|
||
return _fail_job(job, "sources-changed", SOURCES_CHANGED)
|
||
ok, backup = await self._safety_backup_step(job, db, backup_on)
|
||
if not ok:
|
||
return
|
||
try:
|
||
await self._delete_images(job, db, targets, mode, backup)
|
||
finally:
|
||
if backup:
|
||
await self._prune_backup_pool(job, db)
|
||
|
||
async def _delete_images(self, job, db, targets, mode, backup):
|
||
job.update(total=len(targets), status_line=TITLES[mode])
|
||
roots = [str(root) for root in self._get_all_output_dirs()]
|
||
done_group = "trashed" if mode == "trash" else "removed"
|
||
changed, failed = 0, None
|
||
for done, target in enumerate(targets, start=1):
|
||
if failed:
|
||
self._record_delete(job, "not_changed", target, "not-attempted")
|
||
continue
|
||
group, code, label = await self._run_in_executor(
|
||
self._delete_one_image, db, target, mode, roots
|
||
)
|
||
if group == "errors":
|
||
failed = code or RECORDS_FAILED_CODE
|
||
text = REASONS[code] if code else RECORDS_FAILED
|
||
job.record(
|
||
"errors",
|
||
{
|
||
"path": target["rel"],
|
||
"root": target["root_label"],
|
||
"root_index": target["root_index"],
|
||
"root_scope": target["root_scope"],
|
||
"detail": text,
|
||
},
|
||
f"! {target['root_label']}/{target['rel']} · {text}",
|
||
)
|
||
elif group == done_group:
|
||
changed += 1
|
||
detail = (
|
||
f"Moved to the {label}" if label else "Removed from the library"
|
||
)
|
||
self._record_delete(job, group, target, None, detail)
|
||
else:
|
||
self._record_delete(job, group, target, code)
|
||
job.update(processed=done, progress=int(done / len(targets) * 100))
|
||
if changed:
|
||
self.invalidate_gallery_cache()
|
||
verb = "Moved" if mode == "trash" else "Removed"
|
||
where = "to the trash" if mode == "trash" else "from the library"
|
||
line = f"{verb} {changed:,} of {len(targets):,} images {where}"
|
||
job.log_line(line)
|
||
job.update(progress=100, status_line=f"{line} · {backup_note(backup)}")
|
||
if failed:
|
||
# "records-failed" names every stop at an errors row (the page's
|
||
# "Stopped after N of M" toast); the row says which error it was.
|
||
job.scope["failure"] = RECORDS_FAILED_CODE
|
||
job.publish({"type": "error", "code": failed, "message": line})
|
||
job.finish("failed")
|
||
|
||
@staticmethod
|
||
def _record_delete(job, group, target, code, detail=None):
|
||
text = detail or REASONS[code]
|
||
row = {
|
||
"path": target["rel"],
|
||
"root": target["root_label"],
|
||
"root_index": target["root_index"],
|
||
"root_scope": target["root_scope"],
|
||
"code": code,
|
||
"detail": text,
|
||
}
|
||
mark = {"removed": "-", "trashed": "-"}.get(group, "=")
|
||
job.record(
|
||
group, row, f"{mark} {target['root_label']}/{target['rel']} · {text}"
|
||
)
|
||
|
||
def _delete_one_image(self, db, target, mode, roots):
|
||
"""``(group, code, trash label)`` for one reviewed file (blocking)."""
|
||
code, _info = file_check(
|
||
target["abs"],
|
||
target["root"],
|
||
target["rel"],
|
||
expected={
|
||
"size": target["size"],
|
||
"mtime_ns": target["mtime_ns"],
|
||
"inode": target["inode"],
|
||
},
|
||
)
|
||
if code:
|
||
return "not_changed", code, None
|
||
label = None
|
||
if mode == "trash":
|
||
try:
|
||
label = trash.move_to_trash(target["abs"], roots)
|
||
except trash.TrashRefused as refused:
|
||
code = refused.code if refused.code in REASONS else trash.FAILED
|
||
self.logger.info(
|
||
"Not moved to the trash (%s): %s %s",
|
||
code,
|
||
target["rel"],
|
||
refused.detail,
|
||
)
|
||
return "not_changed", code, None
|
||
except trash.TrashUnconfirmed as unconfirmed:
|
||
# The file left its folder: keep its records, stop the job.
|
||
self.logger.warning(
|
||
"Trash move not confirmed: %s %s", target["rel"], unconfirmed.detail
|
||
)
|
||
return "errors", TRASH_UNCONFIRMED, None
|
||
mark = None
|
||
else:
|
||
mark = (target["size"], target["mtime_ns"])
|
||
try:
|
||
db.remove_image_records(target["removal"], mark=mark)
|
||
except Exception:
|
||
self.logger.exception("Removing an image's records failed")
|
||
return "errors", None, label
|
||
_remove_thumbnail(Path(target["root"]), Path(target["rel"]), self.logger)
|
||
return ("trashed" if mode == "trash" else "removed"), None, label
|
||
|
||
# ── the workspace listing ─────────────────────────────────────────
|
||
|
||
def _removal_marks(self) -> dict:
|
||
"""The removal marks, cached until a mark changes or the database does.
|
||
|
||
A listing page reads no table while nothing changed: the cache follows
|
||
this process's marks generation and the database identity (a restore).
|
||
"""
|
||
key = (self._database_identity(self.db), image_removal_schema.generation())
|
||
cached = getattr(self, "_marks_cache", None)
|
||
if cached is None or cached[0] != key:
|
||
cached = (key, self.db.removal_marks())
|
||
self._marks_cache = cached
|
||
return cached[1]
|
||
|
||
def _hidden_entries(self, listings) -> set:
|
||
"""Normalized paths of listed files a removal mark hides (blocking).
|
||
|
||
Only marked locators are looked at, and only those files are
|
||
``lstat``'ed: a mark hides a file while its size and ``mtime_ns`` are
|
||
still the ones it was removed with.
|
||
"""
|
||
marks = self._removal_marks()
|
||
if not marks:
|
||
return set()
|
||
located = self._workspace_locators(listings)
|
||
return {
|
||
entry.normalized
|
||
for locator, entry in ((locator, located.get(locator)) for locator in marks)
|
||
if entry is not None and mark_hides(marks, locator, entry.path)
|
||
}
|
||
|
||
def _removed_paths(self, paths) -> set:
|
||
"""The absolute paths among *paths* a removal mark hides (blocking)."""
|
||
marks = self._removal_marks()
|
||
if not marks:
|
||
return set()
|
||
return {
|
||
str(path)
|
||
for path in paths
|
||
if mark_hides(marks, os.path.normcase(os.path.realpath(str(path))), path)
|
||
}
|
||
|
||
def _removal_check_under(self, root):
|
||
"""``hidden(path) -> bool`` for files a walk of *root* found (blocking).
|
||
|
||
A walk never follows a link below *root*, so a file's locator is
|
||
``normcase(join(realpath(root), relative))``: no ``realpath`` per
|
||
file, and an ``lstat`` only for a marked one. Without marks it
|
||
answers False at once.
|
||
"""
|
||
marks = self._removal_marks()
|
||
if not marks:
|
||
return lambda _path: False
|
||
root = os.fspath(root)
|
||
real = os.path.realpath(root)
|
||
|
||
def hidden(path):
|
||
relative = os.path.relpath(os.fspath(path), root)
|
||
locator = os.path.normcase(os.path.join(real, relative))
|
||
return mark_hides(marks, locator, path)
|
||
|
||
return hidden
|
||
|
||
def _without_removed(self, root, paths) -> list:
|
||
"""*paths* found by a walk of *root*, less the hidden ones (blocking)."""
|
||
removed = self._removal_check_under(root)
|
||
return [path for path in paths if not removed(path)]
|
||
|
||
|
||
def parse_mode(value):
|
||
"""The apply's ``mode``, or None when it is not one of MODES."""
|
||
return value if isinstance(value, str) and value in MODES else None
|