Vito
d88cf9e7c6
* fix(quality): preserve cursor iteration and option validation Keep prefetched rows on ManagedCursor so partial loops and explicit fetches share one position, and stop existing iterators after close or retirement. Preserve chunked reads and native SQLite result lifetime on reexecution and errors. Catch OptionError before ValueError so legacy auto-tag requests receive the actual validation error. Remove the overwritten ViewerJS title option. Regression tests failed before the fixes for lost rows, reads after close/retirement, and misleading option errors. Focused Python checks pass; cursor checks also pass on Python 3.13 and 3.14. Refs #321. * fix(quality): make the test doubles legal raises and hashes The fake queue no longer raises an attribute that starts as None. The oversized kind records hashes instead of raising from __hash__, which also catches a hash that production code swallows. Hostile.__bool__ raises the TypeError a truth test may raise. * fix(quality): drop the extra saveCoverage argument * refactor(quality): remove assignments that are overwritten before use * refactor(quality): remove the remaining overwritten test assignments * refactor(quality): drop imports repeated inside functions * refactor(quality): give duplicate identity keys one shape * refactor(quality): name the unused job event loop variable * refactor(quality): release model gates in finally, not except BaseException * refactor(quality): end the job runners with an explicit return * refactor(quality): use the precise unittest asserts * refactor(quality): import unittest.mock one way in the tests * fix(quality): say why each empty except is safe, log the broad ones * refactor(quality): stop binding values the tests never read * refactor(quality): remove the unused preview aliases and tag page query * refactor(quality): remove re-exports nothing imports * refactor(quality): parenthesize deliberate string joins in lists * refactor(quality): remove unused JavaScript variables and imports * refactor(quality): use the precise assert in the model status test
523 lines
21 KiB
Python
523 lines
21 KiB
Python
"""Delete selected images: remove them from the library, or trash them (#480).
|
||
|
||
``POST /prompt_manager/images/delete/preview {items | filter}`` reviews the
|
||
selection for both modes at once and answers a reviewed plan (the System ›
|
||
Maintenance contract, ``py/api/maintenance.py``): the token, the counts of
|
||
the records each file loses, the refused files with a coded reason (both
|
||
modes, and the trash-only ones), the prompts that lose their last image and
|
||
the safety backup. It writes nothing. ``items`` are Images listing identities
|
||
(at most 500); ``filter`` is "Select all N matching". Each target is matched
|
||
to its entry in the current listing (``not-listed`` otherwise), so every
|
||
later step uses the walked spelling, never the request's text.
|
||
|
||
``POST /prompt_manager/maintenance/apply {token, mode}`` runs the plan as the
|
||
``image_delete`` job (admission ``records``, not cancellable): the sources
|
||
are checked, the promised safety backup is taken, then each file is checked
|
||
again (containment and fingerprint) and
|
||
|
||
- ``mode="library"``: its records are deleted and its removal mark written
|
||
in one transaction; the file stays on disk, hidden;
|
||
- ``mode="trash"``: it is moved to the trash first (``utils/trash.py``); a
|
||
refused move leaves the file and its records alone; after a move its
|
||
records are deleted.
|
||
|
||
No path here unlinks a media file. A failed records write stops the job; the
|
||
files after it are not attempted.
|
||
"""
|
||
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import stat as stat_module
|
||
from pathlib import Path
|
||
|
||
from .admin import DELETABLE_MEDIA_EXTENSIONS, _remove_thumbnail
|
||
from .duplicate_review import (
|
||
CHANGED_CONTENT,
|
||
GONE,
|
||
LINK,
|
||
OUTSIDE_SOURCE,
|
||
UNREADABLE,
|
||
file_problem,
|
||
)
|
||
from .images import _favorite_relative_key, _favorite_root_key
|
||
from .jobs import IMAGE_DELETE_GROUP_NAMES, IMAGE_DELETE_JOB_KIND, JobBusy
|
||
from .job_routes import busy_response
|
||
from .maintenance import MAINTENANCE_PREVIEW_LEASE, _fail_job, _failure, backup_note
|
||
|
||
try:
|
||
from ...database import image_removal_schema
|
||
from ...database.image_removal import mark_hides
|
||
from ...database.annotation_schema import MAX_LOCATOR_LENGTH
|
||
from ...utils import trash
|
||
from ...utils.media_folders import is_trash_folder
|
||
except ImportError:
|
||
from database import image_removal_schema
|
||
from database.image_removal import mark_hides
|
||
from database.annotation_schema import MAX_LOCATOR_LENGTH
|
||
from utils import trash
|
||
from utils.media_folders import is_trash_folder
|
||
|
||
IMAGE_DELETE = "delete_images"
|
||
MODES = ("library", "trash")
|
||
TITLES = {
|
||
"library": "Delete images (remove from the library)",
|
||
"trash": "Delete images (move to the trash)",
|
||
}
|
||
MAX_DETAIL_ROWS = 50
|
||
# The system moved the file but its place in a trash could not be shown (#499).
|
||
TRASH_UNCONFIRMED = "trash-unconfirmed"
|
||
RECORDS_FAILED_CODE = "records-failed"
|
||
|
||
# Reason codes (design §4.4) and the sentence each shows.
|
||
REASONS = {
|
||
"not-listed": "No longer listed in Images.",
|
||
"outside": "Outside the output folders.",
|
||
"link": "A link, not a file.",
|
||
"gone": "The file no longer exists.",
|
||
"unreadable": "The file can't be read.",
|
||
"thumbnail": "A generated preview, not an image.",
|
||
"not-media": "Not a media file.",
|
||
"trash-folder": "Inside a trash folder.",
|
||
"path-too-long": "The path is too long.",
|
||
"changed": "The file changed after the review.",
|
||
trash.NO_TRASH: "This drive has no trash.",
|
||
trash.NETWORK: "Network drives can't use the trash.",
|
||
trash.TRASH_IN_ROOT: "This drive's trash is inside an output folder.",
|
||
trash.LOCKED: "The file is open in another program. Close it and try again.",
|
||
trash.FAILED: "The file could not be moved to the trash.",
|
||
trash.NO_PERMISSION: (
|
||
"PromptManager isn't allowed to move this file (no permission, or it is "
|
||
"locked in Finder)."
|
||
),
|
||
trash.CLOUD_SYNC: (
|
||
"Files in iCloud Drive or a cloud-synced folder can't be moved to the "
|
||
"trash here."
|
||
),
|
||
TRASH_UNCONFIRMED: (
|
||
"Moved by macOS, but its place in the trash could not be confirmed."
|
||
),
|
||
trash.WINDOWS_RECYCLE: (
|
||
"Moving files to the Recycle Bin isn't available on Windows: files larger "
|
||
"than the bin's capacity would be deleted permanently."
|
||
),
|
||
"not-attempted": "Not attempted: the job stopped after an error.",
|
||
}
|
||
_PROBLEM_CODES = {
|
||
OUTSIDE_SOURCE: "outside",
|
||
LINK: "link",
|
||
GONE: "gone",
|
||
UNREADABLE: "unreadable",
|
||
CHANGED_CONTENT: "changed",
|
||
}
|
||
SOURCES_CHANGED = "The sources changed since the review; nothing was changed."
|
||
RECORDS_FAILED = "The image records could not be removed; the job stopped."
|
||
|
||
|
||
def lexical_problem(relative: str) -> str | None:
|
||
"""The refusal code a root-relative POSIX path earns before any file access."""
|
||
parts = relative.split("/")
|
||
if "thumbnails" in parts[:-1]:
|
||
return "thumbnail"
|
||
if any(is_trash_folder(part) for part in parts[:-1]):
|
||
return "trash-folder"
|
||
if os.path.splitext(parts[-1])[1].lower() not in DELETABLE_MEDIA_EXTENSIONS:
|
||
return "not-media"
|
||
return None
|
||
|
||
|
||
def file_check(path, root, relative, *, expected=None):
|
||
"""``(code, lstat)``: why *path* may not be deleted, or ``(None, lstat)``.
|
||
|
||
Text first (inside the root, not a preview or trash folder, a media
|
||
file), then ``duplicate_review.file_problem``: a regular file, not a
|
||
link, whose resolved path is still inside the resolved root. With
|
||
*expected* (``{size, mtime_ns, inode}`` from the review) the file must
|
||
still be those bytes. Every handle is closed when this returns.
|
||
"""
|
||
code = lexical_problem(relative)
|
||
if code:
|
||
return code, None
|
||
try:
|
||
info = os.lstat(path)
|
||
except FileNotFoundError:
|
||
return "gone", None
|
||
except OSError:
|
||
return "unreadable", None
|
||
if getattr(info, "st_file_attributes", 0) & getattr(
|
||
stat_module, "FILE_ATTRIBUTE_REPARSE_POINT", 0
|
||
):
|
||
return "link", info
|
||
if expected is None:
|
||
problem = file_problem(path, root, None, info.st_size, hash_check=False)
|
||
else:
|
||
problem = file_problem(
|
||
path,
|
||
root,
|
||
None,
|
||
expected["size"],
|
||
hash_check=False,
|
||
mtime_ns=expected["mtime_ns"],
|
||
inode=expected["inode"],
|
||
)
|
||
if problem:
|
||
return _PROBLEM_CODES.get(problem, "outside"), info
|
||
if len(os.path.normcase(os.path.realpath(path))) > MAX_LOCATOR_LENGTH:
|
||
return "path-too-long", info
|
||
return None, info
|
||
|
||
|
||
def _row(target, code):
|
||
return {
|
||
"path": target["rel"],
|
||
"root_label": target["root_label"],
|
||
"code": code,
|
||
"detail": REASONS[code],
|
||
}
|
||
|
||
|
||
def _summary(rows):
|
||
counts = {}
|
||
for row in rows:
|
||
counts[row["code"]] = counts.get(row["code"], 0) + 1
|
||
return {"count": len(rows), "counts": counts, "rows": rows[:MAX_DETAIL_ROWS]}
|
||
|
||
|
||
def _fingerprint(targets) -> str:
|
||
material = [
|
||
(t["root_index"], t["rel"], t["size"], t["mtime_ns"], t["inode"])
|
||
for t in targets
|
||
]
|
||
return hashlib.sha256(json.dumps(material).encode("utf-8")).hexdigest()
|
||
|
||
|
||
class ImageDeleteMixin:
|
||
"""The Images Delete… review and its apply job (#480)."""
|
||
|
||
def _register_image_delete_routes(self, routes):
|
||
# A fixed path beside the dynamic /images/{image_id} routes: aiohttp
|
||
# (3.10+) looks fixed paths up first (see image_categories.py).
|
||
@routes.post("/prompt_manager/images/delete/preview")
|
||
async def preview_image_delete_route(request):
|
||
return await self.preview_image_delete(request)
|
||
|
||
# ── review ────────────────────────────────────────────────────────
|
||
|
||
async def preview_image_delete(self, request):
|
||
"""Review the selection under the maintenance preview lease."""
|
||
try:
|
||
async with self._jobs.lease(MAINTENANCE_PREVIEW_LEASE):
|
||
return await self._preview_image_delete(request)
|
||
except JobBusy as busy:
|
||
return busy_response(busy)
|
||
|
||
async def _preview_image_delete(self, request):
|
||
_data, targets, roots, error = await self._category_request(
|
||
request, frozenset()
|
||
)
|
||
if error is not None:
|
||
return error
|
||
db = self.db
|
||
identity = self._database_identity(db)
|
||
try:
|
||
files = {}
|
||
for index in dict.fromkeys(t[0] for t in targets):
|
||
files[index] = await self._get_gallery_files(roots[index])
|
||
review = await self._run_in_executor(
|
||
self._review_image_delete, db, roots, targets, files
|
||
)
|
||
except Exception:
|
||
self.logger.exception("Image delete review failed")
|
||
return _failure("The review failed. Check server logs for details.", 500)
|
||
return await self._plan_response(
|
||
db,
|
||
identity,
|
||
IMAGE_DELETE,
|
||
_fingerprint(review["targets"]),
|
||
review["public"],
|
||
{"targets": review["targets"]},
|
||
)
|
||
|
||
def _review_image_delete(self, db, roots, targets, files):
|
||
"""Every target checked now (blocking); ``{targets, public}``."""
|
||
listings, seen = {}, set()
|
||
acted, refused, trash_refused = [], [], []
|
||
spellings = self._source_spellings()
|
||
root_texts = [str(root) for root in roots]
|
||
for index, _scope, relative in targets:
|
||
root = roots[index]
|
||
if index not in listings:
|
||
listings[index] = self._favorite_listing(root, files[index])
|
||
spelled = listings[index].get(_favorite_relative_key(relative))
|
||
target = {
|
||
"root_index": index,
|
||
"root_scope": _scope,
|
||
"root": str(root),
|
||
"root_label": self._public_path(root),
|
||
"rel": spelled or relative,
|
||
}
|
||
if spelled is None:
|
||
refused.append(_row(target, "not-listed"))
|
||
continue
|
||
absolute = os.path.join(str(root), *spelled.split("/"))
|
||
code, info = file_check(absolute, str(root), spelled)
|
||
if code:
|
||
refused.append(_row(target, code))
|
||
continue
|
||
identity = (info.st_dev, info.st_ino)
|
||
if identity in seen:
|
||
continue # one physical file named twice (nested roots, hardlinks)
|
||
seen.add(identity)
|
||
target.update(self._removal_of(absolute, info, spellings))
|
||
target["trash"] = trash.trash_available(absolute, root_texts)
|
||
if target["trash"]:
|
||
trash_refused.append(_row(target, target["trash"]))
|
||
acted.append(target)
|
||
counted = db.count_removal_records([t["removal"] for t in acted])
|
||
library_count = len(acted)
|
||
public = {
|
||
"operation": IMAGE_DELETE,
|
||
"scope": {"count": library_count, "label": "images"},
|
||
"library": {"count": library_count},
|
||
"trash": {"count": library_count - len(trash_refused)},
|
||
"protected": {"count": len(refused), "reasons": []},
|
||
"refused": _summary(refused),
|
||
"trash_refused": _summary(trash_refused),
|
||
"records": counted["records"],
|
||
"orphaned_prompts": counted["orphaned_prompts"],
|
||
"media": {"deletes_files": False, "label": "Chosen in the review"},
|
||
"details": [],
|
||
}
|
||
return {"targets": acted, "public": public}
|
||
|
||
@staticmethod
|
||
def _removal_of(absolute, info, spellings):
|
||
favorites = [
|
||
(_favorite_root_key(root), _favorite_relative_key(relative))
|
||
for root, relative in spellings.favorites(absolute).items()
|
||
]
|
||
return {
|
||
"abs": absolute,
|
||
"size": info.st_size,
|
||
"mtime_ns": info.st_mtime_ns,
|
||
"inode": [info.st_dev, info.st_ino],
|
||
"removal": {
|
||
"spellings": list(spellings.of(absolute).values()),
|
||
"locator": os.path.normcase(os.path.realpath(absolute)),
|
||
"favorites": favorites,
|
||
"category": spellings.category_key(absolute),
|
||
},
|
||
}
|
||
|
||
# ── apply ─────────────────────────────────────────────────────────
|
||
|
||
def _start_image_delete(self, db, plan, mode, backup_on):
|
||
"""Register the delete job for a reviewed plan (on the loop)."""
|
||
return self._jobs.start(
|
||
IMAGE_DELETE_JOB_KIND,
|
||
TITLES[mode],
|
||
admission="records",
|
||
cancellable=False,
|
||
scope={"operation": IMAGE_DELETE, "mode": mode},
|
||
groups=IMAGE_DELETE_GROUP_NAMES,
|
||
runner=lambda job: self._run_image_delete(job, db, plan, mode, backup_on),
|
||
)
|
||
|
||
async def _run_image_delete(self, job, db, plan, mode, backup_on):
|
||
targets = plan.payload["targets"]
|
||
unchanged = await self._run_in_executor(self._sources_unchanged, targets)
|
||
if not unchanged:
|
||
return _fail_job(job, "sources-changed", SOURCES_CHANGED)
|
||
ok, backup = await self._safety_backup_step(job, db, backup_on)
|
||
if not ok:
|
||
return None
|
||
try:
|
||
await self._delete_images(job, db, targets, mode, backup)
|
||
finally:
|
||
if backup:
|
||
await self._prune_backup_pool(job, db)
|
||
return None
|
||
|
||
async def _delete_images(self, job, db, targets, mode, backup):
|
||
job.update(total=len(targets), status_line=TITLES[mode])
|
||
roots = [str(root) for root in self._get_all_output_dirs()]
|
||
done_group = "trashed" if mode == "trash" else "removed"
|
||
changed, failed = 0, None
|
||
for done, target in enumerate(targets, start=1):
|
||
if failed:
|
||
self._record_delete(job, "not_changed", target, "not-attempted")
|
||
continue
|
||
group, code, label = await self._run_in_executor(
|
||
self._delete_one_image, db, target, mode, roots
|
||
)
|
||
if group == "errors":
|
||
failed = code or RECORDS_FAILED_CODE
|
||
text = REASONS[code] if code else RECORDS_FAILED
|
||
job.record(
|
||
"errors",
|
||
{
|
||
"path": target["rel"],
|
||
"root": target["root_label"],
|
||
"root_index": target["root_index"],
|
||
"root_scope": target["root_scope"],
|
||
"detail": text,
|
||
},
|
||
f"! {target['root_label']}/{target['rel']} · {text}",
|
||
)
|
||
elif group == done_group:
|
||
changed += 1
|
||
detail = (
|
||
f"Moved to the {label}" if label else "Removed from the library"
|
||
)
|
||
self._record_delete(job, group, target, None, detail)
|
||
else:
|
||
self._record_delete(job, group, target, code)
|
||
job.update(processed=done, progress=int(done / len(targets) * 100))
|
||
if changed:
|
||
self.invalidate_gallery_cache()
|
||
verb = "Moved" if mode == "trash" else "Removed"
|
||
where = "to the trash" if mode == "trash" else "from the library"
|
||
line = f"{verb} {changed:,} of {len(targets):,} images {where}"
|
||
job.log_line(line)
|
||
job.update(progress=100, status_line=f"{line} · {backup_note(backup)}")
|
||
if failed:
|
||
# "records-failed" names every stop at an errors row (the page's
|
||
# "Stopped after N of M" toast); the row says which error it was.
|
||
job.scope["failure"] = RECORDS_FAILED_CODE
|
||
job.publish({"type": "error", "code": failed, "message": line})
|
||
job.finish("failed")
|
||
|
||
@staticmethod
|
||
def _record_delete(job, group, target, code, detail=None):
|
||
text = detail or REASONS[code]
|
||
row = {
|
||
"path": target["rel"],
|
||
"root": target["root_label"],
|
||
"root_index": target["root_index"],
|
||
"root_scope": target["root_scope"],
|
||
"code": code,
|
||
"detail": text,
|
||
}
|
||
mark = {"removed": "-", "trashed": "-"}.get(group, "=")
|
||
job.record(
|
||
group, row, f"{mark} {target['root_label']}/{target['rel']} · {text}"
|
||
)
|
||
|
||
def _delete_one_image(self, db, target, mode, roots):
|
||
"""``(group, code, trash label)`` for one reviewed file (blocking)."""
|
||
code, _info = file_check(
|
||
target["abs"],
|
||
target["root"],
|
||
target["rel"],
|
||
expected={
|
||
"size": target["size"],
|
||
"mtime_ns": target["mtime_ns"],
|
||
"inode": target["inode"],
|
||
},
|
||
)
|
||
if code:
|
||
return "not_changed", code, None
|
||
label = None
|
||
if mode == "trash":
|
||
try:
|
||
label = trash.move_to_trash(target["abs"], roots)
|
||
except trash.TrashRefused as refused:
|
||
code = refused.code if refused.code in REASONS else trash.FAILED
|
||
self.logger.info(
|
||
"Not moved to the trash (%s): %s %s",
|
||
code,
|
||
target["rel"],
|
||
refused.detail,
|
||
)
|
||
return "not_changed", code, None
|
||
except trash.TrashUnconfirmed as unconfirmed:
|
||
# The file left its folder: keep its records, stop the job.
|
||
self.logger.warning(
|
||
"Trash move not confirmed: %s %s", target["rel"], unconfirmed.detail
|
||
)
|
||
return "errors", TRASH_UNCONFIRMED, None
|
||
mark = None
|
||
else:
|
||
mark = (target["size"], target["mtime_ns"])
|
||
try:
|
||
db.remove_image_records(target["removal"], mark=mark)
|
||
except Exception:
|
||
self.logger.exception("Removing an image's records failed")
|
||
return "errors", None, label
|
||
_remove_thumbnail(Path(target["root"]), Path(target["rel"]), self.logger)
|
||
return ("trashed" if mode == "trash" else "removed"), None, label
|
||
|
||
# ── the workspace listing ─────────────────────────────────────────
|
||
|
||
def _removal_marks(self) -> dict:
|
||
"""The removal marks, cached until a mark changes or the database does.
|
||
|
||
A listing page reads no table while nothing changed: the cache follows
|
||
this process's marks generation and the database identity (a restore).
|
||
"""
|
||
key = (self._database_identity(self.db), image_removal_schema.generation())
|
||
cached = getattr(self, "_marks_cache", None)
|
||
if cached is None or cached[0] != key:
|
||
cached = (key, self.db.removal_marks())
|
||
self._marks_cache = cached
|
||
return cached[1]
|
||
|
||
def _hidden_entries(self, listings) -> set:
|
||
"""Normalized paths of listed files a removal mark hides (blocking).
|
||
|
||
Only marked locators are looked at, and only those files are
|
||
``lstat``'ed: a mark hides a file while its size and ``mtime_ns`` are
|
||
still the ones it was removed with.
|
||
"""
|
||
marks = self._removal_marks()
|
||
if not marks:
|
||
return set()
|
||
located = self._workspace_locators(listings)
|
||
return {
|
||
entry.normalized
|
||
for locator, entry in ((locator, located.get(locator)) for locator in marks)
|
||
if entry is not None and mark_hides(marks, locator, entry.path)
|
||
}
|
||
|
||
def _removed_paths(self, paths) -> set:
|
||
"""The absolute paths among *paths* a removal mark hides (blocking)."""
|
||
marks = self._removal_marks()
|
||
if not marks:
|
||
return set()
|
||
return {
|
||
str(path)
|
||
for path in paths
|
||
if mark_hides(marks, os.path.normcase(os.path.realpath(str(path))), path)
|
||
}
|
||
|
||
def _removal_check_under(self, root):
|
||
"""``hidden(path) -> bool`` for files a walk of *root* found (blocking).
|
||
|
||
A walk never follows a link below *root*, so a file's locator is
|
||
``normcase(join(realpath(root), relative))``: no ``realpath`` per
|
||
file, and an ``lstat`` only for a marked one. Without marks it
|
||
answers False at once.
|
||
"""
|
||
marks = self._removal_marks()
|
||
if not marks:
|
||
return lambda _path: False
|
||
root = os.fspath(root)
|
||
real = os.path.realpath(root)
|
||
|
||
def hidden(path):
|
||
relative = os.path.relpath(os.fspath(path), root)
|
||
locator = os.path.normcase(os.path.join(real, relative))
|
||
return mark_hides(marks, locator, path)
|
||
|
||
return hidden
|
||
|
||
def _without_removed(self, root, paths) -> list:
|
||
"""*paths* found by a walk of *root*, less the hidden ones (blocking)."""
|
||
removed = self._removal_check_under(root)
|
||
return [path for path in paths if not removed(path)]
|
||
|
||
|
||
def parse_mode(value):
|
||
"""The apply's ``mode``, or None when it is not one of MODES."""
|
||
return value if isinstance(value, str) and value in MODES else None
|