Vito Sansevero
680afb5e20
Duplicate prompts, unused components, missing links, workflow storage and rotated logs now record one protected row per kept item, in the review's wording, and the job report lists every one of them. Kept links and log files keep their path. The duplicate file cleanup records each copy the review protected in its Kept group.
473 lines
17 KiB
Python
473 lines
17 KiB
Python
"""Reviewing duplicate image files: the checks and the plan (#249 S6, T17).
|
|
|
|
Pure helpers of ``py/api/duplicate_files.py``: the keep-list parsing, the
|
|
per-file check ``file_problem`` (containment decided as text before any
|
|
filesystem access, then a regular file of the found size and hash) and the
|
|
public ``duplicate_files`` plan.
|
|
"""
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import stat as stat_module
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
|
|
from .admin import DELETABLE_MEDIA_EXTENSIONS
|
|
from .maintenance import DUPLICATE_FILES
|
|
|
|
try:
|
|
from ...utils.media_folders import in_trash_below_roots, is_trash_folder
|
|
except ImportError:
|
|
from utils.media_folders import in_trash_below_roots, is_trash_folder
|
|
|
|
# Keep entries one review takes; with paths of a few hundred characters the
|
|
# list stays inside the preview body cap (DUPLICATE_PREVIEW_BODY_MAX, 1 MiB).
|
|
MAX_KEEP = 2000
|
|
MAX_DETAILS = 50
|
|
MAX_JOB_ID = 64
|
|
HASH_CHUNK = 1024 * 1024
|
|
|
|
# What ``file_problem`` finds, and how a plan counts it as protected.
|
|
OUTSIDE_SOURCE = "outside"
|
|
LINK = "link"
|
|
CHANGED_CONTENT = "changed"
|
|
GONE = "gone"
|
|
UNREADABLE = "unreadable"
|
|
KEEPER = "keeper"
|
|
SAME = "same"
|
|
MAYBE_SAME = "maybe-same"
|
|
NO_FILE_ID = "no-file-id"
|
|
# Two mounts of one share may round mtimes differently (SMB keeps 100 ns,
|
|
# FAT 2 s): within this, equal sizes on another device may be one file.
|
|
MOUNT_MTIME_SLACK_NS = 2_000_000_000
|
|
PROTECTED_REASONS = {
|
|
MAYBE_SAME: (
|
|
"file may be the same file reached through another mount or share",
|
|
"files may be the same file reached through another mount or share",
|
|
),
|
|
NO_FILE_ID: (
|
|
"file: can't confirm this is a separate file on this drive",
|
|
"files: can't confirm these are separate files on this drive",
|
|
),
|
|
SAME: (
|
|
"file is the same file as its kept copy",
|
|
"files are the same file as their kept copy",
|
|
),
|
|
OUTSIDE_SOURCE: (
|
|
"file is not inside a configured source",
|
|
"files are not inside a configured source",
|
|
),
|
|
LINK: ("file is a link, not a regular file", "files are links, not regular files"),
|
|
CHANGED_CONTENT: (
|
|
"file changed since the search",
|
|
"files changed since the search",
|
|
),
|
|
GONE: ("file no longer exists", "files no longer exist"),
|
|
UNREADABLE: ("file could not be read", "files could not be read"),
|
|
KEEPER: (
|
|
"file kept: its kept copy changed or is gone",
|
|
"files kept: their kept copy changed or is gone",
|
|
),
|
|
}
|
|
# The job's row for a copy the review protected (#586): the review's reason.
|
|
REVIEW_KEPT_DETAILS = {
|
|
MAYBE_SAME: "Kept: may be the same file reached through another mount or share",
|
|
NO_FILE_ID: "Kept: can't confirm this is a separate file on this drive",
|
|
SAME: "Kept: the same file as its kept copy",
|
|
OUTSIDE_SOURCE: "Kept: not inside a configured source",
|
|
LINK: "Kept: a link, not a regular file",
|
|
CHANGED_CONTENT: "Kept: changed since the search",
|
|
GONE: "Kept: no longer exists",
|
|
UNREADABLE: "Kept: could not be read",
|
|
KEEPER: "Kept: its kept copy changed or is gone",
|
|
}
|
|
# Job outcome rows (never an OS error text).
|
|
CHANGED = "The file changed since the review; it was kept."
|
|
KEEPER_CHANGED = "The kept copy changed or is gone; this file was kept."
|
|
SAME_FILE = "This is the same file as its kept copy; it was kept."
|
|
MAYBE_SAME_FILE = (
|
|
"May be the same file reached through another mount or share; it was kept."
|
|
)
|
|
NO_FILE_ID_ROW = "Can't confirm this is a separate file on this drive; it was kept."
|
|
KEEPER_GONE = "keeper-gone"
|
|
KEEPER_WENT = (
|
|
"The kept copy changed or went away during the removal; this file was kept "
|
|
"and its links stay with it."
|
|
)
|
|
ROW_DETAILS = {
|
|
KEEPER_GONE: None, # filled below
|
|
MAYBE_SAME: MAYBE_SAME_FILE,
|
|
NO_FILE_ID: NO_FILE_ID_ROW,
|
|
SAME: SAME_FILE,
|
|
OUTSIDE_SOURCE: "The file is no longer inside its source; it was kept.",
|
|
LINK: "The file is now a link, not a regular file; it was kept.",
|
|
CHANGED_CONTENT: CHANGED,
|
|
GONE: "The file was already gone.",
|
|
UNREADABLE: "The file could not be read; it was kept.",
|
|
KEEPER: KEEPER_CHANGED,
|
|
}
|
|
# An unlink fails only after the file's records moved to its kept copy.
|
|
LOCKED_FILE = (
|
|
"The file is open in another program or read-only; it was not deleted, "
|
|
"and its links now name the kept copy."
|
|
)
|
|
DELETE_FAILED = "The file could not be deleted; its links now name the kept copy."
|
|
NOT_ATTEMPTED = "Not attempted: the job stopped after an error."
|
|
KEPT_COPY = "Kept copy"
|
|
SOURCES_CHANGED = (
|
|
"The sources changed since the review; nothing was deleted. Review again."
|
|
)
|
|
RECORDS_FAILED = (
|
|
"The database could not be updated, so the file was not deleted. "
|
|
"Its links and tags are unchanged."
|
|
)
|
|
FIND_FAILED = "The duplicate search failed."
|
|
FIND_ERROR_EVENT = "An internal error occurred. Check server logs for details."
|
|
FIND_CANCELLED = "Duplicate scan cancelled"
|
|
|
|
|
|
class SourcesChanged(Exception):
|
|
"""A source a review or removal names is no longer listed at its index."""
|
|
|
|
|
|
ROW_DETAILS[KEEPER_GONE] = KEEPER_WENT
|
|
|
|
|
|
def still_kept(target, hold):
|
|
"""Right before an unlink: the held kept copy is still the file at its
|
|
path, and the file is still not the kept copy itself (``keeper_problem``).
|
|
"""
|
|
try:
|
|
if not os.path.samestat(os.fstat(hold.fileno()), os.stat(target["keeper_abs"])):
|
|
return False
|
|
info = os.lstat(target["abs"])
|
|
problem = keeper_problem(
|
|
target["abs"],
|
|
target["root"],
|
|
info,
|
|
target["keeper_abs"],
|
|
target["keeper_root"],
|
|
)
|
|
return problem is None
|
|
except OSError:
|
|
return False
|
|
|
|
|
|
def _plural(count, one, many=None):
|
|
return f"{count:,} {one if count == 1 else (many or one + 's')}"
|
|
|
|
|
|
def human_bytes(size):
|
|
"""``10 B``, ``1.5 KB``, ``2.0 MB``…"""
|
|
if size < 1024:
|
|
return f"{size:,} B"
|
|
for unit in ("KB", "MB", "GB", "TB"):
|
|
size /= 1024
|
|
if size < 1024 or unit == "TB":
|
|
return f"{size:,.1f} {unit}"
|
|
return f"{size:,.1f} TB" # pragma: no cover - unreachable
|
|
|
|
|
|
def _hash_file(path):
|
|
digest = hashlib.sha256()
|
|
with open(path, "rb") as handle:
|
|
for chunk in iter(lambda: handle.read(HASH_CHUNK), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def file_problem(
|
|
path,
|
|
root,
|
|
digest,
|
|
size,
|
|
hasher=None,
|
|
*,
|
|
hash_check=True,
|
|
mtime_ns=None,
|
|
inode=None,
|
|
keeper=None,
|
|
keeper_root=None,
|
|
):
|
|
"""Why ``path`` may not be treated as a copy in ``root``, or None (blocking).
|
|
|
|
Containment is decided as text first, with no filesystem access: the
|
|
normalized path must lie below the normalized root, outside
|
|
``thumbnails``, with a media extension. Only then is the file looked at:
|
|
a regular file, not a link, whose resolved path is still inside the
|
|
resolved root and that is not ``keeper`` itself (``keeper_problem``: a
|
|
hardlink, one file reached through two spellings or two mounts, or a
|
|
volume that cannot tell files apart), of ``size`` bytes.
|
|
``mtime_ns`` and ``inode`` (``[st_dev, st_ino]``), when given, bind it to
|
|
what the search saw; ``hash_check`` compares its content hash with
|
|
``digest`` (the removal job, right before an unlink). Every handle is
|
|
closed when this returns.
|
|
"""
|
|
# py.config imports ComfyUI's server module: only when it is in use.
|
|
from ..config import _canonical_path, _lexical_path, _lexically_inside
|
|
|
|
lexical = _lexical_path(str(path))
|
|
base = _lexical_path(str(root))
|
|
if lexical == base or not _lexically_inside(lexical, [base]):
|
|
return OUTSIDE_SOURCE
|
|
parts = Path(os.path.relpath(lexical, base)).parts
|
|
if "thumbnails" in parts or ".." in parts:
|
|
return OUTSIDE_SOURCE
|
|
if any(is_trash_folder(part) for part in parts[:-1]):
|
|
return OUTSIDE_SOURCE # never removed from a trash folder (#543)
|
|
if os.path.splitext(lexical)[1].lower() not in DELETABLE_MEDIA_EXTENSIONS:
|
|
return OUTSIDE_SOURCE
|
|
try:
|
|
info = os.lstat(path)
|
|
except FileNotFoundError:
|
|
return GONE
|
|
except OSError:
|
|
return UNREADABLE
|
|
if stat_module.S_ISLNK(info.st_mode):
|
|
return LINK
|
|
if not stat_module.S_ISREG(info.st_mode):
|
|
return LINK
|
|
try:
|
|
if not _lexically_inside(_canonical_path(path), [_canonical_path(root)]):
|
|
return OUTSIDE_SOURCE # a linked folder on the way
|
|
if in_trash_below_roots(path, [root]):
|
|
return OUTSIDE_SOURCE # a linked folder into a trash folder
|
|
if keeper is not None:
|
|
same = keeper_problem(path, root, info, keeper, keeper_root)
|
|
if same:
|
|
return same
|
|
if info.st_size != size:
|
|
return CHANGED_CONTENT
|
|
if mtime_ns is not None and info.st_mtime_ns != mtime_ns:
|
|
return CHANGED_CONTENT
|
|
if inode is not None and [info.st_dev, info.st_ino] != list(inode):
|
|
return CHANGED_CONTENT
|
|
if hash_check and (hasher or _hash_file)(path) != digest:
|
|
return CHANGED_CONTENT
|
|
except FileNotFoundError:
|
|
return GONE
|
|
except OSError:
|
|
return UNREADABLE
|
|
return None
|
|
|
|
|
|
def keeper_problem(path, root, info, keeper, keeper_root=None):
|
|
"""Why ``path`` (its ``lstat`` is ``info``) may be ``keeper`` itself, or None.
|
|
|
|
- ``NO_FILE_ID``: a volume reports no file id (``st_ino`` 0), so the two
|
|
cannot be told apart;
|
|
- ``SAME``: ``os.path.samestat`` (a hardlink, or one file reached through
|
|
two spellings of one mount);
|
|
- ``MAYBE_SAME``: another device, the same size, mtimes within
|
|
``MOUNT_MTIME_SLACK_NS``, and the same ``st_ino`` (NFS and SMB report
|
|
the server's file id) or the same path within their sources, or one
|
|
ending with the other on a folder name (one share mounted twice, a
|
|
FUSE pool over a listed disk, sources listed at different depths).
|
|
Protect, never guess: a true copy on another disk at the same relative
|
|
path is kept too.
|
|
|
|
Raises OSError when the kept copy cannot be read (the caller's GONE or
|
|
UNREADABLE).
|
|
"""
|
|
kept = os.stat(keeper)
|
|
if not info.st_ino or not kept.st_ino:
|
|
return NO_FILE_ID
|
|
if os.path.samestat(info, kept):
|
|
return SAME
|
|
if info.st_dev == kept.st_dev or info.st_size != kept.st_size:
|
|
return None
|
|
if abs(info.st_mtime_ns - kept.st_mtime_ns) > MOUNT_MTIME_SLACK_NS:
|
|
return None
|
|
if info.st_ino == kept.st_ino:
|
|
return MAYBE_SAME
|
|
if keeper_root is not None and _same_tail(
|
|
_relative_key(path, root), _relative_key(keeper, keeper_root)
|
|
):
|
|
return MAYBE_SAME
|
|
return None
|
|
|
|
|
|
def _same_tail(one, other):
|
|
"""Whether one relative path is the other, or ends with it on a folder name.
|
|
|
|
Two mounts of one tree may be listed at different depths (``2024/x.png``
|
|
in one source, ``x.png`` in the other); ``ax.png`` never matches ``x.png``.
|
|
"""
|
|
if one is None or other is None:
|
|
return False
|
|
return one == other or one.endswith("/" + other) or other.endswith("/" + one)
|
|
|
|
|
|
def _relative_key(path, root):
|
|
try:
|
|
relative = os.path.relpath(str(path), str(root))
|
|
except ValueError: # another drive (Windows)
|
|
return None
|
|
return os.path.normcase(relative).replace("\\", "/")
|
|
|
|
|
|
@dataclass
|
|
class HeldFind:
|
|
"""One find: where it looked and, once it finished, what it found."""
|
|
|
|
origin: str # "workspace" or "gallery"
|
|
job_id: str = ""
|
|
roots: list = field(default_factory=list) # [(source index or None, root)]
|
|
legacy_dir: str = None
|
|
groups: list = None # [{group_id, hash, size, files:[record]}]
|
|
|
|
def covers(self, legacy_dir):
|
|
"""Whether its events answer the legacy gallery for ``legacy_dir``."""
|
|
if self.origin == "gallery":
|
|
return True
|
|
if not legacy_dir:
|
|
return False
|
|
wanted = _identity(legacy_dir)
|
|
return any(_identity(root) == wanted for _index, root in self.roots)
|
|
|
|
|
|
def _identity(path):
|
|
return os.path.normcase(os.path.realpath(str(path)))
|
|
|
|
|
|
def _group_order(group):
|
|
return (-(len(group["files"]) - 1) * group["size"], group["hash"])
|
|
|
|
|
|
def _keep_entry(entry):
|
|
"""``((group_id, root_index, path), None)`` or ``(None, message)``."""
|
|
if not isinstance(entry, dict) or set(entry) != {"group_id", "root_index", "path"}:
|
|
return None, "Each keep entry is {group_id, root_index, path}"
|
|
group_id, index, path = entry["group_id"], entry["root_index"], entry["path"]
|
|
if not isinstance(group_id, str) or not 0 < len(group_id) <= 128:
|
|
return None, "group_id must be a group's id"
|
|
if index is not None and (
|
|
isinstance(index, bool) or not isinstance(index, int) or index < 0
|
|
):
|
|
return None, "root_index must be a source index"
|
|
if not isinstance(path, str) or not 0 < len(path) <= 4096:
|
|
return None, "path must be a file's path in its group"
|
|
return (group_id, index, path), None
|
|
|
|
|
|
def _parse_options(options):
|
|
"""``(job_id, keep, None)`` for duplicate_files options, else a message."""
|
|
if not isinstance(options, dict) or set(options) != {"job_id", "keep"}:
|
|
return None, None, "duplicate_files takes options {job_id, keep}"
|
|
job_id, keep = options["job_id"], options["keep"]
|
|
if not isinstance(job_id, str) or not 0 < len(job_id) <= MAX_JOB_ID:
|
|
return None, None, "job_id must be a duplicate search's job id"
|
|
if not isinstance(keep, list) or not 0 < len(keep) <= MAX_KEEP:
|
|
return None, None, f"keep must list 1 to {MAX_KEEP:,} files to keep"
|
|
parsed = []
|
|
for entry in keep:
|
|
key, message = _keep_entry(entry)
|
|
if message:
|
|
return None, None, message
|
|
parsed.append(key)
|
|
return job_id, parsed, None
|
|
|
|
|
|
def _select(held, keep):
|
|
"""``({group_id: (group, [keeper records])}, None)`` or ``(None, message)``."""
|
|
groups = {group["group_id"]: group for group in held.groups}
|
|
selection = {}
|
|
for group_id, index, path in keep:
|
|
group = groups.get(group_id)
|
|
if group is None:
|
|
return None, "A kept file names a group this search did not find"
|
|
match = [
|
|
record
|
|
for record in group["files"]
|
|
if record["root_index"] == index and record["rel"] == path
|
|
]
|
|
if not match:
|
|
return None, "A kept file is not in its group"
|
|
_group, keepers = selection.setdefault(group_id, (group, []))
|
|
if match[0] not in keepers:
|
|
keepers.append(match[0])
|
|
return selection, None
|
|
|
|
|
|
SOURCES_CHANGED_REVIEW = (
|
|
"The sources changed since the search. Search for duplicates again."
|
|
)
|
|
|
|
|
|
def _fingerprint(targets):
|
|
digest = hashlib.sha256()
|
|
for target in sorted(targets, key=lambda t: _identity(t["abs"])):
|
|
part = [
|
|
os.path.normcase(target["abs"]),
|
|
target["hash"],
|
|
target["size"],
|
|
os.path.normcase(target["keeper_abs"]),
|
|
]
|
|
digest.update(json.dumps(part).encode("utf-8") + b"\n")
|
|
return "sha256:" + digest.hexdigest()
|
|
|
|
|
|
def _record_parts(records, target):
|
|
"""What moves with one removal: its links, tags and Needs review items."""
|
|
found = records.get(target["abs"], {})
|
|
links, tags, review = (found.get(key, 0) for key in ("links", "tags", "review"))
|
|
parts = [
|
|
(
|
|
(
|
|
f"{_plural(links, 'database link')} "
|
|
f"{'moves' if links == 1 else 'move'} to the kept copy"
|
|
)
|
|
if links
|
|
else "no database links"
|
|
)
|
|
]
|
|
if tags:
|
|
parts.append("its image tags join the kept copy's")
|
|
if review:
|
|
item = "Needs review item is" if review == 1 else "Needs review items are"
|
|
parts.append(f"{review:,} {item} settled")
|
|
return parts
|
|
|
|
|
|
def _public_plan(targets, protected, records):
|
|
count = len(targets)
|
|
groups = len({t["group_id"] for t in targets})
|
|
size = sum(t["size"] for t in targets)
|
|
reasons = [
|
|
f"{n:,} {PROTECTED_REASONS[code][0 if n == 1 else 1]}"
|
|
for code, n in sorted(protected.items())
|
|
]
|
|
details = [
|
|
{
|
|
"label": target["rel"],
|
|
"detail": f"{target['root_label']} · a copy of {target['keeper_rel']}"
|
|
+ "".join(f" · {part}" for part in _record_parts(records, target)),
|
|
}
|
|
for target in targets[:MAX_DETAILS]
|
|
]
|
|
label = f"{_plural(count, 'duplicate file')} in {_plural(groups, 'group')}"
|
|
media = (
|
|
f"Deletes {_plural(count, 'file')} ({human_bytes(size)}) from disk; the "
|
|
"kept copies stay. The safety backup covers the database, not these files."
|
|
)
|
|
tagged = sum(1 for t in targets if records.get(t["abs"], {}).get("tags"))
|
|
review = sum(records.get(t["abs"], {}).get("review", 0) for t in targets)
|
|
if tagged:
|
|
media += (
|
|
f" The image tags of {_plural(tagged, 'removed copy', 'removed copies')}"
|
|
" join their kept copy's."
|
|
)
|
|
if review:
|
|
media += (
|
|
f" {_plural(review, 'pending Needs review item')} of the removed "
|
|
f"{'copy is' if review == 1 else 'copies are'} settled."
|
|
)
|
|
return {
|
|
"operation": DUPLICATE_FILES,
|
|
"scope": {"count": count, "label": label},
|
|
"protected": {"count": sum(protected.values()), "reasons": reasons},
|
|
"media": {"deletes_files": True, "label": media},
|
|
"details": details,
|
|
}
|