nodes/ComfyAssets-ComfyUI_PromptManager
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(maintenance): recheck the kept copy right before each unlink

Browse Source 
Re-review 2 of #313:

- n1: the cross-mount guard also matches when one path within its source
  ends with the other on a whole folder name (sources listed at different
  depths, e.g. 2024/x.png and x.png on a FUSE or noserverino mount).
- n2: after a file's records move and right before its unlink, the held
  kept copy must still be the file at its path (fstat of the hold against
  os.stat) and the cross-mount guard must still hold; otherwise the file
  is kept, its records move back to it, and it counts as not removed.
This commit is contained in:
Vito Sansevero
2026-10-23 20:52:35 -07:00
parent c9f78fac58
commit 4a615a0c8d
5 changed files with 202 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
py/api/duplicate_files.py
+32 -45
View File
@@ -56,6 +56,8 @@ from .duplicate_review import ( # noqa: F401 (CHANGED, OUTSIDE_SOURCE: public)
GONE,
KEEPER,
KEEPER_CHANGED,
KEEPER_GONE,
KEEPER_WENT,
KEPT_COPY,
LOCKED_FILE,
MAYBE_SAME_FILE,
@@ -78,9 +80,10 @@ from .duplicate_review import ( # noqa: F401 (CHANGED, OUTSIDE_SOURCE: public)
_select,
file_problem,
human_bytes,
still_kept,
)
from .duplicate_spellings import Spellings
from .images import _favorite_relative_key, _favorite_root_key, _output_image_entry
from .duplicate_spellings import Spellings, copy_move
from .images import _favorite_root_key, _output_image_entry
from .job_routes import busy_response
from .jobs import (
DUPLICATE_CLEANUP_GROUP_NAMES,
@@ -684,14 +687,20 @@ class DuplicateFilesMixin:
except OSError:
return "kept", KEEPER, None
with hold:
return self._move_then_unlink(db, target, keepers[group_id], spellings)
return self._move_then_unlink(
db, target, keepers[group_id], spellings, hold
)
def _move_then_unlink(self, db, target, kept_stat, spellings):
def _move_then_unlink(self, db, target, kept_stat, spellings, hold):
try:
counts = db.move_copy_records(self._copy_move(target, kept_stat, spellings))
counts = db.move_copy_records(copy_move(target, kept_stat, spellings))
except Exception:
self.logger.exception(f"Could not move the records of {target['rel']}")
return "error", RECORDS_FAILED, None
if not still_kept(target, hold):
# The kept copy went away or changed while the records moved:
# keep this file and give its records back to it.
return "kept", KEEPER_GONE, self._move_back(db, target, spellings)
try:
_unlink(target["abs"])
except FileNotFoundError:
@@ -713,48 +722,26 @@ class DuplicateFilesMixin:
_remove_thumbnail(Path(target["root"]), Path(target["rel"]), self.logger)
return "removed", None, counts
@staticmethod
def _copy_move(target, kept_stat, spellings):
"""What ``move_copy_records`` moves for one removal.
A record found under a folder spelling moves to the kept copy under
the same spelling when that folder holds it too, else to its real
path. Its favorite moves under every listed root holding the file.
"""
removed = spellings.of(target["abs"])
kept = spellings.of(target["keeper_abs"])
targets = {
spelled: kept.get(folder, target["keeper_abs"])
for folder, spelled in removed.items()
def _move_back(self, db, target, spellings):
"""Give the kept copy's records to the file that stays (blocking)."""
swapped = {
**target,
"abs": target["keeper_abs"],
"root": target["keeper_root"],
"rel": target["keeper_rel"],
"locator": target["keeper_locator"],
"keeper_abs": target["abs"],
"keeper_root": target["root"],
"keeper_rel": target["rel"],
"keeper_locator": target["locator"],
}
kept_favorites = spellings.favorites(target["keeper_abs"])
favorites = []
for root, relative in spellings.favorites(target["abs"]).items():
kept_root, kept_rel = (
(root, kept_favorites[root])
if root in kept_favorites
else (target["keeper_root"], target["keeper_rel"])
try:
return db.move_copy_records(
copy_move(swapped, os.stat(target["abs"]), spellings)
)
favorites.append(
(
_favorite_root_key(root),
_favorite_relative_key(relative),
_favorite_root_key(kept_root),
_favorite_relative_key(kept_rel),
)
)
return {
"removed": list(removed.values()),
"kept": list(kept.values()),
"targets": targets,
"kept_path": target["keeper_abs"],
"favorites": favorites,
"removed_locator": target["locator"],
"kept_locator": target["keeper_locator"],
"sha256": target["hash"],
"kept_size": kept_stat.st_size,
"kept_mtime_ns": kept_stat.st_mtime_ns,
}
except Exception:
self.logger.exception(f"Could not give the records back to {target['rel']}")
return None
def _bound_problem(record, group, keeper=None):
py/api/duplicate_review.py
+47 -5
View File
@@ -73,7 +73,13 @@ MAYBE_SAME_FILE = (
"May be the same file reached through another mount or share; it was kept."
)
NO_FILE_ID_ROW = "Can't confirm this is a separate file on this drive; it was kept."
KEEPER_GONE = "keeper-gone"
KEEPER_WENT = (
"The kept copy changed or went away during the removal; this file was kept "
"and its links stay with it."
)
ROW_DETAILS = {
KEEPER_GONE: None, # filled below
MAYBE_SAME: MAYBE_SAME_FILE,
NO_FILE_ID: NO_FILE_ID_ROW,
SAME: SAME_FILE,
@@ -108,6 +114,29 @@ class SourcesChanged(Exception):
"""A source a review or removal names is no longer listed at its index."""
ROW_DETAILS[KEEPER_GONE] = KEEPER_WENT
def still_kept(target, hold):
"""Right before an unlink: the held kept copy is still the file at its
path, and the file is still not the kept copy itself (``keeper_problem``).
"""
try:
if not os.path.samestat(os.fstat(hold.fileno()), os.stat(target["keeper_abs"])):
return False
info = os.lstat(target["abs"])
problem = keeper_problem(
target["abs"],
target["root"],
info,
target["keeper_abs"],
target["keeper_root"],
)
return problem is None
except OSError:
return False
def _plural(count, one, many=None):
return f"{count:,} {one if count == 1 else (many or one + 's')}"
@@ -211,9 +240,11 @@ def keeper_problem(path, root, info, keeper, keeper_root=None):
two spellings of one mount);
- ``MAYBE_SAME``: another device, the same size, mtimes within
``MOUNT_MTIME_SLACK_NS``, and the same ``st_ino`` (NFS and SMB report
the server's file id) or the same path within their sources (one share
mounted twice, a FUSE pool over a listed disk). Protect, never guess:
a true copy on another disk at the same relative path is kept too.
the server's file id) or the same path within their sources, or one
ending with the other on a folder name (one share mounted twice, a
FUSE pool over a listed disk, sources listed at different depths).
Protect, never guess: a true copy on another disk at the same relative
path is kept too.
Raises OSError when the kept copy cannot be read (the caller's GONE or
UNREADABLE).
@@ -229,13 +260,24 @@ def keeper_problem(path, root, info, keeper, keeper_root=None):
return None
if info.st_ino == kept.st_ino:
return MAYBE_SAME
if keeper_root is not None and _relative_key(path, root) == _relative_key(
keeper, keeper_root
if keeper_root is not None and _same_tail(
_relative_key(path, root), _relative_key(keeper, keeper_root)
):
return MAYBE_SAME
return None
def _same_tail(one, other):
"""Whether one relative path is the other, or ends with it on a folder name.
Two mounts of one tree may be listed at different depths (``2024/x.png``
in one source, ``x.png`` in the other); ``ax.png`` never matches ``x.png``.
"""
if one is None or other is None:
return False
return one == other or one.endswith("/" + other) or other.endswith("/" + one)
def _relative_key(path, root):
try:
relative = os.path.relpath(str(path), str(root))
py/api/duplicate_spellings.py
+45
View File
@@ -18,6 +18,8 @@ Blocking: each lookup stats the file's folders once (cached).
import os
from pathlib import Path
from .images import _favorite_relative_key, _favorite_root_key
def _identity(path):
try:
@@ -86,3 +88,46 @@ class Spellings:
if inside is not None:
held[root] = Path(os.path.relpath(path, inside)).as_posix()
return held
def copy_move(target, kept_stat, spellings):
"""What ``move_copy_records`` moves for one removal.
A record found under a folder spelling moves to the kept copy under
the same spelling when that folder holds it too, else to its real
path. Its favorite moves under every listed root holding the file.
"""
removed = spellings.of(target["abs"])
kept = spellings.of(target["keeper_abs"])
targets = {
spelled: kept.get(folder, target["keeper_abs"])
for folder, spelled in removed.items()
}
kept_favorites = spellings.favorites(target["keeper_abs"])
favorites = []
for root, relative in spellings.favorites(target["abs"]).items():
kept_root, kept_rel = (
(root, kept_favorites[root])
if root in kept_favorites
else (target["keeper_root"], target["keeper_rel"])
)
favorites.append(
(
_favorite_root_key(root),
_favorite_relative_key(relative),
_favorite_root_key(kept_root),
_favorite_relative_key(kept_rel),
)
)
return {
"removed": list(removed.values()),
"kept": list(kept.values()),
"targets": targets,
"kept_path": target["keeper_abs"],
"favorites": favorites,
"removed_locator": target["locator"],
"kept_locator": target["keeper_locator"],
"sha256": target["hash"],
"kept_size": kept_stat.st_size,
"kept_mtime_ns": kept_stat.st_mtime_ns,
}