feat(images): Show removed images and Return to library (#498)

- GET /images/output?mode=workspace&removed=1 lists only files a removal
  mark hides; every workspace page carries removed_total.
- POST /images/removed/return {items | filter} starts a scan job,
  "Return images to the library": the marks that still hide their file
  go in one transaction, then exactly those files are scanned, so their
  prompt links come back from their metadata. Tags, categories,
  collections and favorites do not come back.
- The marks generation moves again after the commit, so a reader that
  cached the marks mid-transaction reloads them.
- Images: Show removed images (N) heading toggle (D162), Removed chips,
  a bar with only Return to library and Clear, Return to library in the
  side panel; the view is never saved or restored.

py/api/__init__.py

@@ -76,6 +76,7 @@ from .lora_integration import LoraIntegrationMixin
 from .lora_import_routes import LoraImportRoutesMixin
 from .image_categories import ImageCategoryRoutesMixin
 from .image_delete import ImageDeleteMixin
+from .image_return import ImageReturnMixin
 from .source_rescope import SourceRescopeMixin
 
 try:
@@ -346,6 +347,7 @@ class PromptManagerAPI(
     LoraImportRoutesMixin,
     ImageCategoryRoutesMixin,
     ImageDeleteMixin,
+    ImageReturnMixin,
     SourceRescopeMixin,
 ):
     """REST API handler for PromptManager operations and web interface.
@@ -724,6 +726,7 @@ class PromptManagerAPI(
         self._register_lora_import_routes(routes)
         self._register_image_category_routes(routes)
         self._register_image_delete_routes(routes)
+        self._register_image_return_routes(routes)
         self._register_rescope_routes(routes)
 
         # Register gzip compression middleware (once)

py/api/admin.py

@@ -241,7 +241,12 @@ SCAN_FINAL_STATES = {
     "error": "failed",
 }
 # The scope as the scan log's first line names it (the review dialog's words).
-SCAN_SCOPE_LOG = {"new": "new and unresolved files", "all": "all files"}
+SCAN_SCOPE_LOG = {
+    "new": "new and unresolved files",
+    "all": "all files",
+    "returned": "images returned to the library",
+}
+RETURN_JOB_TITLE = "Return images to the library"  # #498
 MIN_BATCHES_FOR_ESTIMATE = 2  # "about X min left" waits for this many batches
 # Log line prefix per outcome group (spec: + added, = known, ? review, ! error,
 # - no usable metadata).
@@ -1381,13 +1386,17 @@ class AdminRoutesMixin:
             }
         )
 
-    def _start_scan_job(self, files=DEFAULT_SCAN_SCOPE, roots=None, output_dirs=None):
+    def _start_scan_job(
+        self, files=DEFAULT_SCAN_SCOPE, roots=None, output_dirs=None, returned=None
+    ):
         """Register the scan job synchronously; its runner starts on the loop.
 
         ``files`` is the scope (``SCAN_SCOPES``). ``roots`` selects sources by
         their index in ``output_dirs`` (the effective sources the caller
         resolved); None scans every source, resolved when the job starts.
-        Raises JobBusy while another record-changing job runs.
+        Raises JobBusy while another record-changing job runs. ``returned``
+        (#498, ``[(path, locator)]``) makes it "Return images to the library":
+        those files' removal marks go first, then only they are scanned.
         """
         if files not in SCAN_SCOPES:
             raise ValueError(f"Unknown scan scope: {files!r}")
@@ -1395,20 +1404,25 @@ class AdminRoutesMixin:
             output_dirs is None or not roots or max(roots) >= len(output_dirs)
         ):
             raise ValueError("A scoped scan needs the sources its roots index")
+        scope = {"files": files, "roots": None if roots is None else list(roots)}
+        if returned is not None:
+            scope = {"files": "returned", "roots": None, "images": len(returned)}
         job = self._jobs.start(
             SCAN_JOB_KIND,
-            SCAN_JOB_TITLE,
+            SCAN_JOB_TITLE if returned is None else RETURN_JOB_TITLE,
             record_changing=True,
-            scope={"files": files, "roots": None if roots is None else list(roots)},
-            runner=functools.partial(self._run_scan_job, output_dirs=output_dirs),
+            scope=scope,
+            runner=functools.partial(
+                self._run_scan_job, output_dirs=output_dirs, returned=returned
+            ),
             groups=SCAN_GROUP_NAMES,
         )
         self._scan_job = job
         return job
 
-    async def _run_scan_job(self, job, output_dirs=None):
+    async def _run_scan_job(self, job, output_dirs=None, returned=None):
         final = None
-        async for payload in self._scan_images_events(job, output_dirs):
+        async for payload in self._scan_images_events(job, output_dirs, returned):
             job.publish(payload)
             final = payload.get("type")
         state = SCAN_FINAL_STATES.get(final, "finished")
@@ -1446,7 +1460,7 @@ class AdminRoutesMixin:
             )
         return response
 
-    async def _scan_images_events(self, job, output_dirs=None):
+    async def _scan_images_events(self, job, output_dirs=None, returned=None):
         """Yield progress payloads while scanning the output directories.
 
         Every effective source is walked, or only those ``job.scope["roots"]``
@@ -1484,9 +1498,16 @@ class AdminRoutesMixin:
                 0, f"Scanning {len(walked)} directory(ies) for media files..."
             )
 
-            media_files = await self._run_in_executor(
-                self._collect_output_media_sync, walked
-            )
+            if returned is None:
+                media_files = await self._run_in_executor(
+                    self._collect_output_media_sync, walked
+                )
+            else:
+                # #498: only the files whose marks this job deleted.
+                media_files = await self._run_in_executor(
+                    self._clear_returned_marks, returned
+                )
+                job.log_line(f"Returned {_plural(len(media_files), 'image')}")
             counts = dict.fromkeys(SCAN_COUNT_KEYS, 0)
             job.update(
                 total=len(media_files),

py/api/image_categories.py

@@ -232,15 +232,17 @@ class ImageCategoryRoutesMixin:
             return None, _coded(400, "too-many-images", TOO_MANY)
         return tuple(selection.targets), None
 
-    async def _category_request(self, request, fields: frozenset):
+    async def _category_request(self, request, fields: frozenset, data=None):
         """``(data, targets, roots, None)`` or ``(None, None, None, error)``.
 
         The images are ``items`` (1 to 500 targets) or ``filter`` (all images
-        an Images listing filter admits), besides *fields*.
+        an Images listing filter admits), besides *fields*. *data* is the body
+        when the caller has already read it (a body is read once).
         """
-        data, error = await _json_body_or_error(request)
-        if error is not None:
-            return None, None, None, error
+        if data is None:
+            data, error = await _json_body_or_error(request)
+            if error is not None:
+                return None, None, None, error
         names = set(data) - fields
         if set(data) & fields != fields or names not in ({"items"}, {"filter"}):
             return (

py/api/image_return.py

@@ -0,0 +1,148 @@
+"""Return removed images to the library (#498).
+
+``POST /prompt_manager/images/removed/return {items | filter}`` names files a
+removal mark hides (#480, "Remove from the library only"): ``items`` are 1 to
+500 listing identities, ``filter`` the removed view's listing query
+(``removed: "1"`` plus the view's other parameters, at most 500 images). Only
+this route takes ``removed``: every other bulk route refuses it, because an
+explicit write there would show a file again without re-reading it.
+
+The route answers 202 ``{job, returned, not_removed}`` after starting a
+``scan`` job titled "Return images to the library" (admission ``records``:
+409 ``busy`` beside a scan, restore or delete, nothing changed). The job
+first deletes the marks that still hide their file (same size and
+``mtime_ns``), in one transaction, then scans exactly those files with the
+scan's own pipeline: prompts, links, pairs and Needs review items come back
+from the files' embedded metadata. Image tags, auto-tag marks, the own
+category, collection memberships and the favorite were deleted by the remove
+and do not come back. No file is moved, written or deleted.
+"""
+
+import os
+from pathlib import Path
+
+from aiohttp import web
+
+from .image_categories import TOO_MANY, _coded
+from .images import (
+    ListingSourceChanged,
+    _favorite_relative_key,
+    _json_body_or_error,
+    parse_listing_filter,
+)
+from .image_tag_jobs import MAX_JOB_TARGETS
+from .jobs import JobBusy
+from .job_routes import busy_response
+
+try:
+    from ...database.image_removal import mark_hides
+except ImportError:
+    from database.image_removal import mark_hides
+
+NONE_REMOVED = "None of the selected images are removed from the library."
+SOURCE_CHANGED = "The image sources changed. Reload Images and try again."
+
+
+def _bad(message):
+    return web.json_response({"success": False, "error": message}, status=400)
+
+
+class ImageReturnMixin:
+    """Return to library (#498): clear the marks, then scan those files."""
+
+    def _register_image_return_routes(self, routes):
+        # A fixed path beside the dynamic /images/{image_id} routes.
+        @routes.post("/prompt_manager/images/removed/return")
+        async def return_removed_images_route(request):
+            return await self.return_removed_images(request)
+
+    async def return_removed_images(self, request):
+        data, error = await _json_body_or_error(request)
+        if error is not None:
+            return error
+        if set(data) == {"filter"}:
+            targets, roots, error = await self._removed_filter_targets(data["filter"])
+        else:
+            _data, targets, roots, error = await self._category_request(
+                request, frozenset(), data=data
+            )
+        if error is not None:
+            return error
+        files = {}
+        for index in dict.fromkeys(t[0] for t in targets):
+            files[index] = await self._get_gallery_files(roots[index])
+        returned, not_removed = await self._run_in_executor(
+            self._hidden_targets, roots, targets, files
+        )
+        if not returned:
+            return _coded(400, "no-images", NONE_REMOVED)
+        try:
+            job = self._start_scan_job(
+                files="all", output_dirs=list(roots), returned=returned
+            )
+        except JobBusy as busy:
+            return busy_response(busy)
+        return web.json_response(
+            {
+                "success": True,
+                "job": job.summary(),
+                "returned": len(returned),
+                "not_removed": not_removed,
+            },
+            status=202,
+        )
+
+    async def _removed_filter_targets(self, raw):
+        """``(targets, roots, None)`` for the removed view's filter, else an error."""
+        if not isinstance(raw, dict) or raw.get("removed") != "1":
+            return None, None, _bad('filter must hold removed: "1"')
+        rest = {key: value for key, value in raw.items() if key != "removed"}
+        try:
+            parse_listing_filter(rest)
+            selection = await self.resolve_listing_filter(
+                {**rest, "removed": "1"}, MAX_JOB_TARGETS
+            )
+        except ListingSourceChanged:
+            return None, None, _coded(409, "source-changed", SOURCE_CHANGED)
+        except ValueError as exc:
+            return None, None, _bad(str(exc))
+        if selection.truncated:
+            return None, None, _coded(400, "too-many-images", TOO_MANY)
+        if not selection.targets:
+            return None, None, _coded(400, "no-images", NONE_REMOVED)
+        return tuple(selection.targets), list(self._get_all_output_dirs()), None
+
+    def _hidden_targets(self, roots, targets, files):
+        """``([(path, locator)], not_removed)``: the targets a mark hides now.
+
+        Each target is matched to its listed spelling (the walk's). One
+        physical file named twice (nested roots) is returned once; a target
+        that is not listed or not hidden counts in ``not_removed``.
+        """
+        marks = self._removal_marks()
+        listings, seen, returned, not_removed = {}, set(), [], 0
+        for index, _scope, relative in targets:
+            root = roots[index]
+            if index not in listings:
+                listings[index] = self._favorite_listing(root, files[index])
+            spelled = listings[index].get(_favorite_relative_key(relative))
+            if spelled is None:
+                not_removed += 1
+                continue
+            path = os.path.join(str(root), *spelled.split("/"))
+            locator = os.path.normcase(os.path.realpath(path))
+            if locator in seen:
+                continue
+            if not mark_hides(marks, locator, path):
+                not_removed += 1
+                continue
+            seen.add(locator)
+            returned.append((Path(path), locator))
+        return returned, not_removed
+
+    def _clear_returned_marks(self, returned) -> list:
+        """The paths whose marks this deleted (blocking; one transaction)."""
+        cleared = self.db.clear_hiding_marks(
+            [(locator, str(path)) for path, locator in returned]
+        )
+        return [path for path, locator in returned if locator in cleared]

py/api/images.py

@@ -1160,7 +1160,10 @@ def _workspace_filters(query: Mapping) -> dict:
     the linked positive, and ``favorites=1`` keeps only favorites (the
     Favorites page, #249 D51). ``collection=<id>`` (1 to 9 digits) keeps only
     that collection's members (#331); it wins over ``favorites`` (a page never
-    sets both). Raises ValueError for any unknown or oversized value.
+    sets both). ``removed=1`` lists only the files a removal mark hides (#498,
+    Show removed images); such a file has no records, so it never combines
+    with ``favorites``, ``collection``, ``category``, ``tags`` or the rating
+    sort. Raises ValueError for any unknown or oversized value.
     """
     text = query.get("q", "").strip()
     folded = text.casefold()
@@ -1175,6 +1178,12 @@ def _workspace_filters(query: Mapping) -> dict:
     )
     favorites = query.get("favorites", "0")
     collection = query.get("collection", "")
+    removed = query.get("removed", "0")
+    if removed not in ("0", "1") or (
+        removed == "1"
+        and (favorites == "1" or collection or category or tags or order == "rating")
+    ):
+        raise ValueError("Invalid gallery filters")
     if (
         favorites not in ("0", "1")
         or (collection != "" and _COLLECTION_ID.fullmatch(collection) is None)
@@ -1202,6 +1211,7 @@ def _workspace_filters(query: Mapping) -> dict:
         "tags": tags,
         "favorites": favorites == "1" and not collection,
         "collection": int(collection) if collection else None,
+        "removed": removed == "1",
     }
 
 
@@ -2073,6 +2083,7 @@ class ImageRoutesMixin:
                         filters["tags"],
                         filters["favorites"],
                         filters["collection"],
+                        filters["removed"],
                     ]
                 ).encode()
             ).hexdigest()
@@ -2102,6 +2113,11 @@ class ImageRoutesMixin:
                     self._select_workspace_page, listings, filters, after, limit, scope
                 )
             )
+            # #498: Show removed images (N), every removed file in the selected
+            # source(s) whatever the filters (the hidden set is cached work).
+            removed_total = len(
+                await self._run_in_executor(self._hidden_entries, listings)
+            )
         except CollectionMissing:
             return collection_not_found()
         except Exception:
@@ -2128,6 +2144,7 @@ class ImageRoutesMixin:
             "limit": limit,
             "has_more": has_more,
             "next_cursor": next_cursor,
+            "removed_total": removed_total,
         }
         if positive_count is _COUNT_OMITTED:
             del result["positive_count"]
@@ -2451,9 +2468,12 @@ class ImageRoutesMixin:
             def key(entry):
                 return _workspace_key(entry, order)
 
-        # #480: files removed from the library stay on disk, hidden here.
+        # #480: files removed from the library stay on disk, hidden here; the
+        # removed view (#498) lists only them.
         hidden = self._hidden_entries(listings)
-        if hidden:
+        if filters.get("removed"):
+            entries = [entry for entry in entries if entry.normalized in hidden]
+        elif hidden:
             entries = [entry for entry in entries if entry.normalized not in hidden]
         return (
             entries,