From 2f8b7cb3069c76e248ea8fabb4671dd7d1b9494d Mon Sep 17 00:00:00 2001 From: Vito Date: Tue, 27 Oct 2026 22:20:27 +0000 Subject: [PATCH 01/11] feat(images): select many images: ranges, keys, touch and all N matching (#392) (#393) The workspace Images page selects like a file manager in Grid, List and Masonry, across every loaded batch: Shift+click ranges from the anchor, Ctrl/Cmd+click toggles a card without opening it, Ctrl/Cmd+A selects the loaded images, Shift+arrows extend, Escape clears, and a touch long-press starts a selection mode in which taps toggle. A plain click still opens or inspects. Once every loaded image is checked and more match, the bar offers "Select all N matching". Add tags, Auto-tag selected and Set category then send the listing filter instead of N identities: the image-tags job takes `filter` in place of `targets` and /bulk/category takes `images_filter`, both resolved by the listing's own admission code (up to 10,000 images, a Source bound by its root_scope). Export metadata waits for loaded images. Recorded as deviation D135; the concept is unchanged. --- TESTING.md | 62 ++++- py/api/image_tag_jobs.py | 85 +++++- py/api/images.py | 239 +++++++++++++---- py/api/prompts.py | 62 +++++ tests/browser/workspace-images.spec.mjs | 245 ++++++++++++++++++ tests/js/workspace-images-multiselect.test.js | 110 ++++++++ tests/js/workspace-images-state.test.js | 8 + .../js/workspace-images-tagging-bulk.test.js | 26 ++ tests/test_api_workspace_images.py | 159 ++++++++++++ tests/test_image_tag_jobs.py | 106 ++++++++ tools/vendor/frontend-coverage-baseline.json | 6 + web/js/workspace/images-multiselect.mjs | 80 ++++++ web/js/workspace/images-selection.mjs | 194 ++++++++++++-- web/js/workspace/images-state.mjs | 10 + web/js/workspace/images-tagging-bulk.mjs | 41 ++- web/js/workspace/images.mjs | 29 ++- web/js/workspace/workspace.css | 2 + 17 files changed, 1371 insertions(+), 93 deletions(-) create mode 100644 tests/js/workspace-images-multiselect.test.js create mode 100644 web/js/workspace/images-multiselect.mjs diff --git a/TESTING.md b/TESTING.md index 0999efb1..6b05b1e1 100644 --- a/TESTING.md +++ b/TESTING.md @@ -1245,7 +1245,8 @@ Recorded deviations on this page (spec §8): `#bulkbar` shows "N selected" and only the real actions in the concept's order (Add tags, then D85's Auto-tag selected, Set category, Export metadata, Clear). Add to collection waits for collections; "Remove links…" is dropped. (Amended by #249 PR D: - Add tags ships, see "Bulk tagging from the selection bar".) The selection survives Load more and + Add tags ships, see "Bulk tagging from the selection bar"; #392 adds range, keyboard, + touch and "all N matching" selection, see "Selecting many images", D135.) The selection survives Load more and bounded re-mounting and is cleared whenever the collection is replaced (search, category, sort, Filters, source change, rescan, reload). A return to Images restores the saved collection unreplaced and keeps its selection; a full page reload starts @@ -1722,6 +1723,65 @@ busy". Units: | Copy without a concept line: no tags, start fallbacks, the panel heading | D118 | | Activity item and job view for image tagging jobs | D89 (plan D66) | +### Selecting many images (#392) + +**D135** (beyond the concept, which draws only the per-card checkbox): Images selects +like a file manager, in Grid, List (where the concept hides the checkbox) and Masonry, +over every loaded batch. The selection is a set of image keys in display order, so +ranges include cards that bounded rendering has unmounted. + +- Clicking a checkbox toggles it, and that image becomes the anchor. Ctrl/Cmd+click + anywhere on a card (image, star, Info, Tag) also toggles it and makes it the anchor; + nothing opens, inspects or changes the favorite. +- Shift+click on a card selects the range from the anchor to it. Shift+click on a + checkbox gives its range the box's new state, so it can clear a range. The anchor + stays where it is. Without an anchor, the clicked image becomes the anchor. +- In the grid, Ctrl/Cmd+A selects every loaded image; in the search box it keeps its + text meaning. Shift+Arrow, Shift+Home and Shift+End move focus as the plain keys do, + then select from the anchor (the starting card when there is none) to the new card. + Escape in the grid clears the selection. Escape in the bar clears it too, and focus + moves to the heading, as after Clear. +- Touch: a 500 ms long-press (at most 10 px of movement) selects the card and turns on + touch selection mode. The context menu during the press and the click after it are + swallowed, and the card has no image callout. Taps then toggle cards until the + selection is empty. A keyboard Enter or Space keeps its plain meaning. +- A plain click keeps its meaning: the image opens the viewer, and Info inspects. +- When every loaded image is checked and the listing has more, the bar offers + **Select all N matching** right after the count. Pressing it gives **All N matching + selected**: later batches arrive checked, and unchecking any image goes back to the + loaded images minus that one. Clear, Escape, and a new search, filter, sort or rescan + end it. A refresh after a tag save keeps it. Return visits keep the loaded images, + and the offer shows again. +- In that mode, Add tags and Auto-tag selected start the `image-tags` job with + `filter` (the listing's own query as an object: `q`, `subfolder`, `category`, + `tags`, `root`, `media`, `sort`, `favorites`, and with a Source the loaded root's + `root_scope`) instead of `targets`. The server resolves the filter at start, in + listing order, using the listing's own admission code (`_admitted_entries`). That + means "all matching" when the action starts: an output written since then that + matches is included. A `root_scope` that no longer names its root (the sources were + renumbered) answers 409 `source-changed`. The limit is 10,000 images (`MAX_LISTING_SELECTION`). + Over the limit, the client toasts "Narrow the view to at most 10,000 images." and the + server answers 400 `too-many-images`. No match is 400 `no-images`. Auto-tag in this + mode does not offer Review each image, because the review walks images in the + browser. Set category posts `{images_filter, category}` to `/prompt_manager/bulk/category`. + The server sets the category on the distinct linked positives and answers + `prompt_count` and `skipped`. A listing whose `positive_count` is 0 toasts the + nothing-linked sentence and opens no dialog, and over 10,000 matches it toasts the + limit sentence. Export metadata reads every image in + the browser, so it is `aria-disabled` in this mode, and its `title` says why. + +Tests: `tests/browser/workspace-images.spec.mjs` ("Shift+click selects a range across +an appended batch in Grid, List and Masonry", "Ctrl/Cmd+click toggles a card without +opening it, and a plain click still opens the viewer", "Ctrl/Cmd+A, Select all N +matching and Add tags tag every matching image on the server", "Shift+arrow keys extend +the selection from the focused card, and Escape in the bar clears it", "a touch +long-press starts selection mode, taps then toggle, and an empty selection ends it"); +units `tests/js/workspace-images-multiselect.test.js`, +`tests/js/workspace-images-tagging-bulk.test.js` ("all matching sends the listing +filter instead of targets"), `tests/js/workspace-images-state.test.js` +(`listingFilter`); Python `tests/test_image_tag_jobs.py` (`TestFilterStart`) and +`tests/test_api_workspace_images.py` (`TestListingFilterSelection`). + Native image tagging (#249 PR B routes, PR C UI): `POST /prompt_manager/image-annotations/observe` (capture one listed file, its saved set and `tag_revision`), `GET`/`DELETE /prompt_manager/image-annotations/capture/` diff --git a/py/api/image_tag_jobs.py b/py/api/image_tag_jobs.py index 09aeb874..1d611745 100644 --- a/py/api/image_tag_jobs.py +++ b/py/api/image_tag_jobs.py @@ -2,7 +2,9 @@ ``POST /prompt_manager/jobs/image-tags`` starts one ``image-tags`` job over 1..500 listed images, named by the listing identity ``{root_index, -root_scope, path}``. The job is record-changing (it writes image tag sets and +root_scope, path}``, or over every image a listing ``filter`` admits (at most +10,000, resolved at start in the listing's order; "Select all N matching", +#392). The job is record-changing (it writes image tag sets and new vocabulary names), so it never runs beside the metadata scan. - **Add tags** (``mode: "add"``) hashes each current file and unions a fixed @@ -54,7 +56,14 @@ from .annotations import ( _target, ) from .autotag_routes import BATCH_CANCELLED, MODEL_BUSY_ERROR, model_busy_response -from .images import _current_fingerprint, _listing_posix, _public_path +from .images import ( + MAX_LISTING_SELECTION, + ListingSourceChanged, + _current_fingerprint, + _listing_posix, + _public_path, + parse_listing_filter, +) from .job_routes import busy_response from .jobs import ( AUTOTAG_BATCH_JOB_KIND, @@ -90,6 +99,11 @@ except ImportError: from utils.video_metadata import is_video_path MAX_JOB_TARGETS = 500 +# A filter start resolves on the server, so the request body does not bound it. +MAX_FILTER_TARGETS = MAX_LISTING_SELECTION +NO_IMAGES = "No images match this view." +FILTER_LISTING_FAILED = "The matching images could not be listed. Try again." +TOO_MANY_IMAGES = "Narrow the view to at most {cap:,} images." # 500 maximal targets plus the options: a path of MAX_TARGET_PATH code points # takes at most 6 bytes each on the wire (a \uXXXX escape; raw UTF-8 is at most # 4), and the root index and scope fit in 256. The annotation routes' 64 KiB @@ -103,9 +117,10 @@ ADD_TITLE = "Add tags to images" AUTO_TITLE = "Auto-tag images" GATE_OWNER_KIND = "image-tag-job" +# Each mode's fields besides the images, named by ``targets`` or ``filter``. _FIELDS = { - "add": frozenset({"mode", "targets", "tags"}), - "auto": frozenset({"mode", "targets", "model_type", "options"}), + "add": frozenset({"mode", "tags"}), + "auto": frozenset({"mode", "model_type", "options"}), } _VERBS = {"add": "Add tags", "auto": "Auto-tag"} _LOG_PREFIXES = {"tagged": "+", "unchanged": "=", "skipped": "-", "errors": "!"} @@ -164,6 +179,8 @@ class _Spec: tags: tuple = () model_type: str | None = None options: dict | None = None + # A listing filter the start resolves into ``targets`` (#392). + filter: dict | None = None @property def title(self) -> str: @@ -222,21 +239,35 @@ def parse_image_tag_request(data: dict): """A :class:`_Spec` for a start body, or ``(None, error response)``. ``{mode:"add", targets, tags}`` or ``{mode:"auto", targets, model_type, - options}`` exactly; options follow the Tag dialog's Generate rules. + options}`` exactly; options follow the Tag dialog's Generate rules. A + ``filter`` (a listing query object, parse_listing_filter) may stand in for + ``targets``; the start resolves it. """ fields = _FIELDS.get(data.get("mode")) if fields is None: return None, bad_request('mode must be "add" or "auto"') + fields = fields | {"filter" if "filter" in data else "targets"} if set(data) != fields: return None, bad_request( "Request must contain exactly " + ", ".join(sorted(fields)) ) - targets, error = _parse_targets(data["targets"]) - if error is not None: - return None, error + listing_filter, targets = None, () + if "filter" in data: + try: + listing_filter = parse_listing_filter(data["filter"]) + except ValueError as exc: + return None, bad_request(str(exc)) + else: + targets, error = _parse_targets(data["targets"]) + if error is not None: + return None, error if data["mode"] == "add": - return _parse_add(targets, data) - return _parse_auto(targets, data) + spec, error = _parse_add(targets, data) + else: + spec, error = _parse_auto(targets, data) + if spec is not None and listing_filter is not None: + spec = dataclasses.replace(spec, filter=listing_filter) + return spec, error def _capture_outcome(exc: CaptureError, root, relative: str): @@ -398,6 +429,10 @@ class ImageTagJobsMixin: spec, error = parse_image_tag_request(data) if error is not None: return error + if spec.filter is not None: + spec, error = await self._resolve_job_filter(spec) + if error is not None: + return error roots = await self._run_in_executor(self._job_roots, spec.targets) if any(root is None for root, _label in roots.values()): return _coded("source-changed") @@ -639,6 +674,36 @@ class ImageTagJobsMixin: job.log_line(f"Skipped {_plural(skipped, 'image')} before tagging") return remaining + async def _resolve_job_filter(self, spec: _Spec): + """``spec`` with its filter resolved into targets, or ``(None, error)``. + + The listing's own identities, in its order, kept only where a target + could have named them (a media path the job can tag). + """ + cap = MAX_FILTER_TARGETS + try: + selection = await self.resolve_listing_filter(spec.filter, cap) + except ListingSourceChanged: + return None, _coded("source-changed") + except ValueError as exc: + return None, bad_request(str(exc)) + except Exception: + # Never echo it: it can carry SQL or database text. + self.logger.error("A tagging filter could not be resolved", exc_info=True) + return None, _error(500, "listing-failed", FILTER_LISTING_FAILED) + if selection.truncated: + return None, _error(400, "too-many-images", TOO_MANY_IMAGES.format(cap=cap)) + targets = [] + for index, scope, relative in selection.targets: + target, error = _target( + {"root_index": index, "root_scope": scope, "path": relative} + ) + if error is None: + targets.append(target) + if not targets: + return None, _error(400, "no-images", NO_IMAGES) + return dataclasses.replace(spec, targets=tuple(targets)), None + # -- runner ---------------------------------------------------------------- def _job_roots(self, targets) -> dict: diff --git a/py/api/images.py b/py/api/images.py index 95d5e2ec..439a2800 100644 --- a/py/api/images.py +++ b/py/api/images.py @@ -1099,37 +1099,94 @@ def _workspace_filters(query: Mapping) -> dict: } +# The listing parameters a bulk request may name as its filter (#392): the +# workspace query without mode, limit, cursor and the Activity-only ``path``, +# plus the chosen root's ``root_scope`` (a renumbered source is refused). +LISTING_FILTER_KEYS = frozenset( + { + "q", + "subfolder", + "category", + "tags", + "root", + "root_scope", + "media", + "sort", + "favorites", + } +) +MAX_LISTING_FILTER_VALUE = 2048 +# The most images "Select all N matching" acts on in one request (#392). +MAX_LISTING_SELECTION = 10_000 + + +def parse_listing_filter(raw: object) -> dict: + """A bulk request's listing filter (#392): the listing's own query as an object. + + Keys are a subset of LISTING_FILTER_KEYS, each value a string of at most + MAX_LISTING_FILTER_VALUE characters, validated as the listing validates its + query. The root's bounds need the current roots, so resolve_listing_filter + checks those. Raises ValueError. + """ + if not isinstance(raw, dict) or not set(raw) <= LISTING_FILTER_KEYS: + raise ValueError("filter must be an object of listing parameters") + if any( + not isinstance(value, str) or len(value) > MAX_LISTING_FILTER_VALUE + for value in raw.values() + ): + raise ValueError("filter values must be short strings") + if "root_scope" in raw and "root" not in raw: + raise ValueError("root_scope needs root") + _workspace_filters(raw) + return dict(raw) + + +class ListingSourceChanged(Exception): + """A listing filter's ``root_scope`` no longer names its root (#392).""" + + +class ListingSelection(NamedTuple): + """Every image a listing filter admits, in the listing's order (#392). + + ``targets`` are listing identities ``(root_index, root_scope, path)``; + ``positive_ids`` the distinct linked positives (first seen), ``unlinked`` + the admitted images without one, and ``truncated`` that more than the cap + matched (the lists then stop at the cap). + """ + + targets: list + positive_ids: list + unlinked: int + truncated: bool + + # A plain listing's cursor page leaves positive_count out (see # _select_workspace_page); None instead means the count could not be read. _COUNT_OMITTED = object() -def _filtered_page( +def _admitted_entries( entries: list, - start: int, - size: int, filters: dict, links: dict | None, favorites: frozenset | None = None, image_tags: dict | None = None, -) -> tuple[int, list, int | None]: - """Count the *entries* the filters admit and take *size* from *start* on. +): + """Yield ``(position, entry, link match)`` for each entry the filters admit. *links* maps normalized paths to (hit, rating, positive id) for every unambiguously linked image (see PromptDatabase.get_image_link_matches); a category or tag filter admits only those, and ``q`` admits a filename match - or a linked-positive hit. Returns the total, the page and the number of - distinct linked positives among all admitted entries (None without links). - *favorites*, when given, admits only those listed paths (``favorites=1``). - *image_tags* maps normalized paths to their _ImageTagMatch: a tag filter - also admits a file whose current image tags (with its linked positive's - tags) carry every tag, and ``q`` also admits an image tag name hit. + or a linked-positive hit. *favorites*, when given, admits only those listed + paths (``favorites=1``). *image_tags* maps normalized paths to their + _ImageTagMatch: a tag filter also admits a file whose current image tags + (with its linked positive's tags) carry every tag, and ``q`` also admits an + image tag name hit. """ query, media, folder = filters["query"], filters["media"], filters["folder"] members_only = filters["category"] is not None or bool(filters["tags"]) native_folder = folder.replace("/", os.sep) nested = native_folder + os.sep - total, page, positives = 0, [], set() exact = filters.get("path", "") for position, entry in enumerate(entries): if exact and entry.folded != exact: @@ -1153,6 +1210,28 @@ def _filtered_page( and not (image and image.query) ): continue + yield position, entry, match + + +def _filtered_page( + entries: list, + start: int, + size: int, + filters: dict, + links: dict | None, + favorites: frozenset | None = None, + image_tags: dict | None = None, +) -> tuple[int, list, int | None]: + """Count the *entries* the filters admit and take *size* from *start* on. + + Admission is _admitted_entries'. Returns the total, the page and the number + of distinct linked positives among all admitted entries (None without + links). + """ + total, page, positives = 0, [], set() + for position, entry, match in _admitted_entries( + entries, filters, links, favorites, image_tags + ): total += 1 if match is not None: positives.add(match[2]) @@ -2131,35 +2210,23 @@ class ImageRoutesMixin: ) return matches - def _select_workspace_page( - self, - listings: list[tuple], - filters: dict, - after: tuple | None, - limit: int, - scope: str, + def _workspace_admission( + self, listings: list[tuple], filters: dict, *, read_links: bool, strict=False ) -> tuple: - """One keyset page of the filtered listing (blocking; run in executor). + """The ordered entries and what admits them (blocking; run in executor). - One SELECT (get_image_link_matches) reads every linked positive of the - listing: the filters, the rating sort and the positive count share it. - A plain listing (no ``q``, category, tags or rating sort) reads it only - for its first page; its cursor pages read no links and the returned - count is _COUNT_OMITTED, so the response leaves ``positive_count`` out - and the frontend keeps the first page's value. On that plain first - page a failed read leaves the listing usable and the count unknown - (None); so does a failed read under ``q`` alone, whose search then - matches filenames only (as before the linked search existed). + Returns ``(entries, key, links, image_tags, favorites)``: the listing in + the filters' order, its keyset key, and the link, image-tag and favorite + maps _admitted_entries reads. One SELECT (get_image_link_matches) reads + every linked positive when *read_links*. A failed read raises when the + filters need it (category, tags, the rating sort) or *strict*; else the + listing stays usable without links (a search then matches filenames + only). A failed image-tag read raises under a Tags filter; a search then + matches without image tags. """ order, category, tags = filters["order"], filters["category"], filters["tags"] - needed = ( - bool(filters["query"]) - or category is not None - or bool(tags) - or order == "rating" - ) links = None - if needed or after is None: + if read_links: try: links = self.db.get_image_link_matches( filters["text"], @@ -2169,10 +2236,7 @@ class ImageRoutesMixin: wanted=self._workspace_wanted(listings), ) except Exception: - # Category, tags and the rating sort cannot be served without - # the links. A plain listing and a filename search can: the - # search then matches filenames only, and the count is None. - if category is not None or tags or order == "rating": + if strict or category is not None or tags or order == "rating": raise self.logger.warning( "Workspace prompt links could not be read; %s", @@ -2188,8 +2252,6 @@ class ImageRoutesMixin: try: image_tags = self._image_tag_matches(listings, filters) except Exception: - # A Tags filter cannot be served without them (as without the - # links); a search then matches without image tags. if tags: raise self.logger.warning( @@ -2228,6 +2290,41 @@ class ImageRoutesMixin: def key(entry): return _workspace_key(entry, order) + favorites = ( + self._workspace_favorite_paths(listings) if filters["favorites"] else None + ) + return entries, key, links, image_tags, favorites + + def _select_workspace_page( + self, + listings: list[tuple], + filters: dict, + after: tuple | None, + limit: int, + scope: str, + ) -> tuple: + """One keyset page of the filtered listing (blocking; run in executor). + + The filters, the rating sort and the positive count share one link + read (_workspace_admission). A plain listing (no ``q``, category, tags + or rating sort) reads it only for its first page; its cursor pages read + no links and the returned count is _COUNT_OMITTED, so the response + leaves ``positive_count`` out and the frontend keeps the first page's + value. On that plain first page a failed read leaves the listing usable + and the count unknown (None); so does a failed read under ``q`` alone, + whose search then matches filenames only (as before the linked search + existed). + """ + order = filters["order"] + needed = ( + bool(filters["query"]) + or filters["category"] is not None + or bool(filters["tags"]) + or order == "rating" + ) + entries, key, links, image_tags, favorites = self._workspace_admission( + listings, filters, read_links=needed or after is None + ) start = 0 if after is not None: if order == "name-desc": @@ -2235,9 +2332,6 @@ class ImageRoutesMixin: start = len(entries) - bisect_left(ascending, after, key=key) else: start = bisect_right(entries, after, key=key) - favorites = ( - self._workspace_favorite_paths(listings) if filters["favorites"] else None - ) total, page, positive_count = _filtered_page( entries, start, limit + 1, filters, links, favorites, image_tags ) @@ -2254,6 +2348,61 @@ class ImageRoutesMixin: positive_count, ) + def _select_workspace_matches( + self, listings: list[tuple], filters: dict, cap: int + ) -> ListingSelection: + """Every admitted entry, up to *cap*, as a ListingSelection (blocking). + + The links are read strictly: the positives must be the listing's own. + """ + entries, _key, links, image_tags, favorites = self._workspace_admission( + listings, filters, read_links=True, strict=True + ) + targets, positives, unlinked, scopes = [], {}, 0, {} + for _position, entry, match in _admitted_entries( + entries, filters, links, favorites, image_tags + ): + if len(targets) == cap: + return ListingSelection(targets, list(positives), unlinked, True) + scope = scopes.get(entry.index) + if scope is None: + scope = scopes[entry.index] = self._gallery_root_scope( + entry.root, entry.index + ) + relative = entry.path.relative_to(entry.root).as_posix() + targets.append((entry.index, scope, relative)) + if match is None or match[2] is None: + unlinked += 1 + else: + positives.setdefault(match[2]) + return ListingSelection(targets, list(positives), unlinked, False) + + async def resolve_listing_filter( + self, params: Mapping, cap: int = MAX_LISTING_SELECTION + ) -> ListingSelection: + """The images a parse_listing_filter result admits right now (#392). + + Raises ValueError for a root outside the current roots, and + ListingSourceChanged when the filter's ``root_scope`` no longer names + its root (the sources were renumbered since the listing). + """ + output_dirs = list(self._get_all_output_dirs()) + root_index = _workspace_root(params, output_dirs) + error = self._gallery_scope_error(params, output_dirs) + if error is not None: + if error.status == 409: + raise ListingSourceChanged() + raise ValueError("Invalid gallery root scope") + filters = _workspace_filters(params) + listings = [ + (index, root, await self._get_gallery_files(root)) + for index, root in enumerate(output_dirs) + if root_index is None or index == root_index + ] + return await self._run_in_executor( + self._select_workspace_matches, listings, filters, cap + ) + async def serve_image(self, request): """Serve the actual image file using streamed FileResponse.""" try: diff --git a/py/api/prompts.py b/py/api/prompts.py index 22b2b3dc..e48296ea 100644 --- a/py/api/prompts.py +++ b/py/api/prompts.py @@ -1178,6 +1178,8 @@ class PromptRoutesMixin: return web.json_response( {"success": False, "error": "Expected a JSON object"}, status=400 ) + if "images_filter" in data: + return await self._bulk_category_by_filter(data) if not data.get("prompt_ids"): return web.json_response( {"success": False, "error": "No prompt IDs provided"}, status=400 @@ -1214,6 +1216,66 @@ class PromptRoutesMixin: status=500, ) + async def _bulk_category_by_filter(self, data: dict) -> web.Response: + """Set the category on every linked positive a listing filter admits. + + #392 "Select all N matching": ``{images_filter, category}``, the filter + being the Images listing's own query (parse_listing_filter), resolved + now. The answer adds ``prompt_count`` (the distinct linked positives) + and ``skipped`` (matching images without one); none linked changes + nothing. + """ + from .images import ( + MAX_LISTING_SELECTION, + ListingSourceChanged, + parse_listing_filter, + ) + + def refused(message, **extra): + return web.json_response( + {"success": False, "error": message, **extra}, status=400 + ) + + if set(data) != {"images_filter", "category"}: + return refused("Request must contain exactly category, images_filter") + category = data["category"] + try: + params = parse_listing_filter(data["images_filter"]) + validate_category(category) + selection = await self.resolve_listing_filter(params) + except ListingSourceChanged: + return web.json_response( + { + "success": False, + "code": "source-changed", + "error": "The source changed. Refresh the image collection.", + }, + status=409, + ) + except ValueError as exc: + return refused(str(exc)) + if selection.truncated: + return refused( + f"Narrow the view to at most {MAX_LISTING_SELECTION:,} images.", + code="too-many-images", + ) + category = (category or "").strip() + ids = selection.positive_ids + updated = ( + await self._run_in_executor(self.db.bulk_set_category, ids, category) + if ids + else 0 + ) + return web.json_response( + { + "success": True, + "message": f"Set category for {updated} prompts", + "updated_count": updated, + "prompt_count": len(ids), + "skipped": selection.unlinked, + } + ) + def _all_prompts_for_export(self): """Every prompt, fetched in EXPORT_PAGE_SIZE pages (blocking).""" prompts, offset = [], 0 diff --git a/tests/browser/workspace-images.spec.mjs b/tests/browser/workspace-images.spec.mjs index 292d8891..f088e149 100644 --- a/tests/browser/workspace-images.spec.mjs +++ b/tests/browser/workspace-images.spec.mjs @@ -2189,3 +2189,248 @@ test('scan for metadata opens the review dialog from Import', async ({ page, req await expect.poll(async () => (await (await request.get('/prompt_manager/jobs')).json()).jobs .map(job => [job.kind, job.state])).toEqual([['backup', 'finished']]); }); + +// #392 (D135): file-manager selection beyond the concept's checkbox. +const selectedCount = page => bulkbar(page).locator('.bulkbar > strong'); +const cardAt = (page, index) => page.locator(`article[data-image-key][data-index="${index}"]`); +const boxAt = (page, index) => cardAt(page, index).getByRole('checkbox', { name: /^Select / }); +const checkedIndexes = page => page.locator('article[data-image-key]').evaluateAll(nodes => nodes + .filter(node => node.querySelector('.image-overlays input[type="checkbox"]').checked) + .map(node => Number(node.dataset.index)).sort((a, b) => a - b)); +const range = (from, to) => Array.from({ length: to - from + 1 }, (_, index) => from + index); +async function bulkReset(page, request) { + await page.addInitScript(() => { window.IntersectionObserver = undefined; }); + await expect(await request.post('/__test__/reset', { data: { extra_images: 200 } })).toBeOK(); + await page.goto(workspace); + await expect(cards(page)).toHaveCount(40); +} +/** A touch press on `locator` the way a finger makes one: pointerdown, (held), pointerup. */ +const touchDown = locator => locator.evaluate(node => { + const { left, top } = node.getBoundingClientRect(); + node.dispatchEvent(new PointerEvent('pointerdown', { bubbles: true, isPrimary: true, pointerType: 'touch', clientX: left + 5, clientY: top + 5 })); +}); +const touchUpAndClick = locator => locator.evaluate(node => { + node.dispatchEvent(new PointerEvent('pointerup', { bubbles: true, isPrimary: true, pointerType: 'touch' })); + // A tap's click counts one press (`detail` 1); a keyboard click (and `HTMLElement.click()`) counts none. + node.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, detail: 1 })); +}); +async function touchTap(locator) { + await touchDown(locator); + await touchUpAndClick(locator); +} + +test('Shift+click selects a range across an appended batch in Grid, List and Masonry', async ({ page, request }) => { + await bulkReset(page, request); + await page.getByRole('button', { name: 'Load more images', exact: true }).click(); + await expect(caption(page)).toContainText('80 of 202 images'); + for (const mode of ['Grid', 'List', 'Masonry']) { + await page.getByRole('group', { name: 'Image layout', exact: true }).getByRole('button', { name: mode, exact: true }).click(); + // The anchor is the last image toggled on its own (List hides the concept's checkbox: the card carries it). + await cardAt(page, 3).locator('.media-info h3').click({ modifiers: ['ControlOrMeta'] }); + await expect(selectedCount(page)).toHaveText('1 selected'); + // The range runs into the second batch (index 40 on), and backwards from the same anchor. + await cardAt(page, 45).locator('.media-info h3').click({ modifiers: ['Shift'] }); + await expect(selectedCount(page)).toHaveText('43 selected'); + expect(await checkedIndexes(page)).toEqual(range(3, 45)); + await cardAt(page, 0).locator('.media-info h3').click({ modifiers: ['Shift'] }); + await expect(selectedCount(page)).toHaveText('46 selected'); + expect(await checkedIndexes(page)).toEqual(range(0, 45)); + await expect(page.locator('dialog[open]')).toHaveCount(0); + if (mode !== 'List') { + // A Shift+click on a checkbox gives its range the box's new state: here, unchecked. + await boxAt(page, 3).uncheck(); + await boxAt(page, 10).click({ modifiers: ['Shift'] }); + await expect(boxAt(page, 10)).not.toBeChecked(); + await expect(selectedCount(page)).toHaveText('38 selected'); + expect(await checkedIndexes(page)).toEqual([0, 1, 2, ...range(11, 45)]); + } + await bulkbar(page).getByRole('button', { name: 'Clear', exact: true }).click(); + await expect(bulkbar(page)).toBeEmpty(); + } +}); + +test('Ctrl/Cmd+click toggles a card without opening it, and a plain click still opens the viewer', async ({ page }) => { + await page.goto(workspace); + await expect(cards(page)).toHaveCount(2); + const first = cards(page).first(); + const inspectorBefore = await details(page).innerText(); + await first.getByRole('button', { name: 'Open same.png', exact: true }).click({ modifiers: ['ControlOrMeta'] }); + await expect(selectedCount(page)).toHaveText('1 selected'); + await expect(checkbox(first, 'same.png')).toBeChecked(); + await cards(page).last().getByRole('button', { name: 'Inspect same.png', exact: true }).click({ modifiers: ['ControlOrMeta'] }); + await expect(selectedCount(page)).toHaveText('2 selected'); + await star(first, 'same.png').click({ modifiers: ['ControlOrMeta'] }); + await expect(selectedCount(page)).toHaveText('1 selected'); + await expect(star(first, 'same.png')).toHaveAttribute('aria-pressed', 'false'); + await expect(page.locator('dialog[open]')).toHaveCount(0); + expect(await details(page).innerText()).toBe(inspectorBefore); + // A plain click keeps its meaning: the image opens the viewer. + await first.getByRole('button', { name: 'Open same.png', exact: true }).click(); + await expect(page.locator('#viewer-dialog[open]')).toHaveCount(1); + await expect(selectedCount(page)).toHaveText('1 selected'); +}); + +test('Ctrl/Cmd+A, Select all N matching and Add tags tag every matching image on the server', async ({ page, request }) => { + await bulkReset(page, request); + // Ctrl/Cmd+A in the search box stays the field's own. + await search(page).focus(); + await page.keyboard.press('ControlOrMeta+a'); + await expect(bulkbar(page)).toBeEmpty(); + await cardAt(page, 0).getByRole('button', { name: /^Open / }).focus(); + await page.keyboard.press('ControlOrMeta+a'); + await expect(selectedCount(page)).toHaveText('40 selected'); + const offer = bulkbar(page).getByRole('button', { name: 'Select all 202 matching', exact: true }); + await expect(offer).toBeVisible(); + await expect(bulkbar(page).locator('.bulkbar > *').nth(1)).toHaveText('Select all 202 matching'); + await offer.click(); + await expect(selectedCount(page)).toHaveText('All 202 matching selected'); + await expect(offer).toHaveCount(0); + await expect(bulkbar(page).getByRole('button', { name: 'Export metadata', exact: true })).toHaveAttribute('aria-disabled', 'true'); + // A later batch arrives checked. + await page.getByRole('button', { name: 'Load more images', exact: true }).click(); + await expect(caption(page)).toContainText('80 of 202 images'); + await expect(boxAt(page, 70)).toBeChecked(); + await expect(selectedCount(page)).toHaveText('All 202 matching selected'); + // Unchecking one image leaves "all matching": the other loaded images stay selected. + await boxAt(page, 0).uncheck(); + await expect(selectedCount(page)).toHaveText('79 selected'); + await expect(bulkbar(page).getByRole('button', { name: 'Export metadata', exact: true })).not.toHaveAttribute('aria-disabled', 'true'); + await cardAt(page, 1).getByRole('button', { name: /^Open / }).focus(); + await page.keyboard.press('ControlOrMeta+a'); + await bulkbar(page).getByRole('button', { name: 'Select all 202 matching', exact: true }).click(); + await expect(selectedCount(page)).toHaveText('All 202 matching selected'); + // Add tags sends the listing filter, not 202 identities, and the server tags every match. + await bulkbar(page).getByRole('button', { name: 'Add tags', exact: true }).click(); + const dialog = page.getByRole('dialog', { name: 'Tag selected images', exact: true }); + await dialog.getByRole('textbox', { name: 'Tags, separated by commas', exact: true }).fill('picked392'); + const started = page.waitForRequest(item => item.method() === 'POST' && new URL(item.url()).pathname === '/prompt_manager/jobs/image-tags'); + await dialog.getByRole('button', { name: 'Apply', exact: true }).click(); + expect((await started).postDataJSON()).toEqual({ mode: 'add', filter: { sort: 'newest' }, tags: ['picked392'] }); + await expect.poll(async () => (await (await request.get('/prompt_manager/images/output?mode=workspace&limit=1&tags=picked392')).json()).total, + { timeout: 20000 }).toBe(202); + await expect(selectedCount(page)).toHaveText('All 202 matching selected'); + // Escape in the grid clears the selection. + await cardAt(page, 2).getByRole('button', { name: /^Open / }).focus(); + await page.keyboard.press('Escape'); + await expect(bulkbar(page)).toBeEmpty(); + await expect(boxAt(page, 2)).not.toBeChecked(); +}); + +test('Shift+arrow keys extend the selection from the focused card, and Escape in the bar clears it', async ({ page, request }) => { + await bulkReset(page, request); + await cardAt(page, 0).getByRole('button', { name: /^Open / }).focus(); + await page.keyboard.press('Shift+ArrowRight'); + await page.keyboard.press('Shift+ArrowRight'); + await expect(cardAt(page, 2).getByRole('button', { name: /^Open / })).toBeFocused(); + await expect(selectedCount(page)).toHaveText('3 selected'); + expect(await checkedIndexes(page)).toEqual([0, 1, 2]); + // A plain arrow moves without selecting; Shift+End reaches the last loaded card from the same anchor. + await page.keyboard.press('ArrowRight'); + await expect(selectedCount(page)).toHaveText('3 selected'); + await page.keyboard.press('Shift+End'); + await expect(selectedCount(page)).toHaveText('40 selected'); + await bulkbar(page).getByRole('button', { name: 'Clear', exact: true }).focus(); + await page.keyboard.press('Escape'); + await expect(bulkbar(page)).toBeEmpty(); + await expect(page.locator('#workspace-main h1')).toBeFocused(); +}); + +test('a touch long-press starts selection mode, taps then toggle, and an empty selection ends it', async ({ page }, testInfo) => { + await page.goto(workspace); + await expect(cards(page)).toHaveCount(2); + const [first, second] = [cards(page).first(), cards(page).last()]; + const open = card => card.getByRole('button', { name: 'Open same.png', exact: true }); + await touchDown(open(first)); + // The context menu of a held press belongs to the long press. + expect(await open(first).evaluate(node => !node.dispatchEvent(new MouseEvent('contextmenu', { bubbles: true, cancelable: true })))).toBe(true); + // Held: the long press selects the card once its delay passed. + await expect(selectedCount(page)).toHaveText('1 selected'); + await touchUpAndClick(open(first)); + await expect(page.locator('dialog[open]')).toHaveCount(0); + await expect(checkbox(first, 'same.png')).toBeChecked(); + // Taps now toggle instead of opening. + if (testInfo.project.name === 'touch') await open(second).tap(); + else await touchTap(open(second)); + await expect(selectedCount(page)).toHaveText('2 selected'); + // A keyboard Enter keeps its meaning in touch selection mode: the image opens. + await open(first).focus(); + await page.keyboard.press('Enter'); + await expect(page.locator('#viewer-dialog[open]')).toHaveCount(1); + await page.keyboard.press('Escape'); + await expect(page.locator('#viewer-dialog[open]')).toHaveCount(0); + await expect(selectedCount(page)).toHaveText('2 selected'); + await touchTap(open(first)); + await touchTap(open(second)); + await expect(bulkbar(page)).toBeEmpty(); + await expect(page.locator('dialog[open]')).toHaveCount(0); + // The selection is empty, so touch selection mode ended: a tap opens the image again. + await touchTap(open(first)); + await expect(page.locator('#viewer-dialog[open]')).toHaveCount(1); +}); + +test('all matching sets the category through the listing filter and survives a return visit as the loaded images', async ({ page, request }) => { + await bulkReset(page, request); + await cardAt(page, 0).getByRole('button', { name: /^Open / }).focus(); + await page.keyboard.press('ControlOrMeta+a'); + await bulkbar(page).getByRole('button', { name: 'Select all 202 matching', exact: true }).click(); + // The offer left with its press: focus moves to the bar's first action. + await expect(bulkbar(page).getByRole('button', { name: 'Add tags', exact: true })).toBeFocused(); + await cardAt(page, 0).getByRole('button', { name: /^Open / }).focus(); + await page.keyboard.press('ControlOrMeta+a'); + await expect(selectedCount(page)).toHaveText('All 202 matching selected'); + // Export metadata waits for loaded images: a press does nothing. + const exportButton = bulkbar(page).getByRole('button', { name: 'Export metadata', exact: true }); + await expect(exportButton).toHaveAttribute('title', 'Export metadata reads each image in the browser, so it works on loaded images only.'); + // No match linked (a stubbed answer): the nothing-linked sentence, and the selection stays. + await page.route('**/prompt_manager/bulk/category', route => route.fulfill({ json: { success: true, updated_count: 0, prompt_count: 0, skipped: 202 } }), { times: 1 }); + const categorize = page.getByRole('dialog', { name: 'Categorize selected images', exact: true }); + await bulkbar(page).getByRole('button', { name: 'Set category', exact: true }).click(); + await categorize.getByRole('textbox', { name: 'Category', exact: true }).fill('Bulk392'); + await categorize.getByRole('button', { name: 'Apply', exact: true }).click(); + await expect(toast(page)).toHaveText("None of the selected images is linked to a saved prompt, so there is no prompt category to set."); + await expect(selectedCount(page)).toHaveText('All 202 matching selected'); + // The real write: one request with the filter, every matching image's positive changes. + await bulkbar(page).getByRole('button', { name: 'Set category', exact: true }).click(); + await categorize.getByRole('textbox', { name: 'Category', exact: true }).fill('Bulk392'); + const posted = page.waitForRequest(item => item.method() === 'POST' && new URL(item.url()).pathname === '/prompt_manager/bulk/category'); + await categorize.getByRole('button', { name: 'Apply', exact: true }).click(); + expect((await posted).postDataJSON()).toEqual({ images_filter: { sort: 'newest' }, category: 'Bulk392' }); + await expect(toast(page)).toHaveText(/^Category set on \d+ prompts?\.$/); + await expect(bulkbar(page)).toBeEmpty(); + const filtered = await (await request.get('/prompt_manager/images/output?mode=workspace&limit=1&category=Bulk392')).json(); + expect(filtered.total).toBe(202); + // A return visit keeps "all matching" as the loaded images, and the offer shows again. + await cardAt(page, 0).getByRole('button', { name: /^Open / }).focus(); + await page.keyboard.press('ControlOrMeta+a'); + await bulkbar(page).getByRole('button', { name: 'Select all 202 matching', exact: true }).click(); + await page.evaluate(() => { window.location.hash = '/overview'; }); + await expect(page.locator('#workspace-main h1')).toHaveText('Overview'); + await page.evaluate(() => { window.location.hash = '/images'; }); + await expect(selectedCount(page)).toHaveText('40 selected'); + await expect(bulkbar(page).getByRole('button', { name: 'Select all 202 matching', exact: true })).toBeVisible(); + // Escape in the bar with nothing selected does nothing; with a selection it clears. + await bulkbar(page).getByRole('button', { name: 'Clear', exact: true }).focus(); + await page.keyboard.press('Escape'); + await expect(bulkbar(page)).toBeEmpty(); +}); + +test('all matching over 10,000 images says so before any dialog or request', async ({ page, request }) => { + await page.route(url => url.pathname === '/prompt_manager/images/output' && url.searchParams.get('limit') !== '1', async route => { + const response = await route.fetch(); + const body = await response.json(); + await route.fulfill({ response, json: { ...body, total: 20002 } }); + }); + await bulkReset(page, request); + await cardAt(page, 0).getByRole('button', { name: /^Open / }).focus(); + await page.keyboard.press('ControlOrMeta+a'); + await bulkbar(page).getByRole('button', { name: 'Select all 20,002 matching', exact: true }).click(); + await expect(selectedCount(page)).toHaveText('All 20,002 matching selected'); + let posted = 0; + page.on('request', item => { if (item.method() === 'POST') posted += 1; }); + for (const action of ['Set category', 'Add tags', 'Auto-tag selected']) { + await bulkbar(page).getByRole('button', { name: action, exact: true }).click(); + await expect(toast(page)).toHaveText('Narrow the view to at most 10,000 images.'); + await expect(page.locator('dialog[open]')).toHaveCount(0); + } + expect(posted).toBe(0); +}); diff --git a/tests/js/workspace-images-multiselect.test.js b/tests/js/workspace-images-multiselect.test.js new file mode 100644 index 00000000..9b622f9e --- /dev/null +++ b/tests/js/workspace-images-multiselect.test.js @@ -0,0 +1,110 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const source = import('../../web/js/workspace/images-multiselect.mjs'); + +const KEYS = Object.freeze(['0:a', '0:b', '0:c', '0:d', '1:e']); + +test('a range runs from the anchor to the target in display order, either way', async () => { + const { rangeKeys } = await source; + assert.deepEqual(rangeKeys(KEYS, '0:b', '0:d'), ['0:b', '0:c', '0:d']); + assert.deepEqual(rangeKeys(KEYS, '1:e', '0:c'), ['0:c', '0:d', '1:e']); + assert.deepEqual(rangeKeys(KEYS, '0:c', '0:c'), ['0:c']); +}); + +test('a range without a listed anchor is the target alone, and an unlisted target is empty', async () => { + const { rangeKeys } = await source; + assert.deepEqual(rangeKeys(KEYS, null, '0:c'), ['0:c']); + assert.deepEqual(rangeKeys(KEYS, '9:gone', '0:c'), ['0:c']); + assert.deepEqual(rangeKeys(KEYS, '0:a', '9:gone'), []); +}); + +test('applying keys adds or removes them and keeps the same frozen set when nothing changes', async () => { + const { applyKeys } = await source; + const start = Object.freeze(new Set(['0:a'])); + const added = applyKeys(start, ['0:b', '0:c'], true); + assert.deepEqual([...added], ['0:a', '0:b', '0:c']); + assert.ok(Object.isFrozen(added)); + assert.deepEqual([...start], ['0:a']); + const removed = applyKeys(added, ['0:a', '0:b'], false); + assert.deepEqual([...removed], ['0:c']); + assert.equal(applyKeys(removed, ['0:c'], true), removed); + assert.equal(applyKeys(removed, ['0:a'], false), removed); +}); + +test('every listed key is checked only when each one is in the set', async () => { + const { everyChecked } = await source; + assert.equal(everyChecked(new Set(KEYS), KEYS), true); + assert.equal(everyChecked(new Set(KEYS.slice(1)), KEYS), false); + assert.equal(everyChecked(new Set(['x', ...KEYS.slice(1)]), KEYS), false); + assert.equal(everyChecked(new Set(), []), false); +}); + +test('the select-all-matching offer needs every loaded image checked and more to load', async () => { + const { offersAllMatching } = await source; + const all = new Set(KEYS); + assert.equal(offersAllMatching(all, KEYS, { total: 9, hasMore: true }), true); + assert.equal(offersAllMatching(all, KEYS, { total: 5, hasMore: false }), false); + assert.equal(offersAllMatching(new Set(KEYS.slice(1)), KEYS, { total: 9, hasMore: true }), false); + assert.equal(offersAllMatching(new Set(), [], { total: 9, hasMore: true }), false); +}); + +test('the counts read "N selected" and "All N matching selected", grouped', async () => { + const { matchingLabel, matchingOffer } = await source; + assert.equal(matchingLabel(12345), 'All 12,345 matching selected'); + assert.equal(matchingOffer(12345), 'Select all 12,345 matching'); +}); + +function fakeTimers() { + const timers = new Map(); + let next = 1; + return { + set: (callback, delay) => { timers.set(next, { callback, delay }); return next++; }, + clear: id => timers.delete(id), + fire: () => { for (const [id, { callback }] of [...timers]) { timers.delete(id); callback(); } }, + pending: () => timers.size, + delays: () => [...timers.values()].map(timer => timer.delay), + }; +} + +test('a held press fires once after the delay and swallows the click that follows', async () => { + const { createLongPress, LONG_PRESS_MS } = await source; + const timers = fakeTimers(), fired = []; + const press = createLongPress({ onLongPress: target => fired.push(target), setTimer: timers.set, clearTimer: timers.clear }); + press.down(10, 10, 'card-a'); + assert.deepEqual(timers.delays(), [LONG_PRESS_MS]); + assert.equal(LONG_PRESS_MS, 500); + press.move(14, 13); + timers.fire(); + assert.deepEqual(fired, ['card-a']); + press.up(); + assert.equal(press.consumeClick(), true); + assert.equal(press.consumeClick(), false); +}); + +test('moving past the slop, lifting early or cancelling never fires', async () => { + const { createLongPress, LONG_PRESS_SLOP } = await source; + assert.equal(LONG_PRESS_SLOP, 10); + const timers = fakeTimers(), fired = []; + const press = createLongPress({ onLongPress: target => fired.push(target), setTimer: timers.set, clearTimer: timers.clear }); + press.down(0, 0, 'a'); press.move(0, 11); + assert.equal(timers.pending(), 0); + press.down(0, 0, 'b'); press.up(); + assert.equal(timers.pending(), 0); + press.down(0, 0, 'c'); + assert.equal(press.pending(), true); + press.cancel(); + assert.equal(press.pending(), false); + assert.equal(timers.pending(), 0); + press.move(0, 0); press.up(); press.cancel(); + assert.deepEqual(fired, []); + assert.equal(press.consumeClick(), false); + // A new press drops a long press whose click never came. + press.down(0, 0, 'd'); timers.fire(); + press.down(0, 0, 'e'); + assert.equal(press.consumeClick(), false); + timers.fire(); + assert.deepEqual(fired, ['d', 'e']); + // A cancel (a mouse press on a hybrid device) forgets a fired press whose click never came. + press.cancel(); + assert.equal(press.consumeClick(), false); +}); diff --git a/tests/js/workspace-images-state.test.js b/tests/js/workspace-images-state.test.js index bb81a6e9..49cc0801 100644 --- a/tests/js/workspace-images-state.test.js +++ b/tests/js/workspace-images-state.test.js @@ -494,3 +494,11 @@ test('defaults apply to a new visit but not to the current view or a saved view' assert.deepEqual(initialImagesView({}), { layout: 'grid', density: 'comfortable' }); assert.deepEqual(initialImagesView({ preferences: {} }), { layout: 'grid', density: 'comfortable' }); }); + +test('listingFilter is the listing query as an object, without paging (#392)', async () => { + const { listingFilter } = await source; + assert.deepEqual(listingFilter({q:'cat', root:'1', subfolder:'a/b', category:'Animals', tags:[' x ','y'], media:'video', sort:'oldest', favorites:true}), + {q:'cat', root:'1', subfolder:'a/b', category:'Animals', tags:'x,y', media:'video', sort:'oldest', favorites:'1'}); + assert.deepEqual(listingFilter({root:'all', q:'', tags:[], media:'all'}), {sort:'newest'}); + assert.throws(() => listingFilter({sort:'sideways'}), TypeError); +}); diff --git a/tests/js/workspace-images-tagging-bulk.test.js b/tests/js/workspace-images-tagging-bulk.test.js index 4ed2fa17..ac6dbeff 100644 --- a/tests/js/workspace-images-tagging-bulk.test.js +++ b/tests/js/workspace-images-tagging-bulk.test.js @@ -196,3 +196,29 @@ test('a preset starts the Tag image dialog on the chosen model and options', asy assert.throws(() => createImageTaggingController({ api, target, preset: { model: 'gguf', options: { instructions: '' } } }), TypeError); assert.throws(() => createImageTaggingController({ api, target, preset: { model: 'gguf', options: { nope: 1 } } }), TypeError); }); + +test('all matching sends the listing filter instead of targets, up to 10,000 images (#392)', async () => { + const { openAddTags, openAutoTag, MAX_FILTER_IMAGES, TOO_MANY_MATCHING } = module(); + assert.equal(MAX_FILTER_IMAGES, 10000); + assert.equal(TOO_MANY_MATCHING, 'Narrow the view to at most 10,000 images.'); + const toasts = [], posted = [], tagged = []; + let ended = null; + const store = { start: async (kind, body) => { posted.push([kind, body]); return { job: { id: JOB_ID, state: 'running' } }; }, + find: () => ended, subscribe: () => () => {} }; + const filter = { q: 'cat', sort: 'newest' }; + // The loaded images can be more than 500: the filter, not the identities, is sent. + const loaded = Array.from({ length: 600 }, (_, index) => image(`bulk/${index}.png`)); + let dialog = null; + openAddTags({ scope: pageScope(), store, images: loaded, filter, count: 1234, notify: text => toasts.push(text), + onTagged: keys => tagged.push(keys.size), openInput: args => { dialog = args; } }); + assert.ok(dialog, 'the Add tags dialog opens'); + ended = { id: JOB_ID, state: 'finished', counters: { tagged: 3 } }; + await dialog.onApply('sunset'); + assert.deepEqual(posted, [['image-tags', { mode: 'add', filter, tags: ['sunset'] }]]); + // The loaded files are read again when the job tagged any. + assert.deepEqual(tagged, [600]); + openAddTags({ scope: pageScope(), store, images: loaded, filter, count: 10001, notify: text => toasts.push(text), + openInput: () => assert.fail('no dialog over the cap') }); + openAutoTag({ api: {}, scope: pageScope(), store, images: loaded, filter, count: 10001, notify: text => toasts.push(text) }); + assert.deepEqual(toasts, [TOO_MANY_MATCHING, TOO_MANY_MATCHING]); +}); diff --git a/tests/test_api_workspace_images.py b/tests/test_api_workspace_images.py index cb2ff6dd..f30f3947 100644 --- a/tests/test_api_workspace_images.py +++ b/tests/test_api_workspace_images.py @@ -2335,3 +2335,162 @@ class TestWorkspaceImageTags(ImageAPITestCase): ) ] self.assertFalse([step for step in plan if step.startswith("SCAN")], plan) + + +class TestListingFilterSelection(ImageAPITestCase): + """#392: "Select all N matching" resolves the listing filter on the server.""" + + async def tearDownAsync(self): + self.api.db.close_all() + await super().tearDownAsync() + + def media(self, name, mtime=100): + target = self.output_dir / name + target.parent.mkdir(parents=True, exist_ok=True) + make_png(target) + os.utime(target, (mtime, mtime)) + return target + + async def listing(self, **params): + response = await self.client.get( + "/prompt_manager/images/output", + params={"mode": "workspace", "limit": 500, **params}, + ) + return (await response.json())["images"] + + async def test_resolve_lists_every_match_in_listing_order_beyond_a_page(self): + for number in range(7): + self.media(f"bulk/{number:02}.png", 100 + number) + self.media("other/x.png", 500) + from py.api.images import parse_listing_filter + + params = parse_listing_filter({"subfolder": "bulk", "sort": "newest"}) + resolved = await self.api.resolve_listing_filter(params, cap=100) + + listed = await self.listing(subfolder="bulk", sort="newest") + self.assertEqual( + [list(target) for target in resolved.targets], + [[i["root_index"], i["root_scope"], i["path"]] for i in listed], + ) + self.assertEqual(len(resolved.targets), 7) + self.assertFalse(resolved.truncated) + + async def test_resolve_reports_linked_positives_and_unlinked_count(self): + first = self.media("a.png", 300) + second = self.media("b.png", 200) + self.media("c.png", 100) + prompt = self._save_prompt("shared positive") + self.api.db.link_image_to_prompt(prompt, str(first)) + self.api.db.link_image_to_prompt(prompt, str(second)) + from py.api.images import parse_listing_filter + + resolved = await self.api.resolve_listing_filter(parse_listing_filter({}), 100) + + self.assertEqual(resolved.positive_ids, [prompt]) + self.assertEqual(resolved.unlinked, 1) + + async def test_resolve_marks_more_than_the_cap_as_truncated(self): + for number in range(4): + self.media(f"{number}.png", 100 + number) + from py.api.images import parse_listing_filter + + resolved = await self.api.resolve_listing_filter(parse_listing_filter({}), 3) + + self.assertTrue(resolved.truncated) + + def test_parse_listing_filter_rejects_anything_but_listing_strings(self): + from py.api.images import parse_listing_filter + + self.assertEqual( + parse_listing_filter({"q": "cat", "tags": "a,b", "root": "0"}), + {"q": "cat", "tags": "a,b", "root": "0"}, + ) + for bad in ( + None, + [], + "q=cat", + {"mode": "workspace"}, + {"cursor": "x"}, + {"path": "a.png"}, + {"q": 1}, + {"q": "x" * 2049}, + {"sort": "sideways"}, + {"media": "audio"}, + {"favorites": "yes"}, + {"subfolder": "../up"}, + {"root": "-1"}, + ): + with self.subTest(bad=bad): + with self.assertRaises(ValueError): + params = parse_listing_filter(bad) + # Root bounds need the roots: the resolver checks them. + from py.api.images import _workspace_root + + _workspace_root(params, [self.output_dir]) + + async def test_bulk_category_takes_the_listing_filter(self): + first = self.media("keep/a.png", 300) + second = self.media("keep/b.png", 200) + self.media("keep/c.png", 100) + other = self.media("skip/d.png", 400) + prompt = self._save_prompt("in the filter") + outside = self._save_prompt("outside the filter") + self.api.db.link_image_to_prompt(prompt, str(first)) + self.api.db.link_image_to_prompt(prompt, str(second)) + self.api.db.link_image_to_prompt(outside, str(other)) + + response = await self.client.post( + "/prompt_manager/bulk/category", + json={"images_filter": {"subfolder": "keep"}, "category": "Kept"}, + ) + data = await response.json() + + self.assertEqual(response.status, 200, data) + self.assertEqual( + (data["updated_count"], data["prompt_count"], data["skipped"]), (1, 1, 1) + ) + self.assertEqual(self.api.db.get_prompt_by_id(prompt)["category"], "Kept") + self.assertNotEqual(self.api.db.get_prompt_by_id(outside)["category"], "Kept") + + async def test_bulk_category_filter_with_no_linked_image_changes_nothing(self): + self.media("a.png") + + response = await self.client.post( + "/prompt_manager/bulk/category", + json={"images_filter": {}, "category": "Kept"}, + ) + data = await response.json() + + self.assertEqual(response.status, 200, data) + self.assertEqual( + (data["updated_count"], data["prompt_count"], data["skipped"]), (0, 0, 1) + ) + + async def test_bulk_category_refuses_a_bad_or_doubled_filter(self): + self.media("a.png") + prompt = self._save_prompt("p") + for body in ( + {"images_filter": {"sort": "sideways"}, "category": "x"}, + {"images_filter": "all", "category": "x"}, + {"images_filter": {}, "prompt_ids": [prompt], "category": "x"}, + {"images_filter": {"root": "9"}, "category": "x"}, + ): + with self.subTest(body=body): + response = await self.client.post( + "/prompt_manager/bulk/category", json=body + ) + self.assertEqual(response.status, 400, await response.text()) + + async def test_bulk_category_refuses_a_renumbered_source(self): + self.media("a.png") + + response = await self.client.post( + "/prompt_manager/bulk/category", + json={ + "images_filter": {"root": "0", "root_scope": "f" * 64}, + "category": "x", + }, + ) + + self.assertEqual(response.status, 409) + self.assertEqual((await response.json())["code"], "source-changed") diff --git a/tests/test_image_tag_jobs.py b/tests/test_image_tag_jobs.py index 25943b81..0af74981 100644 --- a/tests/test_image_tag_jobs.py +++ b/tests/test_image_tag_jobs.py @@ -11,6 +11,7 @@ engine is B2's fake service. import asyncio import hashlib import json +import os import sqlite3 import threading import unittest @@ -828,3 +829,108 @@ class TestFixRound(ImageTagJobTestCase): if __name__ == "__main__": unittest.main() + + +class TestFilterStart(ImageTagJobTestCase): + """#392: "Select all N matching" starts the job with the listing filter.""" + + async def test_filter_start_tags_every_matching_file_in_listing_order(self): + for number in range(6): + path = self.png(f"bulk/{number:02}.png", color=(number, 1, 1)) + os.utime(path, (100 + number, 100 + number)) + self.png("other/x.png") + + job = await self.run_job( + { + "mode": "add", + "filter": {"subfolder": "bulk", "sort": "oldest"}, + "tags": ["picked"], + } + ) + + self.assertEqual(job.scope, {"mode": "add", "images": 6}) + self.assertEqual(job.counters["tagged"], 6) + self.assertEqual( + [path for path, _detail in self.rows(job, "tagged")], + [f"bulk/{number:02}.png" for number in range(6)], + ) + self.assertEqual(self.stored("bulk/03.png")["tags"], ["picked"]) + self.assertEqual(self.stored("other/x.png")["tags"], []) + + async def test_auto_filter_start_runs_the_model_on_the_matching_files(self): + self.png("bulk/a.png") + self.png("other/b.png", color=(1, 2, 3)) + + job = await self.run_job( + { + "mode": "auto", + "filter": {"subfolder": "bulk"}, + "model_type": "wd14-vit", + "options": dict(WD14), + } + ) + + self.assertEqual(job.scope["images"], 1) + self.assertEqual(job.counters["tagged"], 1) + self.assertEqual(self.stored("other/b.png")["tags"], []) + + async def test_filter_with_no_match_is_refused_without_a_job(self): + self.png("a.png") + + status, data = await self.start( + {"mode": "add", "filter": {"subfolder": "missing"}, "tags": ["x"]} + ) + + self.assertEqual((status, data["code"]), (400, "no-images")) + self.assertEqual(self.api._jobs.list(), []) + + async def test_filter_over_the_cap_is_refused_without_a_job(self): + from py.api import image_tag_jobs + + for number in range(3): + self.png(f"{number}.png", color=(number, 0, 0)) + with patch.object(image_tag_jobs, "MAX_FILTER_TARGETS", 2): + status, data = await self.start( + {"mode": "add", "filter": {}, "tags": ["x"]} + ) + + self.assertEqual((status, data["code"]), (400, "too-many-images")) + self.assertEqual(self.api._jobs.list(), []) + self.assertEqual(image_tag_jobs.MAX_FILTER_TARGETS, 10_000) + + async def test_filter_bodies_are_validated(self): + self.png("a.png") + target = self.target("a.png") + for body in ( + {"mode": "add", "filter": {}, "targets": [target], "tags": ["x"]}, + {"mode": "add", "filter": "all", "tags": ["x"]}, + {"mode": "add", "filter": {"sort": "sideways"}, "tags": ["x"]}, + {"mode": "add", "filter": {"cursor": "x"}, "tags": ["x"]}, + {"mode": "add", "filter": {"root": "7"}, "tags": ["x"]}, + {"mode": "add", "filter": {}, "tags": []}, + ): + with self.subTest(body=body): + status, data = await self.start(body) + self.assertEqual(status, 400, data) + self.assertEqual(self.api._jobs.list(), []) + + async def test_a_filter_whose_source_was_renumbered_is_refused(self): + self.png("a.png") + scope = self.target("a.png")["root_scope"] + + job = await self.run_job( + {"mode": "add", "filter": {"root": "0", "root_scope": scope}, "tags": ["x"]} + ) + self.assertEqual(job.counters["tagged"], 1) + status, data = await self.start( + { + "mode": "add", + "filter": {"root": "0", "root_scope": "f" * 64}, + "tags": ["x"], + } + ) + self.assertEqual((status, data["code"]), (409, "source-changed")) + status, data = await self.start( + {"mode": "add", "filter": {"root_scope": scope}, "tags": ["x"]} + ) + self.assertEqual(status, 400, data) diff --git a/tools/vendor/frontend-coverage-baseline.json b/tools/vendor/frontend-coverage-baseline.json index 140905b4..de6c2416 100644 --- a/tools/vendor/frontend-coverage-baseline.json +++ b/tools/vendor/frontend-coverage-baseline.json @@ -275,6 +275,12 @@ "functions": 85, "statements": 98 }, + "workspace/images-multiselect.mjs": { + "lines": 98, + "branches": 98, + "functions": 98, + "statements": 98 + }, "workspace/images-selection.mjs": { "lines": 98, "branches": 93, diff --git a/web/js/workspace/images-multiselect.mjs b/web/js/workspace/images-multiselect.mjs new file mode 100644 index 00000000..4fd2436e --- /dev/null +++ b/web/js/workspace/images-multiselect.mjs @@ -0,0 +1,80 @@ +/** + * File-manager selection on the Images page (#392, D135): pure helpers for the selection bar (images-selection.mjs). + * Keys are image keys (`root_index:path`) in the listing's display order, so a range spans every loaded batch + * whether or not its cards are mounted. + */ + +/** The press that turns on touch selection mode, and how far it may drift before it counts as a scroll. */ +export const LONG_PRESS_MS = 500; +export const LONG_PRESS_SLOP = 10; + +/** + * The keys from `from` to `to`, inclusive, in `keys` order (either direction). Without a listed `from` (no anchor + * yet, or it left the listing) the range is `to` alone; an unlisted `to` gives none. + */ +export function rangeKeys(keys, from, to) { + const end = keys.indexOf(to); + if (end < 0) return []; + const start = from === null || from === undefined ? -1 : keys.indexOf(from); + if (start < 0) return [to]; + return keys.slice(Math.min(start, end), Math.max(start, end) + 1); +} + +/** `selected` with `keys` added (`on`) or removed, frozen; the same set when nothing changes. */ +export function applyKeys(selected, keys, on) { + if (keys.every(key => selected.has(key) === on)) return selected; + const next = new Set(selected); + for (const key of keys) { + if (on) next.add(key); else next.delete(key); + } + return Object.freeze(next); +} + +/** Whether `keys` is non-empty and every one of them is in `selected`. */ +export function everyChecked(selected, keys) { + return keys.length > 0 && selected.size >= keys.length && keys.every(key => selected.has(key)); +} + +/** "Select all N matching" shows once every loaded image is checked and the listing holds more than is loaded. */ +export function offersAllMatching(selected, keys, { total, hasMore }) { + return Boolean(hasMore) && total > keys.length && everyChecked(selected, keys); +} + +const grouped = count => count.toLocaleString('en-US'); +export const matchingLabel = total => `All ${grouped(total)} matching selected`; +export const matchingOffer = total => `Select all ${grouped(total)} matching`; + +/** + * A touch long-press: `down` starts the timer, a `move` past the slop, `up` or `cancel` stops it, and + * `onLongPress(target)` runs once it fires. `consumeClick()` answers whether the click the press ends in belongs to + * the long press (so the page swallows it); it answers once. A new `down` forgets an unclaimed long press. + */ +export function createLongPress({ onLongPress, delay = LONG_PRESS_MS, slop = LONG_PRESS_SLOP, + setTimer = setTimeout, clearTimer = clearTimeout }) { + let timer = null, origin = null, fired = false; + const stop = () => { + if (timer !== null) clearTimer(timer); + timer = null; origin = null; + }; + return { + down(x, y, target) { + stop(); + fired = false; + origin = { x, y }; + timer = setTimer(() => { timer = null; origin = null; fired = true; onLongPress(target); }, delay); + }, + move(x, y) { + if (origin && Math.hypot(x - origin.x, y - origin.y) > slop) stop(); + }, + up: stop, + /** A press that ends without its click (a mouse press next, a scroll): its long press is forgotten too. */ + cancel() { stop(); fired = false; }, + /** Whether a touch press is held and has not yet fired (its context menu belongs to the long press). */ + pending: () => timer !== null, + consumeClick() { + const claimed = fired; + fired = false; + return claimed; + }, + }; +} diff --git a/web/js/workspace/images-selection.mjs b/web/js/workspace/images-selection.mjs index c30894af..ee5cd69c 100644 --- a/web/js/workspace/images-selection.mjs +++ b/web/js/workspace/images-selection.mjs @@ -4,11 +4,16 @@ import { inspectPath, toInspectionRecord } from './inspection-state.mjs'; import { BUNDLE_FILENAME, metadataBundle, collectImagesMetadata, exportOutcome } from './images-export.mjs'; import { CATEGORY_DIALOG, CATEGORY_FAILED, CATEGORY_MAX_LENGTH, CATEGORY_NONE_LINKED, categoryOutcome, categoryTargets, updatedCount } from './images-category.mjs'; import { openInputDialog } from './workspace-dialogs.mjs'; -import { openAddTags, openAutoTag } from './images-tagging-bulk.mjs'; +import { MAX_FILTER_IMAGES, TOO_MANY_MATCHING, openAddTags, openAutoTag } from './images-tagging-bulk.mjs'; import { showWorkspaceToast } from './toast.mjs'; +import { applyKeys, createLongPress, matchingLabel, matchingOffer, offersAllMatching, rangeKeys } from './images-multiselect.mjs'; const CHECKBOX = '.image-overlays input[type="checkbox"]'; +const CHECKBOX_LABEL = '.image-overlays > label'; +const CARD = 'article[data-image-key]'; const CATEGORY_URL = '/prompt_manager/bulk/category'; +// D135: Export metadata reads every image's inspection in the browser, so it waits for loaded images. +export const EXPORT_LOADED_ONLY = 'Export metadata reads each image in the browser, so it works on loaded images only.'; function el(tag, text, className) { const element = document.createElement(tag); @@ -29,60 +34,196 @@ function el(tag, text, className) { * - `retain(kept)` drops the checked files a refreshed listing no longer holds (a tag save under a filter, I7). * - With the Activity store `activity`, Add tags and Auto-tag selected (D85) follow the count, as the concept's Add tags * leads its actions; `onTagged(keys)` re-reads the files a finished tagging job tagged. The selection stays. + * + * File-manager selection (#392, D135, beyond the concept's checkbox): the anchor is the last image toggled on its own; + * Shift+click (a checkbox or anywhere on a card) applies the anchor's range, Ctrl/Cmd+click toggles a card without + * opening it, a touch long-press starts touch selection mode in which taps toggle. `selectAll`, `extend` and + * `clear` serve the grid's keys (images.mjs). `getListing()` gives `{total, hasMore, positiveCount}` and + * `getFilter()` the listing filter: once every loaded image is checked and more match, the bar offers "Select all N + * matching", whose actions send that filter to the server. */ export function mountSelectionBar({ scope, api, grid, getItems, focusFallback, onCategorySet, activity = null, onTagged, - checked: initial = Object.freeze(new Set()) }) { + getListing, getFilter, checked: initial = Object.freeze(new Set()) }) { let checked = initial; let exporting = false, categorizing = false; + // D135: the range anchor, "all N matching" mode, touch selection mode and a pending Shift+click on a checkbox. + let anchor = null, allMatching = false, touchMode = false, lastPointer = 'mouse', shiftBox = null; const element = el('div'); element.id = 'bulkbar'; const bulk = el('div', undefined, 'bulkbar'), count = el('strong'); + const selectMatching = createButton(''); + scope.listen(selectMatching, 'click', () => { + allMatching = true; + render(); + focusAfterOffer(); + }); const setCategory = createButton('Set category'); scope.listen(setCategory, 'click', () => openCategory(setCategory)); const exportMetadata = createButton('Export metadata'); - scope.listen(exportMetadata, 'click', () => exportChecked(exportMetadata)); + scope.listen(exportMetadata, 'click', () => { if (!allMatching) exportChecked(exportMetadata); }); const clearButton = createButton('Clear'); // Clear hides the bar it sits in, so focus moves to the page heading instead of falling to the page. scope.listen(clearButton, 'click', () => { clear(); focusFallback.focus({ preventScroll: true }); }); bulk.append(count); if (activity) { const addTags = createButton('Add tags'); - scope.listen(addTags, 'click', () => openAddTags({ scope, store: activity, images: checkedImages(), trigger: addTags, onTagged })); + scope.listen(addTags, 'click', () => openAddTags({ scope, store: activity, images: checkedImages(), trigger: addTags, onTagged, + ...matchingTarget() })); const autoTag = createButton('Auto-tag selected'); // D85 - scope.listen(autoTag, 'click', () => openAutoTag({ api, scope, store: activity, images: checkedImages(), trigger: autoTag, onTagged })); + scope.listen(autoTag, 'click', () => openAutoTag({ api, scope, store: activity, images: checkedImages(), trigger: autoTag, onTagged, + ...matchingTarget() })); bulk.append(addTags, autoTag); } bulk.append(setCategory, exportMetadata, clearButton); /** The count updates in place: the same bar and `strong`, its text written only when it changes. */ function render() { - if (!checked.size) { bulk.remove(); return; } - const label = selectionLabel(checked.size); + if (!checked.size && !allMatching) { touchMode = false; bulk.remove(); return; } + const listing = getListing(); + const label = allMatching ? matchingLabel(listing.total) : selectionLabel(checked.size); if (count.textContent !== label) count.textContent = label; + // "Select all N matching" (D135) follows the count while every loaded image is checked and more match. + if (!allMatching && offersAllMatching(checked, loadedKeys(), listing)) { + const offer = matchingOffer(listing.total); + if (selectMatching.textContent !== offer) selectMatching.textContent = offer; + if (count.nextElementSibling !== selectMatching) count.after(selectMatching); + } else selectMatching.remove(); + if (allMatching) { + exportMetadata.setAttribute('aria-disabled', 'true'); + exportMetadata.title = EXPORT_LOADED_ONLY; + } else if (exportMetadata.title) { + exportMetadata.removeAttribute('title'); + if (!exporting) exportMetadata.removeAttribute('aria-disabled'); + } if (bulk.parentNode !== element) element.append(bulk); } - function clear() { - if (!checked.size) return; - checked = Object.freeze(new Set()); - for (const box of grid.querySelectorAll(CHECKBOX)) box.checked = false; + /** Every rendered checkbox shows the selection (cards mounted later read `isChecked`). */ + function paint() { + for (const box of grid.querySelectorAll(CHECKBOX)) { + const on = isChecked(box.closest(CARD).dataset.imageKey); + if (box.checked !== on) box.checked = on; + } + } + function update(next) { + checked = next; + paint(); render(); } + /** Leaving "all N matching": the loaded images become the explicit selection the next change applies to. */ + function settleMatching() { + if (!allMatching) return; + allMatching = false; + checked = Object.freeze(new Set(loadedKeys())); + } + function clear() { + anchor = null; touchMode = false; + if (!checked.size && !allMatching) return; + allMatching = false; + update(Object.freeze(new Set())); + } /** Keep only the checked files `kept(key)` still lists (a refreshed result), so the count is what the actions apply to. */ function retain(kept) { + if (allMatching) { render(); return; } const next = [...checked].filter(kept); if (next.length === checked.size) return; checked = Object.freeze(new Set(next)); render(); } + function loadedKeys() { + return getItems().map(imageKey); + } + function isChecked(key) { + return allMatching || checked.has(key); + } function checkedImages() { - return getItems().filter(image => checked.has(imageKey(image))); + return allMatching ? getItems() : getItems().filter(image => checked.has(imageKey(image))); + } + /** The actions' images for "all N matching": the listing filter and its total, which the server resolves. */ + function matchingTarget() { + return allMatching ? { filter: getFilter(), count: getListing().total } : {}; + } + /** Toggle one image on its own; it becomes the anchor. */ + function toggle(key, on = !isChecked(key)) { + settleMatching(); + anchor = key; + update(toggleSelection(checked, key, on)); + } + /** The anchor's range to `key` takes `on`; without an anchor `key` is toggled on and becomes it. */ + function range(key, on = true) { + settleMatching(); + const keys = rangeKeys(loadedKeys(), anchor, key); + if (anchor === null || keys.length === 1) anchor = key; + update(applyKeys(checked, keys, on)); + } + /** Shift+arrow keys: the range from the anchor (`from`, the card focus left, when there is none) to `to`. */ + function extend(from, to) { + if (anchor === null || !loadedKeys().includes(anchor)) anchor = from; + range(to, true); + } + function selectAll() { + if (allMatching) return; + update(applyKeys(checked, loadedKeys(), true)); + } + function focusAfterOffer() { + // The pressed offer left the bar: focus moves to the count's first action instead of the page. + (bulk.querySelector('button:not([aria-disabled="true"])') ?? focusFallback).focus({ preventScroll: true }); } // Every card carries its image key; its only form control is the checkbox, so a toggle never opens or inspects. scope.listen(grid, 'change', event => { const box = event.target; if (!box.matches(CHECKBOX)) return; - checked = toggleSelection(checked, box.closest('article[data-image-key]').dataset.imageKey, box.checked); - render(); + const key = box.closest(CARD).dataset.imageKey; + if (shiftBox === key) range(key, box.checked); + else toggle(key, box.checked); + shiftBox = null; + }); + // D135: modified clicks select instead of opening, inspecting or starring. Captured, so the page's own click + // handler (images.mjs) never sees them; a plain click keeps its meaning. + grid.addEventListener('click', event => { + const card = event.target.closest(CARD); + if (!card) return; + const key = card.dataset.imageKey; + if (longPress.consumeClick()) { swallow(event); return; } + if (event.target.closest(CHECKBOX_LABEL)) { + // The box toggles natively; its change event applies the range. + if (event.shiftKey) shiftBox = key; + return; + } + // A keyboard click (Enter, Space: `detail` 0) keeps its plain meaning in touch selection mode. + const tapped = touchMode && lastPointer !== 'mouse' && event.detail !== 0; + if (event.shiftKey) { swallow(event); range(key, true); } + else if (event.ctrlKey || event.metaKey || tapped) { swallow(event); toggle(key); } + }, { capture: true, signal: scope.signal }); + function swallow(event) { + event.preventDefault(); + event.stopPropagation(); + } + // A touch long-press selects the card and starts touch selection mode (D135). + const longPress = createLongPress({ onLongPress: key => { + touchMode = true; + anchor = key; + if (!isChecked(key)) toggle(key, true); + } }); + scope.cleanup(() => longPress.cancel()); + scope.listen(grid, 'pointerdown', event => { + lastPointer = event.pointerType || 'mouse'; + shiftBox = null; + const card = event.target.closest(CARD); + if (lastPointer === 'mouse' || !card || !event.isPrimary) { longPress.cancel(); return; } + longPress.down(event.clientX, event.clientY, card.dataset.imageKey); + }); + scope.listen(grid, 'pointermove', event => longPress.move(event.clientX, event.clientY)); + for (const type of ['pointerup', 'pointercancel', 'pointerleave']) scope.listen(grid, type, () => longPress.up()); + // A long-press's context menu (Android) would cover the selection it just started. + scope.listen(grid, 'contextmenu', event => { + if (lastPointer !== 'mouse' && (touchMode || longPress.pending())) event.preventDefault(); + }); + // Escape in the bar clears the selection; the bar hides, so focus moves to the page heading as after Clear. + scope.listen(element, 'keydown', event => { + if (event.key !== 'Escape' || (!checked.size && !allMatching)) return; + event.preventDefault(); + clear(); + focusFallback.focus({ preventScroll: true }); }); /** Busy until it settles: `aria-disabled` now, released in a `finally` so a throw cannot leave it stuck. */ @@ -126,8 +267,11 @@ export function mountSelectionBar({ scope, api, grid, getItems, focusFallback, o */ function openCategory(button) { if (categorizing) return; - const targets = categoryTargets(checkedImages()); - if (!targets.promptIds.length) { showWorkspaceToast(CATEGORY_NONE_LINKED); return; } + // All matching (D135): the listing's positive count says whether any match is linked; the server resolves. + if (allMatching && getListing().total > MAX_FILTER_IMAGES) { showWorkspaceToast(TOO_MANY_MATCHING); return; } + const targets = allMatching ? {filter: getFilter()} : categoryTargets(checkedImages()); + const linked = allMatching ? getListing().positiveCount !== 0 : targets.promptIds.length > 0; + if (!linked) { showWorkspaceToast(CATEGORY_NONE_LINKED); return; } openInputDialog({ scope, trigger: button, title: CATEGORY_DIALOG.title, label: CATEGORY_DIALOG.label, action: CATEGORY_DIALOG.action, maxLength: CATEGORY_MAX_LENGTH, onApply: category => applyCategory(button, targets, category) }); } @@ -135,12 +279,12 @@ export function mountSelectionBar({ scope, api, grid, getItems, focusFallback, o * The write carries no page signal: once sent it finishes even if the page is left (`writing` then tells the page * not to save a visit that predates it). A failure keeps the selection; a success reloads through `onCategorySet`. */ - async function applyCategory(button, { promptIds, skipped }, category) { + async function applyCategory(button, { promptIds, skipped, filter }, category) { categorizing = true; const release = busy(button); let result = null; try { - result = await api.post(CATEGORY_URL, { prompt_ids: promptIds, category }); + result = await api.post(CATEGORY_URL, filter ? { images_filter: filter, category } : { prompt_ids: promptIds, category }); } catch { // Reported below; the selection stays as it was. } finally { @@ -149,7 +293,9 @@ export function mountSelectionBar({ scope, api, grid, getItems, focusFallback, o } if (scope.signal.aborted) return; if (!result) { showWorkspaceToast(CATEGORY_FAILED); return; } - showWorkspaceToast(categoryOutcome(updatedCount(result, promptIds.length), skipped)); + if (filter && !result.prompt_count) { showWorkspaceToast(CATEGORY_NONE_LINKED); return; } + showWorkspaceToast(filter ? categoryOutcome(updatedCount(result, result.prompt_count), Number(result.skipped) || 0) + : categoryOutcome(updatedCount(result, promptIds.length), skipped)); const focused = document.activeElement === button; onCategorySet(); // The reload hides the bar with the focused action in it, as Clear does: focus moves to the page heading. @@ -159,10 +305,16 @@ export function mountSelectionBar({ scope, api, grid, getItems, focusFallback, o render(); return { element, - isChecked: key => checked.has(key), - checked: () => checked, + isChecked, + // A saved visit keeps "all N matching" as its loaded images; the offer then shows again. + checked: () => (allMatching ? Object.freeze(new Set(loadedKeys())) : checked), + hasSelection: () => allMatching || checked.size > 0, clear, retain, + selectAll, + extend, + /** A loaded batch or a refreshed total: the count and the offer follow it. */ + listingChanged: () => { if (checked.size || allMatching) render(); }, /** Whether a Set category write is still in flight. */ writing: () => categorizing, }; diff --git a/web/js/workspace/images-state.mjs b/web/js/workspace/images-state.mjs index c61daf46..3b702f79 100644 --- a/web/js/workspace/images-state.mjs +++ b/web/js/workspace/images-state.mjs @@ -56,6 +56,16 @@ function queryFor(filters, batchSize) { return query.toString(); } +/** + * The listing's filter as the bulk routes take it (#392 "Select all N matching"): the same parameters the listing + * query carries, as an object of strings, without `mode`, `limit` and `cursor`. + */ +export function listingFilter(filters) { + const query = new URLSearchParams(queryFor(filters, 1)); + query.delete('mode'); query.delete('limit'); + return Object.fromEntries(query); +} + function validatePage(page, cursor) { if (!page || !Array.isArray(page.images) || !Number.isSafeInteger(page.total) || page.total < 0 || typeof page.has_more !== 'boolean' || diff --git a/web/js/workspace/images-tagging-bulk.mjs b/web/js/workspace/images-tagging-bulk.mjs index b4788bec..31fd6294 100644 --- a/web/js/workspace/images-tagging-bulk.mjs +++ b/web/js/workspace/images-tagging-bulk.mjs @@ -9,6 +9,10 @@ * apply** (D90, ruling T3-bulk): **Apply automatically** starts the job, **Review each image** starts the browser * review session (images-tagging-review.mjs). A model that is not downloaded keeps Start disabled and says why. * + * - **All N matching** (#392, D135): with `filter` (the listing's, images-state.mjs `listingFilter`) and `count` (its + * total), both send the filter instead of the targets and the server resolves it, up to 10,000 images. Review each + * image walks images in the browser, so it is not offered. + * * The toast says the job started (the Activity store's "Tagging started." / "Auto-tagging started."), or why it did * not: a running record-changing job (named, with its link), a held model, a refused body. Activity shows progress. * The selection stays either way. When a started job ends having tagged files, `onTagged(keys)` re-reads them. @@ -26,6 +30,9 @@ import { showWorkspaceToast } from './toast.mjs'; export const IMAGE_TAG_JOB = 'image-tags'; export const MAX_BULK_IMAGES = 500; export const TOO_MANY = 'Select at most 500 images.'; +// #392: a filter start resolves on the server (py/api/images.py MAX_LISTING_SELECTION). +export const MAX_FILTER_IMAGES = 10000; +export const TOO_MANY_MATCHING = 'Narrow the view to at most 10,000 images.'; export const NO_TAGS = 'Enter at least one tag.'; // D118 export const ADD_DIALOG = Object.freeze({title: 'Tag selected images', label: 'Tags, separated by commas', action: 'Apply'}); export const AUTO_TITLE = 'Auto-tag selected images'; @@ -53,6 +60,16 @@ export function bulkTargets(images) { return images.map(imageTarget).filter(Boolean); } +/** + * The job body's images: `{filter}` for all matching (null over MAX_FILTER_IMAGES), else `{targets}` (null over 500). + * Returns `[images, refusal]`, the refusal being the toast to show. + */ +function jobImages(images, filter, count) { + if (filter) return count > MAX_FILTER_IMAGES ? [null, TOO_MANY_MATCHING] : [{filter}, null]; + const targets = bulkTargets(images); + return targets ? [{targets}, null] : [null, TOO_MANY]; +} + /** The Add tags field's names, deduplicated as the Tag image dialog's manual field is. */ export const parseBulkTags = parseManualTags; @@ -107,13 +124,13 @@ function followTagged({store, scope, job, images, onTagged}) { * to no names keeps the dialog and its text, and reports NO_TAGS on the field (D118). `openInput` is the dialog seam. */ export function openAddTags({scope, store, images, trigger, onTagged = () => {}, notify = showWorkspaceToast, - openInput = openInputDialog}) { - const targets = bulkTargets(images); - if (!targets) { notify(TOO_MANY); return; } + openInput = openInputDialog, filter = null, count = images.length}) { + const [chosen, refusal] = jobImages(images, filter, count); + if (refusal) { notify(refusal); return; } openInput({scope, trigger, ...ADD_DIALOG, validate: text => (parseBulkTags(text).length ? null : NO_TAGS), onApply: async text => { const tags = parseBulkTags(text); - const job = await startTagJob({store, scope, body: {mode: 'add', targets, tags}, notify}); + const job = await startTagJob({store, scope, body: {mode: 'add', ...chosen, tags}, notify}); if (job) followTagged({store, scope, job, images, onTagged}); }}); } @@ -141,15 +158,16 @@ function applyChoice(onChange) { * Auto-tag selected (D85): the model panel, How to apply (D90), Cancel and Start. Start validates the fields as * Generate does, then starts the job or the review session over `images` (the checked files, taken now). */ -export function openAutoTag({api, scope, store, images, trigger, onTagged = () => {}, notify = showWorkspaceToast}) { - const targets = bulkTargets(images); - if (!targets) { notify(TOO_MANY); return; } +export function openAutoTag({api, scope, store, images, trigger, onTagged = () => {}, notify = showWorkspaceToast, + filter = null, count = images.length}) { + const [chosen, refusal] = jobImages(images, filter, count); + if (refusal) { notify(refusal); return; } const listed = images.filter(imageTarget); const local = createScope(); const options = createTaggingOptionsController({api}); // D85: the Tag image dialog's model panel without Generate; its Suggestions field is the job's count (D91). const {element: panel, validate} = renderModelPanel({scope: local, controller: options, - heading: `Auto-tag ${countOf(listed.length, 'image')}`, generate: false}); // D118 + heading: `Auto-tag ${countOf(filter ? count : listed.length, 'image')}`, generate: false}); // D118 let choice = APPLY_CHOICES[0]; const actions = el('div', undefined, 'dialog-actions'); const cancel = createButton('Cancel'); @@ -159,7 +177,8 @@ export function openAutoTag({api, scope, store, images, trigger, onTagged = () = const status = el('div', undefined, 'small'); status.id = 'tag-generation-status'; status.setAttribute('role', 'status'); - const choices = applyChoice(next => { choice = next; start.textContent = next.action; }); // D90 + // D90; all matching (D135) has no Review each image, which walks images in the browser. + const choices = filter ? null : applyChoice(next => { choice = next; start.textContent = next.action; }); const render = state => { if (status.textContent !== state.unavailable) status.textContent = state.unavailable; if (state.canStart) start.removeAttribute('aria-disabled'); @@ -178,11 +197,11 @@ export function openAutoTag({api, scope, store, images, trigger, onTagged = () = return; } closeWorkspaceDialog(); - startTagJob({store, scope, body: {mode: 'auto', targets, ...request}, notify}).then(job => { + startTagJob({store, scope, body: {mode: 'auto', ...chosen, ...request}, notify}).then(job => { if (job) followTagged({store, scope, job, images: listed, onTagged}); }); }); - openContentDialog({scope, title: AUTO_TITLE, nodes: [panel, choices, status, actions], trigger, onClose: () => { + openContentDialog({scope, title: AUTO_TITLE, nodes: [panel, choices, status, actions].filter(Boolean), trigger, onClose: () => { document.getElementById('modal')?.classList.remove('tagging-modal'); local.dispose(); options.dispose(); diff --git a/web/js/workspace/images.mjs b/web/js/workspace/images.mjs index df8e8bd2..2b45744d 100644 --- a/web/js/workspace/images.mjs +++ b/web/js/workspace/images.mjs @@ -1,5 +1,5 @@ import { createScope, createButton } from '../shared/toolkit/core.js'; -import { createImageStream, withItem, imageKey, countLine, captionDate, scopeLabel, activeRefinements, initialImagesView } from './images-state.mjs'; +import { createImageStream, withItem, imageKey, countLine, captionDate, scopeLabel, activeRefinements, initialImagesView, listingFilter } from './images-state.mjs'; import { createImageCard } from './image-card.mjs'; import { layoutImages, visibleImages, gridMetrics, stacksCardInfo } from './image-layout.mjs'; import { mountFileInspector } from './file-inspector.mjs'; @@ -254,8 +254,15 @@ export function mountImages(main, api, parentScope, {embedded = false, fetchPage body.append(collection, details); // The checked cards and the concept's selection bar (#249 D8), between the filter row and the gallery layout. The // selection is kept across appended batches and cleared with the collection; the Library-embedded gallery has none. - const bar = embedded ? null : mountSelectionBar({scope, api, grid, getItems:() => state.items, focusFallback, - checked:visit?.checked, onCategorySet:() => { loadCategories(); reset(); }, activity, onTagged:keys => refreshTagged(keys)}); + const bar = embedded ? null : mountSelectionBar({scope, api, grid, getItems:() => state?.items ?? [], focusFallback, + checked:visit?.checked, onCategorySet:() => { loadCategories(); reset(); }, activity, onTagged:keys => refreshTagged(keys), + // #392 "Select all N matching": the listing's total and its filter, as the server lists them. + getListing:() => ({total:state?.total ?? 0, hasMore:Boolean(state?.hasMore), positiveCount:state?.positiveCount ?? null}), + // A Source filter carries its root's scope, so a renumbered source is refused (source-changed), not acted on. + getFilter:() => { + const filter = listingFilter(filters), scope = state?.items[0]?.root_scope; + return filter.root !== undefined && typeof scope === 'string' ? {...filter, root_scope:scope} : filter; + }}); if (bar) main.append(bar.element); main.append(body); const mediaViewer = createMediaViewer({api,container:main,parentScope:scope,inspectionController:inspection,sender, @@ -386,8 +393,19 @@ export function mountImages(main, api, parentScope, {embedded = false, fetchPage const columns = geometry.columns; const target = { Home: 0, End: state.items.length - 1, ArrowRight: index + 1, ArrowLeft: index - 1, ArrowDown: index + columns, ArrowUp: index - columns }[event.key]; - if (target !== undefined) { event.preventDefault(); focusImage(Math.max(0, Math.min(state.items.length - 1, target))); } - else if (event.key === 'Tab') { + if (target !== undefined) { + event.preventDefault(); + const next = Math.max(0, Math.min(state.items.length - 1, target)); + focusImage(next); + // D135: Shift extends the selection from the anchor (this card when there is none) to the new card. + if (event.shiftKey && bar) bar.extend(card.dataset.imageKey, imageKey(state.items[next])); + } else if (bar && (event.ctrlKey || event.metaKey) && !event.altKey && event.key.toLowerCase() === 'a') { + event.preventDefault(); + bar.selectAll(); + } else if (bar && event.key === 'Escape' && bar.hasSelection()) { + event.preventDefault(); + bar.clear(); + } else if (event.key === 'Tab') { const backward = event.shiftKey && event.target.matches('.image-open'); const forward = !event.shiftKey && event.target.closest('.image-card-actions'); const adjacent = backward ? index - 1 : index + 1; @@ -462,6 +480,7 @@ export function mountImages(main, api, parentScope, {embedded = false, fetchPage if (replaced && selection !== null && !itemIndexes.has(selection)) dropSelection(); if (replaced) bar?.retain(key => itemIndexes.has(key)); state = next; + bar?.listingChanged(); if (!embedded) { setText(count, countLine(next)); setText(captionDay, captionDate(next.items, filters.sort)); diff --git a/web/js/workspace/workspace.css b/web/js/workspace/workspace.css index 95bc9e79..8d29e3f2 100644 --- a/web/js/workspace/workspace.css +++ b/web/js/workspace/workspace.css @@ -142,6 +142,8 @@ main:focus-visible { outline: 2px solid var(--ca-focus); outline-offset: 4px; } absolute. Columns and gap still come from the concept's `.image-grid` tracks. */ .image-grid { position: relative; /* D3 */ } .image-grid > .media-card { position: absolute; } +/* #392 D135: a long-press on a card selects it, so no image callout (iOS) takes the press. */ +#image-grid > .media-card { -webkit-touch-callout: none; } /* #249 D3: a list row's info keeps its own height so it can be measured; centred, it looks as the stretched concept row. #249 D31: it may also shrink below a long nowrap title (the h3 ellipsis then applies, as in the grid). */ .list-grid .media-info { align-self: center; min-width: 0; /* D3, D31 */ } From d11b354ed682bd6a97ec094bbd6da63e348a9d0a Mon Sep 17 00:00:00 2001 From: Vito Date: Tue, 27 Oct 2026 22:40:57 +0000 Subject: [PATCH 02/11] fix(tagging): legacy pages tag each image, not its prompt (#379, PR 2) (#394) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(tagging): Tag output images tags each image, not its prompt (#379, PR 1) The native "Tag output images" job now runs the image-tags loop over every still image the output folders list, linked or not: each image gets its own model tags (at most "Suggestions per image") in its image tag set, and its bytes are marked auto-tagged in a new media_image_autotags table. No prompt tag or prompt text is written. - database/autotag_schema.py: one forward-only WITHOUT ROWID table, optional in old backups, refused without the annotation group. - add_image_tags/replace_image_tags take mark_autotagged; autotag_state and autotagged_locators read the marks through the one current-record rule. - "Skip already tagged" skips images whose current bytes are marked, counted in a stat-only pre-pass; hand tags do not count. - The review counts images and estimates the time (WD14 on CPU or CUDA). - System > Auto-tagging review rows and copy, D134 and the Auto-tagging contract follow. Part of #379 * fix(tagging): end the batch job after its cleanup and report pre-pass progress (#379) The job read as ended while it still held the model gate, so a quick restart could be refused with model-busy. Pre-pass chunks now publish progress, and a cancel between chunks keeps exact counts. Part of #379 * fix(tagging): measured GPU rate, and an ended image-tags job shows no time left (#379) The live check measured WD14 on CUDA at 0.167 s per image (100 images in 16.7 s with the model load), so the review estimate uses 0.17 s. A finished or cancelled Auto-tag/Add tags job kept "Processing file N of N · about X min left"; it now ends with " · N files processed". Part of #379 * fix(tagging): legacy pages tag each image, not its prompt (#379, PR 2) The admin and gallery batch starts the shared Tag output images job ("Tag output images (legacy page)") and relays its progress; closing the window cancels it. Review asks per image with Skip first and applies by image and sha256; apply refuses a prompt_id body. The Tag image dialog marks an image auto-tagged only after Generate suggestions on that capture. A second legacy batch (also one that wins the start race) answers JSON 409, and both pages now show that sentence. * test(tagging): gallery review units apply by image; record each progress change The gallery review unit tests still pinned the removed prompt_id apply. The legacy batch spec read the progress text once per observer callback, so a phase frame and the first image frame landing in one task lost the phase (WebKit); it now records every mutation record. --- TESTING.md | 55 +- database/operations.py | 31 - py/api/__init__.py | 3 + py/api/annotations.py | 20 + py/api/autotag_routes.py | 929 ++++++++++++++------------ tests/browser/legacy-autotag.spec.mjs | 120 +++- tests/browser/workflows.spec.mjs | 11 +- tests/js/gallery-review.test.js | 33 +- tests/js/image-helpers.test.js | 26 + tests/js/sse-stream.test.js | 24 + tests/test_api_annotation_suggest.py | 13 +- tests/test_api_autotag.py | 480 ++----------- tests/test_autotag_legacy.py | 617 +++++++++++++++++ tests/test_database.py | 44 -- tests/test_job_admission.py | 6 +- web/admin.html | 8 +- web/gallery.html | 1 + web/js/admin.js | 117 ++-- web/js/gallery.js | 36 +- web/js/image-helpers.js | 54 +- web/js/sse-stream.js | 16 +- 21 files changed, 1622 insertions(+), 1022 deletions(-) create mode 100644 tests/test_autotag_legacy.py diff --git a/TESTING.md b/TESTING.md index 6b05b1e1..ce4bd6b2 100644 --- a/TESTING.md +++ b/TESTING.md @@ -3826,7 +3826,9 @@ threshold off that grid is refused ("General-tag confidence must be a multiple o 0.05.", `validate_saved_threshold`; `draftProblem` holds back Preview tagging review and the batch Review with the same sentence) and falls back on load; an accepted one is stored exactly on the grid (#369). A per-request threshold keeps its own check: `/image-annotations/suggest` and -`/jobs/autotag` take 0–1; the legacy `/autotag/single` and `/autotag/start` only require a number. `GET /autotag/models` reports the +`/jobs/autotag` take 0–1; the legacy `/autotag/start` passes its options through +`validate_options` and starts the `autotag` job titled "Tag output images (legacy page)"; +the legacy `/autotag/single` only requires a number. `GET /autotag/models` reports the saved values (`wd14_general_threshold`, `wd14_character_threshold`, `suggestion_limit`, `default_prompt`, `model_type`, `existing`), so the Tag image and Auto-tag selected dialogs start from them; a preset (the bulk review, Preview tagging review) wins. @@ -3892,6 +3894,57 @@ images", "Tag output images (legacy page)", "Tagging one image (legacy page)", is pinned by `tests/test_autotag_schema.py` (forward-only: one empty `WITHOUT ROWID` table, optional in old backups, refused without the annotation group). +- **The legacy pages (#379, PR 2)** write image tags, never prompt tags. + `POST /autotag/start` (the admin and gallery batch) reads `model_type`, `prompt`, + `skip_tagged` (Skip or Merge), `keep_in_memory` and the thresholds (else the saved + ones), checks them with `validate_options` (WD14: both thresholds; GGUF/HF: + `instructions` = the non-blank `prompt`, else the saved template; the saved + "Suggestions per image") and starts the `autotag` job "Tag output images (legacy + page)" (gate owner `legacy-batch`) through the native start, so the safety backup and + the disk check apply. A second legacy batch answers JSON 409 `{success: false, error: + "Auto-tagging is already running"}` (also when another legacy start wins the race past + that check); `SseStream` hands a non-2xx JSON `error` to `onerror` as `serverMessage`, + which both pages show instead of "Connection lost during auto tagging". Any other refusal (a job in the way: " + already running", `model-unavailable`, `insufficient-space`, `model-busy`, a bad + option) is one `{type: "error", status: "error", message}` frame. The stream relays + the job: the phases as `{type: "progress", progress: 0 | 5 | 8, status: "Taking a + safety backup…" | "Loading model…" | "Listing output images…"}`, each image as + `{type: "progress", progress: 10 + progress × 85 / 100, status, processed, tagged, + skipped}` (at most every 0.5 s; `processed` = Tagged + Unchanged + Errors), and the end + as `complete` (`processed`, `tagged`, `skipped`, `errors`, `skipped_outside_roots: 0`, + `status` "Complete" or "No still images found", `model_status`, `model_loaded`), + `{type: "cancelled", message: "Cancelled. Tags already added were kept."}` or an + `error` frame with the job's status line. A job quiet for 5 s repeats its last frame, + and a closed stream cancels the job (the legacy pages have no other stop). + `POST /autotag/single` takes `existing` (`"skip"` or `"merge"`, default merge). Under + Skip, an image whose current bytes are auto-tagged answers `{success, already_tagged: + true, image_tags, prompt_id, filename}` after one stat, without the model gate. + Otherwise the image is captured and the model runs on the capture under the gate with + the saved cap; the answer adds `tags`, `sha256` (the captured bytes) and `image_tags`, + and `service.custom_prompt` is never set. Refusals: `409 model-unavailable`, `413 + too-large`, `404 file-not-found`, `409 file-changed`, `503 busy`, `415 not-a-still` + (the image is marked), `409 model-busy`. `POST /autotag/apply` takes exactly + `{image_id, sha256, tags}` or `{path, root_index?, sha256, tags}` (64 KiB), re-hashes + the file and unions the tags into that image (`add_image_tags`, marked): `{success, + added_tags, total_tags}`; an empty list only marks the image. A body with `prompt_id` + is `400` "This page is out of date. Reload it."; a changed file `409 file-changed` + ("The image changed since its tags were generated."); a full set `409 tag-limit`; a + restore `409 storage-changed`. The admin review asks each image with Skip first: an + auto-tagged one shows "Image Already Auto-tagged" / "A tagging model already tagged + this image. Generate new tags?" with its image tags (Skip, Skip All Tagged, Re-tag asks + again with Merge); the gallery review always merges. Apply & Next sends the image and + its `sha256` (linked or not). The batch copy: "Skip images already auto-tagged", + "Auto tagging complete! Tagged N images." ("1 image"), and under both progress bars + "Closing this window stops tagging. Tags already added are kept. Prompt Manager's + Activity page shows the time left.". Pinned by `tests/test_autotag_legacy.py`, + `tests/js/image-helpers.test.js` and `legacy-autotag.spec`. +- **Tag image dialog marks (DEC4).** A save after Generate suggestions ran on that + capture marks the image auto-tagged (also a replayed save, and a saved retry after + `tag-conflict`); a save without Generate does not. The API keeps the suggested capture + tokens; each new one drops the tokens whose capture is gone (`expires_in`, which never + resets a capture's idle clock), a release or a saved receipt drops its own, and + closing the captures clears them (`TestDialogMarks`). + No browser test downloads or runs a model: the injected `TestModel` answers, its `download_model` raises unless `/__test__/tagging/download {mode: hold | succeed | fail}` installs a fake that waits on a fixture event, and every tagging reset releases diff --git a/database/operations.py b/database/operations.py index 78ba02cd..0f45097f 100644 --- a/database/operations.py +++ b/database/operations.py @@ -1808,37 +1808,6 @@ class PromptDatabase( reader = DocumentReader(conn) return [self._image_row_to_dict(row, reader) for row in cursor] - def autotag_batch_candidates(self) -> List[Dict[str, Any]]: - """The images a Tag output images batch looks at (#328). - - ``{id, image_path, prompt_id, prompt_tags}`` for every image with a - stored path that is linked to a prompt, newest first (the - ``get_all_images`` order, ties newest row first). Never reads - ``workflow_data`` or the other documents: a batch needs none of them. - """ - sql = ( - "SELECT gi.id, gi.image_path, gi.prompt_id, " - "(SELECT GROUP_CONCAT(t.name, '|||') FROM prompt_tags pt " - "JOIN tags t ON pt.tag_id = t.id WHERE pt.prompt_id = gi.prompt_id) " - "AS prompt_tags " - "FROM generated_images gi " - "INNER JOIN prompts p ON gi.prompt_id = p.id " - "WHERE gi.image_path IS NOT NULL AND gi.image_path != '' " - "ORDER BY gi.generation_time DESC, gi.id DESC" - ) - with self.model.get_connection() as conn: - return [ - { - "id": row["id"], - "image_path": row["image_path"], - "prompt_id": row["prompt_id"], - "prompt_tags": ( - row["prompt_tags"].split("|||") if row["prompt_tags"] else [] - ), - } - for row in conn.execute(sql) - ] - def get_all_images( self, limit: int = 0, diff --git a/py/api/__init__.py b/py/api/__init__.py index d45f2c47..485578c7 100644 --- a/py/api/__init__.py +++ b/py/api/__init__.py @@ -363,6 +363,9 @@ class PromptManagerAPI( # Private media captures of the annotation routes (py/api/annotations.py), # created on first use with their restore subscription. self._captures = None + # Capture tokens Generate suggestions ran on: a save of one marks the + # image auto-tagged (#379). + self._suggested_tokens = set() self._captures_unsubscribe = None # Set by close_annotation_captures (shutdown): no store is recreated. self._captures_closed = False diff --git a/py/api/annotations.py b/py/api/annotations.py index 8c799ff3..47f905e6 100644 --- a/py/api/annotations.py +++ b/py/api/annotations.py @@ -296,6 +296,7 @@ class AnnotationRoutesMixin: self._captures_closed = True unsubscribe, self._captures_unsubscribe = self._captures_unsubscribe, None store, self._captures = self._captures, None + self._suggested_tokens = set() if unsubscribe is not None: unsubscribe() if store is not None: @@ -515,6 +516,7 @@ class AnnotationRoutesMixin: store = self._captures if store is not None and _TOKEN.fullmatch(token) is not None: store.release(token) + self._suggested_tokens.discard(token) return web.json_response( {"success": True}, headers={"Cache-Control": "no-store"} ) @@ -564,6 +566,9 @@ class AnnotationRoutesMixin: return _coded("capture-expired", _OTHER_FILE) if facts.sha256 != capture.sha256: return _coded("file-changed") + # A save after Generate suggestions ran on this capture marks the + # image auto-tagged (#379), so a Skip batch leaves the review alone. + suggested = token in self._suggested_tokens try: receipt = await self._run_in_executor( self.db.replace_image_tags, @@ -574,6 +579,7 @@ class AnnotationRoutesMixin: expected_epoch=expected_epoch, expected_tag_revision=revision, tags=tags, + mark_autotagged=suggested, ) except AnnotationCommandError as exc: current = exc.current or {} @@ -590,10 +596,23 @@ class AnnotationRoutesMixin: except Exception: self.logger.exception("Image tag save failed") return _error(500, "save-failed", "The tags could not be saved.") + # Dropped only after a saved or replayed receipt: a retry after a + # tag-conflict still marks. + self._suggested_tokens.discard(token) return web.json_response( {"success": True, **receipt}, headers={"Cache-Control": "no-store"} ) + def _remember_suggested(self, store, token: str) -> None: + """Note that Generate ran on *token*; forget tokens whose capture is + gone. ``expires_in`` does not reset a capture's idle clock.""" + self._suggested_tokens = { + kept + for kept in self._suggested_tokens + if store.expires_in(kept) is not None + } + self._suggested_tokens.add(token) + # -- suggest --------------------------------------------------------------- async def suggest_image_tags(self, request: web.Request) -> web.Response: @@ -675,6 +694,7 @@ class AnnotationRoutesMixin: finally: if not started: store.release_use(token) + self._remember_suggested(store, token) return web.json_response( { "success": True, diff --git a/py/api/autotag_routes.py b/py/api/autotag_routes.py index 5d0b7fe0..7314f0f7 100644 --- a/py/api/autotag_routes.py +++ b/py/api/autotag_routes.py @@ -6,7 +6,6 @@ import functools import json import os import re -import threading import time from pathlib import Path from typing import Optional @@ -33,6 +32,8 @@ AUTOTAG_JOB_TITLE = "Tag output images" AUTOTAG_PREVIEW_LEASE = "Tag output images review" AUTOTAG_OWNER_KIND = "autotag-job" BATCH_PREVIEW_BODY_MAX = 1024 +APPLY_BODY_MAX = 64 * 1024 # POST /autotag/apply: one image and its tags +_HEX64 = re.compile(r"[0-9a-f]{64}") BATCH_JOB_BODY_MAX = 32 * 1024 REVIEW_REQUIRED = ( "Replacing tags needs a review of each image. Use Preview tagging review, " @@ -133,16 +134,6 @@ _MEDIA_EXTENSIONS = frozenset( _SCAN_MAX_FILES = 50_000 _SCAN_MAX_DEPTH = 12 -# One batch run at a time; a second autotag/start gets 409 while this is held. -_autotag_batch_lock = threading.Lock() - - -def _file_exists(path) -> bool: - try: - return Path(str(path)).exists() - except (OSError, ValueError): - return False - def path_is_within(candidate, root) -> bool: """True when ``candidate`` is strictly inside ``root`` on the real filesystem. @@ -178,27 +169,39 @@ async def _single(event): _monotonic = time.monotonic # the legacy frames' throttle clock (patched in tests) -# What one image came to in a batch (``on_item``): fixed strings, never an -# exception's text. -SKIP_DETAILS = { - "unlinked": "File not found", - "prompt already processed": "Prompt handled by another image", - "outside allowed directories": "Outside the output folders", - "file missing": "File not found", - "already tagged": "Prompt already tagged", +# The legacy pages' batch (#379): the ``autotag`` job, relayed as their frames. +LEGACY_BATCH_TITLE = "Tag output images (legacy page)" +LEGACY_FRAME_INTERVAL = 0.5 # progress frames at most this often +LEGACY_REPEAT_AFTER = 5.0 # a quiet job repeats its last frame (notices a closed page) +_LEGACY_PHASES = { + "backup": (0, "Taking a safety backup…"), + "model": (5, "Loading model…"), + "listing": (8, "Listing output images…"), } -def _added(count): - return f"Added {count} tag" if count == 1 else f"Added {count} tags" +def _legacy_busy(): + """The JSON 409 a legacy start gets while another legacy batch runs.""" + return web.json_response({"success": False, "error": _BUSY_ERROR}, status=409) -async def _refusal(title): - """The single error event of an auto-tag batch refused by another job.""" - message = f"{title} already running" +async def _error_frame(message): + """The single error frame of a legacy batch that could not start.""" yield {"type": "error", "status": "error", "message": message} +def _legacy_counts(job) -> dict: + """Legacy counters: ``processed`` = images the model ran on.""" + counters = job.counters + return { + "processed": counters.get("tagged", 0) + + counters.get("unchanged", 0) + + counters.get("errors", 0), + "tagged": counters.get("tagged", 0), + "skipped": counters.get("skipped", 0), + } + + def wd14_device(service) -> str: """Worker thread: ``"gpu"`` when WD14 runs (or would run) on CUDA, else ``"cpu"``; a service without the probe is ``"cpu"``.""" @@ -476,303 +479,181 @@ class AutotagRoutesMixin: params.update(body) return params - @staticmethod - def _real_tags(tags) -> list: - """Normalise a tag field to a list without bookkeeping markers.""" - if isinstance(tags, str): - tags = [t.strip() for t in tags.split(",") if t.strip()] - return [ - t - for t in (tags or []) - if t != "auto-scanned" - and not t.startswith("prepend:") - and not t.startswith("append:") - ] - - async def _apply_tags_to_prompt(self, prompt_id, tags) -> int: - """Merge ``tags`` into the prompt; how many new tags were added.""" - existing_prompt = await self._run_in_executor( - self.db.get_prompt_by_id, prompt_id - ) - if not existing_prompt: - return 0 - existing_tags = existing_prompt.get("tags", []) - if isinstance(existing_tags, str): - existing_tags = [t.strip() for t in existing_tags.split(",") if t.strip()] - new_tags = list(dict.fromkeys(t for t in tags if t not in existing_tags)) - if not new_tags: - return 0 - await self._run_in_executor( - self.db.update_prompt_metadata, prompt_id, tags=existing_tags + new_tags - ) - return len(new_tags) - - async def _autotag_batch_events(self, service, opts: dict): - """Yield SSE progress dicts while tagging every image linked to a prompt. - - Both batches run here: the legacy page's (its frames unchanged) and - the ``autotag`` job's. ``opts``: ``model_type``, ``keep_in_memory``, - ``existing`` (``skip`` leaves prompts with real tags alone, ``merge`` - adds to them), the two thresholds, and optionally ``gate_owner``, - ``prompt`` (the template, passed to each call; the legacy page sets - ``custom_prompt`` instead), ``cancelled`` (checked before each image: - a cancelled run returns without a ``complete`` frame) and - ``on_item(index, total, group, image_path, detail)`` (one outcome per - image, ``SKIP_DETAILS`` for the skips). - """ - model_type = opts["model_type"] - keep_in_memory = opts["keep_in_memory"] - cancelled = opts.get("cancelled") or (lambda: False) - on_item = opts.get("on_item") or (lambda *_args: None) - status = await self._run_in_executor(service.get_models_status) - if not status.get(model_type, {}).get("downloaded"): - yield {"type": "error", "message": f"Model {model_type} not downloaded"} - return - - if cancelled(): - return - yield {"type": "progress", "progress": 0, "status": "Loading model..."} - run = self._gated_runner(opts.get("gate_owner")) - try: - await run(service.load_model, model_type, True) - except Exception: - self.logger.exception("Failed to load model") - yield { - "type": "error", - "message": "Failed to load model. Check server logs for details.", - } - return - - if opts.get("custom_prompt"): - service.custom_prompt = opts["custom_prompt"] - tag_kwargs = { - "general_threshold": opts["general_threshold"], - "character_threshold": opts["character_threshold"], - } - if opts.get("prompt"): - tag_kwargs["prompt"] = opts["prompt"] - - yield { - "type": "progress", - "progress": 5, - "status": "Model loaded. Fetching all images from database...", - } - images = await self._run_in_executor(self.db.autotag_batch_candidates) - total = len(images) - if total == 0: - yield { - "type": "complete", - "processed": 0, - "tagged": 0, - "skipped": 0, - "errors": 0, - "skipped_outside_roots": 0, - "status": "No images with linked prompts found", - } - if not keep_in_memory: - service.unload_model() - return - - yield { - "type": "progress", - "progress": 10, - "status": f"Found {total} images. Processing...", - } - - counts = { - "processed": 0, - "tagged": 0, - "skipped": 0, - "errors": 0, - "skipped_outside_roots": 0, - } - tagged_prompt_ids = set() - last_update = _monotonic() - finished = False - - def progress_event(i, message): - return { - "type": "progress", - "progress": 10 + int((i + 1) / total * 85), - "status": message, - "processed": counts["processed"], - "tagged": counts["tagged"], - "skipped": counts["skipped"], - "skipped_outside_roots": counts["skipped_outside_roots"], - } - - try: - for i, image_data in enumerate(images): - if cancelled(): - return - image_path = image_data.get("image_path") - prompt_id = image_data.get("prompt_id") - skip_reason = None - allowed_path = None - if not image_path or not prompt_id: - skip_reason = "unlinked" - elif prompt_id in tagged_prompt_ids: - skip_reason = "prompt already processed" - else: - # Rows can point anywhere (restored DB, hand edits): only - # media files inside the served output roots are tagged. - allowed_path = self._resolve_allowed_image_path(str(image_path)) - if allowed_path is None: - if _file_exists(image_path): - skip_reason = "outside allowed directories" - counts["skipped_outside_roots"] += 1 - else: - skip_reason = "file missing" - elif opts["existing"] == "skip" and self._real_tags( - image_data.get("prompt_tags", []) - ): - tagged_prompt_ids.add(prompt_id) - skip_reason = "already tagged" - - if skip_reason is not None: - counts["skipped"] += 1 - on_item(i, total, "skipped", image_path, SKIP_DETAILS[skip_reason]) - now = _monotonic() - if (now - last_update) >= 0.5 or i == total - 1: - yield progress_event( - i, f"Skipping {i + 1}/{total} ({skip_reason})..." - ) - await asyncio.sleep(0.01) - last_update = now - continue - - try: - tags = await run(service.generate_tags, allowed_path, **tag_kwargs) - counts["processed"] += 1 - added = 0 - if tags: - added = await self._apply_tags_to_prompt(prompt_id, tags) - if added: - counts["tagged"] += 1 - on_item(i, total, "tagged", image_path, _added(added)) - else: - counts["skipped"] += 1 - on_item(i, total, "unchanged", image_path, "No new tags") - tagged_prompt_ids.add(prompt_id) - except Exception as img_err: - self.logger.error( - f"Error processing {os.path.basename(str(image_path))}: " - f"{img_err}" - ) - counts["errors"] += 1 - counts["processed"] += 1 - on_item(i, total, "errors", image_path, "Tagging failed") - - yield progress_event(i, f"Processing {i + 1}/{total}...") - await asyncio.sleep(0.01) - last_update = _monotonic() - - finished = True - if not keep_in_memory: - service.unload_model() - yield { - "type": "complete", - "progress": 100, - **counts, - "status": "Complete", - "model_status": ( - "Model kept in memory" if keep_in_memory else "Model unloaded" - ), - "model_loaded": keep_in_memory, - } - finally: - if not finished and not keep_in_memory: - service.unload_model() - async def start_autotag(self, request): - """Start batch auto-tagging with streaming progress (POST only).""" + """The legacy pages' batch (POST only): the ``autotag`` job as frames. + + Its parameters (``model_type``, ``prompt``, ``skip_tagged``, + ``keep_in_memory``, the thresholds or the saved ones) become the job's + options, checked by ``validate_options``. The job, titled "Tag output + images (legacy page)", tags each output image and never a prompt + (#379). A second legacy batch answers JSON 409; every other refusal is + one ``error`` frame (both pages read this route as a stream). Closing + the stream cancels the job. + """ + from ..autotag import MODELS + from ..config import MAX_SUGGESTION_LIMIT, AutotagConfig + from .annotation_suggest import OptionError, validate_options + params = await self._read_start_params(request) if params is None: return self._bad_request("Request body must be a JSON object") - - opts = { - "model_type": params.get("model_type", "gguf"), - "custom_prompt": params.get("prompt", ""), - "existing": ( - "skip" - if str(params.get("skip_tagged", "true")).lower() == "true" - else "merge" - ), - "keep_in_memory": str(params.get("keep_in_memory", "true")).lower() - == "true", - "general_threshold": params.get("general_threshold"), - "character_threshold": params.get("character_threshold"), - } + model_type = params.get("model_type", "gguf") try: - opts["general_threshold"], opts["character_threshold"] = ( - _thresholds_or_saved( - opts["general_threshold"], opts["character_threshold"] - ) + general, character = _thresholds_or_saved( + params.get("general_threshold"), params.get("character_threshold") ) except (ValueError, TypeError): return self._bad_request( "general_threshold and character_threshold must be numeric" ) - - errors = [] # an error event ends the run; Activity lists it failed - - async def events(): - try: - from ..autotag import get_autotag_service - - async for event in self._autotag_batch_events( - get_autotag_service(), opts - ): - if event.get("type") == "error": - errors.append(event) - yield event - except Exception: - self.logger.exception("AutoTag error") - errors.append(_INTERNAL_ERROR) - yield { - "type": "error", - "message": _INTERNAL_ERROR, - } - - # The batch writes prompt tags: a records hold on the registry, so it - # never runs beside a scan, restore or the workspace tagging job. + existing = ( + "skip" + if str(params.get("skip_tagged", "true")).lower() == "true" + else "merge" + ) + keep_in_memory = str(params.get("keep_in_memory", "true")).lower() == "true" + if not isinstance(model_type, str) or model_type not in MODELS: + return await self._stream_sse( + request, _error_frame(f"Invalid model type: {model_type}") + ) + if model_type.startswith("wd14"): + options = {"general_threshold": general, "character_threshold": character} + else: + prompt = params.get("prompt") + template = prompt if isinstance(prompt, str) and prompt.strip() else None + options = {"instructions": template or AutotagConfig.TEMPLATE} + try: + _model, options = validate_options( + model_type, + options, + default_limit=AutotagConfig.suggestion_limit(), + max_limit=MAX_SUGGESTION_LIMIT, + ) + except OptionError as error: + return await self._stream_sse(request, _error_frame(str(error))) + running = self._jobs.active(AUTOTAG_BATCH_JOB_KIND) + if running is not None and running.title == LEGACY_BATCH_TITLE: + return _legacy_busy() + # A job in the way is named before the engine is asked anything. busy = self._jobs.blocker(AUTOTAG_BATCH_JOB_KIND, admission="records") if busy is not None: - # A second legacy batch keeps its JSON 409; the workspace's Tag - # output images job is named in a stream frame. - if busy.kind == AUTOTAG_BATCH_JOB_KIND and busy.title != AUTOTAG_JOB_TITLE: - return web.json_response( - {"success": False, "error": _BUSY_ERROR}, status=409 - ) - # Both legacy pages read this route as a stream, where a 409 reads - # as a lost connection: refuse with one error frame instead (the - # admin page keys on ``type``, the gallery on ``status``). - return await self._stream_sse(request, _refusal(busy.title)) - async with self._jobs.hold( - AUTOTAG_BATCH_JOB_KIND, - OWNER_LABELS["legacy-batch"], - admission="records", - ) as job: - response = await self._start_autotag_batch(request, opts, events) - if response.status >= 400 or errors: - job.finish("failed") - return response - - async def _start_autotag_batch(self, request, opts, events): - if not _autotag_batch_lock.acquire(blocking=False): - return web.json_response( - {"success": False, "error": _BUSY_ERROR}, status=409 + return await self._stream_sse( + request, _error_frame(f"{busy.title} already running") ) try: - opts["gate_owner"] = model_gate.new_owner("legacy-batch") - if not model_gate.try_acquire(opts["gate_owner"]): - return model_busy_response() - try: - return await self._stream_sse(request, events()) - finally: - model_gate.release(opts["gate_owner"]) + job, refusal = await self._start_autotag( + model_type=model_type, + existing=existing, + options=options, + keep_in_memory=keep_in_memory, + title=LEGACY_BATCH_TITLE, + owner_kind="legacy-batch", + ) + except Exception: + self.logger.exception("AutoTag error") + return await self._stream_sse(request, _error_frame(_INTERNAL_ERROR)) + if refusal is not None: + if refusal.message == f"{LEGACY_BATCH_TITLE} already running": + # Another legacy start won the race past the check above. + return _legacy_busy() + return await self._stream_sse(request, _error_frame(refusal.message)) + return await self._stream_sse( + request, self._legacy_batch_frames(job, keep_in_memory) + ) + + async def _legacy_batch_frames(self, job, keep_in_memory): + """The legacy frames of a running ``autotag`` job, then its end frame. + + Phases map to fixed frames; progress frames come at most every 0.5 s; + a job quiet for 5 s repeats the last frame, so a closed page is noticed + (aiohttp sees a disconnect only on a write). A stream closed before the + job ended cancels it. + """ + events = job.events() + pending = None + last = None + last_progress = None + try: + while True: + if pending is None: + pending = asyncio.ensure_future(events.__anext__()) + done, _ = await asyncio.wait({pending}, timeout=LEGACY_REPEAT_AFTER) + if not done: + if last is not None: + yield last + continue + try: + payload = pending.result() + except StopAsyncIteration: + pending = None + break + pending = None + frame = self._legacy_frame(job, payload) + if frame is None: + continue + if frame.get("phase") is None: + now = _monotonic() + if ( + last_progress is not None + and now - last_progress < LEGACY_FRAME_INTERVAL + ): + continue + last_progress = now + frame.pop("phase", None) + last = frame + yield frame + yield self._legacy_end_frame(job, keep_in_memory) finally: - _autotag_batch_lock.release() + if pending is not None: + pending.cancel() + with contextlib.suppress(BaseException): + await pending + with contextlib.suppress(BaseException): + await events.aclose() + if job.running: + self._jobs.cancel(job.id) + + @staticmethod + def _legacy_frame(job, payload): + """One job event as a legacy progress frame, or None (an end event).""" + kind = payload.get("type") + if kind == "phase": + progress, status = _LEGACY_PHASES.get(payload.get("phase"), (0, "")) + if not status: + return None + return { + "type": "progress", + "progress": progress, + "status": status, + "phase": True, + } + if kind != "progress" or not job.total: + return None + return { + "type": "progress", + "progress": 10 + job.progress * 85 // 100, + "status": job.status_line, + **_legacy_counts(job), + } + + @staticmethod + def _legacy_end_frame(job, keep_in_memory): + if job.state == "finished": + return { + "type": "complete", + "progress": 100, + **_legacy_counts(job), + "errors": job.counters.get("errors", 0), + "skipped_outside_roots": 0, + "status": "Complete" if job.total else "No still images found", + "model_status": ( + "Model kept in memory" if keep_in_memory else "Model unloaded" + ), + "model_loaded": keep_in_memory, + } + if job.state == "cancelled": + return {"type": "cancelled", "message": BATCH_CANCELLED} + return {"type": "error", "status": "error", "message": job.status_line} # ── Tag output images (#328) ──────────────────────────────────────── @@ -969,59 +850,69 @@ class AutotagRoutesMixin: status=403, ) - async def _select_autotag_target(self, data: dict): - """Pick the image to tag from an autotag/single body. + def _root_of(self, resolved: str): + """``(root_index, root, relative)`` of a resolved file inside a root. - Returns ``(image_path, prompt_id, None)`` on success or - ``(None, None, error_response)`` when the request must be rejected. + The first output root that holds it (as ``path_is_within`` checks); + ``relative`` is ``/``-separated below the root's real path, so the + capture store's alias check holds. None when no root holds it. + """ + for index, root in enumerate(self._get_all_output_dirs()): + if path_is_within(resolved, root): + real_root = os.path.realpath(str(root)) + relative = os.path.relpath(resolved, real_root).replace(os.sep, "/") + return index, root, relative + return None + + async def _select_autotag_target(self, data: dict): + """Pick the image a legacy single/apply body names. + + Returns ``(target, None)`` with ``target`` = ``{path, prompt_id, + root_index, root, relative}``, or ``(None, error_response)``. """ image_id = data.get("image_id") if image_id is not None: if isinstance(image_id, bool) or not isinstance(image_id, (int, str)): - return None, None, self._bad_request("image_id must be an integer") + return None, self._bad_request("image_id must be an integer") try: image_id = int(image_id) except (TypeError, ValueError): - return None, None, self._bad_request("image_id must be an integer") + return None, self._bad_request("image_id must be an integer") record = await self._run_in_executor(self.db.get_image_by_id, image_id) if not record or not record.get("image_path"): - return ( - None, - None, - web.json_response( - {"success": False, "error": f"Image {image_id} not found"}, - status=404, - ), + return None, web.json_response( + {"success": False, "error": f"Image {image_id} not found"}, + status=404, ) stored_path = record["image_path"] resolved = self._resolve_allowed_image_path(stored_path) if resolved is None: - return None, None, self._forbidden(stored_path) - return resolved, record.get("prompt_id"), None + return None, self._forbidden(stored_path) + return self._target_of(resolved, record.get("prompt_id"), stored_path) raw_path = data.get("path") or data.get("image_path") if not raw_path or not isinstance(raw_path, str): - return None, None, self._bad_request("image_id or path is required") + return None, self._bad_request("image_id or path is required") if "root_index" in data: root_index = data["root_index"] if isinstance(root_index, bool) or not isinstance(root_index, int): - return None, None, self._bad_request("root_index must be an integer") + return None, self._bad_request("root_index must be an integer") roots = self._get_all_output_dirs() if not 0 <= root_index < len(roots): - return None, None, self._forbidden(raw_path) + return None, self._forbidden(raw_path) root = roots[root_index] if Path(raw_path).is_absolute() or ".." in raw_path.replace( "\\", "/" ).split("/"): - return None, None, self._forbidden(raw_path) + return None, self._forbidden(raw_path) raw_path = str(Path(root) / raw_path) if not path_is_within(raw_path, root): - return None, None, self._forbidden(raw_path) + return None, self._forbidden(raw_path) resolved = self._resolve_allowed_image_path(raw_path) if resolved is None: - return None, None, self._forbidden(raw_path) + return None, self._forbidden(raw_path) prompt_id = None try: @@ -1034,125 +925,345 @@ class AutotagRoutesMixin: ) except Exception as e: self.logger.warning(f"Could not find linked prompt: {e}") - return resolved, prompt_id, None + return self._target_of(resolved, prompt_id, raw_path) + + def _target_of(self, resolved, prompt_id, shown): + located = self._root_of(resolved) + if located is None: + return None, self._forbidden(shown) + index, root, relative = located + return { + "path": resolved, + "prompt_id": prompt_id, + "root_index": index, + "root": root, + "relative": relative, + }, None @staticmethod def _bad_request(message: str): return web.json_response({"success": False, "error": message}, status=400) + @staticmethod + def _coded_error(status: int, code: str, message: str): + return web.json_response( + {"success": False, "code": code, "error": message}, status=status + ) + + def _capture_refusal(self, exc, target): + """The legacy answer for a file the capture store refused (#379).""" + from .image_tag_jobs import FILE_CHANGED, FILE_NOT_FOUND, TOO_LARGE + + name = type(exc).__name__ + if name == "CaptureTooLarge": + return self._coded_error(413, "too-large", TOO_LARGE) + if name == "CaptureCapacity": + return self._coded_error( + 503, "busy", "The file could not be read now. Try again." + ) + if name == "CaptureChanged" and os.path.lexists(target["path"]): + return self._coded_error(409, "file-changed", FILE_CHANGED) + if name in ("CaptureDenied", "CaptureChanged"): + return self._coded_error(404, "file-not-found", FILE_NOT_FOUND) + return self._coded_error(500, "read-failed", "The file could not be read.") + + def _single_options(self, data, model_type): + """The validated suggestion options of a legacy single body.""" + from ..config import MAX_SUGGESTION_LIMIT, AutotagConfig + from .annotation_suggest import validate_options + + general, character = _thresholds_or_saved( + data.get("general_threshold"), data.get("character_threshold") + ) + if model_type.startswith("wd14"): + options = {"general_threshold": general, "character_threshold": character} + else: + prompt = data.get("prompt") + template = prompt if isinstance(prompt, str) and prompt.strip() else None + options = {"instructions": template or AutotagConfig.TEMPLATE} + use_gpu = data.get("use_gpu", True) + options["use_gpu"] = use_gpu if isinstance(use_gpu, bool) else True + return validate_options( + model_type, + options, + default_limit=AutotagConfig.suggestion_limit(), + max_limit=MAX_SUGGESTION_LIMIT, + )[1] + async def autotag_single(self, request): - """Generate tags for a single image selected by image_id or gallery path.""" + """Suggest tags for one image (the legacy review), never writing. + + ``existing: "skip"`` answers ``already_tagged`` without taking the + model gate when the image's current bytes are auto-tagged (#379). + Otherwise the image is captured and the model runs on the capture + under the gate, with at most the saved "Suggestions per image"; + the answer carries the captured ``sha256`` that ``/autotag/apply`` + writes against, and the image's current ``image_tags``. + """ + from ..autotag import MODELS + from .annotation_suggest import OptionError + try: data = await request.json() - if not isinstance(data, dict): - return self._bad_request("Request body must be a JSON object") + except (json.JSONDecodeError, UnicodeDecodeError): + return self._bad_request("Request body must be valid JSON") + if not isinstance(data, dict): + return self._bad_request("Request body must be a JSON object") + try: model_type = data.get("model_type", "gguf") - custom_prompt = data.get("prompt") - use_gpu = data.get("use_gpu", True) - general_threshold = data.get("general_threshold") - character_threshold = data.get("character_threshold") - try: - general_threshold, character_threshold = _thresholds_or_saved( - general_threshold, character_threshold + if not isinstance(model_type, str) or model_type not in MODELS: + return self._bad_request( + "model_type must be one of " + ", ".join(sorted(MODELS)) ) + existing = data.get("existing", "merge") + if existing not in ("skip", "merge"): + return self._bad_request('existing must be "skip" or "merge"') + try: + options = self._single_options(data, model_type) except (ValueError, TypeError): return self._bad_request( "general_threshold and character_threshold must be numeric" ) - - image_path, prompt_id, error = await self._select_autotag_target(data) + except OptionError as error: + return self._bad_request(str(error)) + target, error = await self._select_autotag_target(data) if error is not None: return error - - from ..autotag import get_autotag_service - - service = get_autotag_service() - owner = model_gate.new_owner("legacy-single") - if not model_gate.try_acquire(owner): - return model_busy_response() - try: - if ( - not service.is_model_loaded() - or service.get_loaded_model_type() != model_type - ): - await model_gate.run(owner, service.load_model, model_type, use_gpu) - - if custom_prompt: - service.custom_prompt = custom_prompt - - tags = await model_gate.run( - owner, - service.generate_tags, - image_path, - general_threshold=general_threshold, - character_threshold=character_threshold, - ) - finally: - model_gate.release(owner) - - return web.json_response( - { - "success": True, - "tags": tags, - "prompt_id": prompt_id, - "filename": os.path.basename(image_path), - } - ) - - except json.JSONDecodeError: - return self._bad_request("Request body must be valid JSON") + base = { + "success": True, + "prompt_id": target["prompt_id"], + "filename": os.path.basename(target["path"]), + } + if existing == "skip": + marked = await self._run_in_executor(self._marked_tags, target) + if marked is not None: + return web.json_response( + { + **base, + "already_tagged": True, + "tags": [], + "image_tags": marked, + } + ) + return await self._suggest_single(target, model_type, options, base) except Exception as e: self.logger.error(f"AutoTag single error: {e}") return web.json_response( {"success": False, "error": self._public_error(e)}, status=500 ) - async def apply_autotag(self, request): - """Apply selected tags to a prompt.""" + def _locator_of(self, target) -> str: + return os.path.normcase( + os.path.join( + os.path.realpath(str(target["root"])), *target["relative"].split("/") + ) + ) + + def _marked_tags(self, target): + """Worker thread: the current image tags when the image is + auto-tagged, else None. One stat, no hash.""" + from .images import _current_fingerprint + + fingerprint = _current_fingerprint(target["path"]) + if fingerprint is None: + return None + locator = self._locator_of(target) + if not self.db.autotag_state({locator: fingerprint}).get(locator): + return None + return self.db.current_image_tags({locator: fingerprint}).get(locator, []) + + async def _suggest_single(self, target, model_type, options, base): + from ..autotag import MODELS, get_autotag_service + from .annotation_suggest import unavailable_message + from .annotations import _pinned_suggestions, _settled, _submit + try: - data = await request.json() - prompt_id = data.get("prompt_id") - tags = data.get("tags", []) + from ...utils.media_capture import CaptureError + except ImportError: + from utils.media_capture import CaptureError - if not prompt_id: - return web.json_response( - {"success": False, "error": "prompt_id is required"}, status=400 + service = get_autotag_service() + status = await self._run_in_executor(service.get_models_status) + if not (status.get(model_type) or {}).get("downloaded"): + message = unavailable_message(MODELS[model_type]["name"]) + return self._coded_error(409, "model-unavailable", message) + owner = model_gate.new_owner("legacy-single") + if not model_gate.try_acquire(owner): + return model_busy_response() + store = self._capture_store() + try: + epoch = await self._run_in_executor(self.db.get_execution_epoch) + scope = self._gallery_root_scope(target["root"], target["root_index"]) + try: + capture = await _settled( + _submit( + store.capture, + target["root"], + target["relative"], + epoch=epoch, + scope=scope, + root_index=target["root_index"], + ), + lambda made: made is not None and store.release(made.token), ) - - if not tags: - return web.json_response( - {"success": True, "message": "No tags to apply"} + except CaptureError as exc: + return self._capture_refusal(exc, target) + try: + if not capture.inferable: + await self._mark_untaggable(capture, epoch) + return self._coded_error( + 415, "not-a-still", "Not a supported still image" + ) + if store.acquire(capture.token) is None: + return self._coded_error( + 409, + "storage-changed", + "Prompt storage was restored. Reload the page.", + ) + pairs = await model_gate.run( + owner, + _pinned_suggestions, + store, + capture.token, + service, + capture, + model_type, + options, + on_skipped=functools.partial(store.release_use, capture.token), ) + finally: + store.release(capture.token) + finally: + model_gate.release(owner) + current = await self._run_in_executor( + self.db.read_image_tags, capture.locator, capture.sha256 + ) + return web.json_response( + { + **base, + "tags": [str(name) for name, _score in pairs], + "sha256": capture.sha256, + "image_tags": current["tags"], + } + ) - prompt = await self._run_in_executor(self.db.get_prompt_by_id, prompt_id) - if not prompt: - return web.json_response( - {"success": False, "error": f"Prompt {prompt_id} not found"}, - status=404, - ) - - existing_tags = prompt.get("tags", []) - if isinstance(existing_tags, str): - existing_tags = [ - t.strip() for t in existing_tags.split(",") if t.strip() - ] - - new_tags = [t for t in tags if t not in existing_tags] - all_tags = existing_tags + new_tags - + async def _mark_untaggable(self, facts, epoch): + try: await self._run_in_executor( - self.db.update_prompt_metadata, prompt_id, tags=all_tags + functools.partial( + self.db.add_image_tags, + locator=facts.locator, + sha256=facts.sha256, + size=facts.size, + mtime_ns=facts.mtime_ns, + expected_epoch=epoch, + tags=[], + mark_autotagged=True, + ) ) + except Exception: + self.logger.warning("An unsupported image could not be marked") - return web.json_response( - {"success": True, "added_tags": new_tags, "total_tags": len(all_tags)} + async def apply_autotag(self, request): + """Union reviewed tags into one image's tags and mark it (#379). + + The body names the image (``image_id``, or ``path`` with optional + ``root_index``), the ``sha256`` ``/autotag/single`` returned and the + ``tags``; it is written only onto those exact bytes. A body with + ``prompt_id`` (a page loaded before the update) is refused: no path + writes prompt tags. + """ + try: + from ...database.annotations import ImageTagLimitError, normalize_image_tags + from ...database.library_commands import LibraryCommandError + from ...utils.media_capture import CaptureError + except ImportError: + from database.annotations import ImageTagLimitError, normalize_image_tags + from database.library_commands import LibraryCommandError + from utils.media_capture import CaptureError + + data, error = await self._read_json_body(request, max_bytes=APPLY_BODY_MAX) + if error is not None: + return error + if "prompt_id" in data: + return self._bad_request("This page is out of date. Reload it.") + keys = set(data) + if keys not in ( + {"image_id", "sha256", "tags"}, + {"path", "sha256", "tags"}, + {"path", "root_index", "sha256", "tags"}, + ): + return self._bad_request( + "The body must be {image_id, sha256, tags} or " + "{path, root_index?, sha256, tags}" ) + sha256 = data["sha256"] + if not isinstance(sha256, str) or not _HEX64.fullmatch(sha256): + return self._bad_request( + "sha256 must be 64 lowercase hexadecimal characters" + ) + try: + tags = normalize_image_tags(data["tags"]) + except ImageTagLimitError as exc: + return self._coded_error(400, "tag-limit", str(exc)) + except ValueError as exc: + return self._bad_request(str(exc)) + try: + epoch = await self._run_in_executor(self.db.get_execution_epoch) + target, error = await self._select_autotag_target(data) + if error is not None: + return error + store = self._capture_store() + try: + facts = await self._run_in_executor( + store.verify, target["root"], target["relative"] + ) + except CaptureError as exc: + return self._capture_refusal(exc, target) + if facts.sha256 != sha256: + return self._coded_error( + 409, + "file-changed", + "The image changed since its tags were generated.", + ) + try: + receipt = await self._run_in_executor( + functools.partial( + self.db.add_image_tags, + locator=facts.locator, + sha256=facts.sha256, + size=facts.size, + mtime_ns=facts.mtime_ns, + expected_epoch=epoch, + tags=tags, + mark_autotagged=True, + ) + ) + except LibraryCommandError as exc: + if exc.code == "storage-changed": + return self._coded_error( + 409, + "storage-changed", + "Prompt storage was restored. Reload the page.", + ) + if exc.code == "tag-limit": + from .annotations import _MESSAGES + return self._coded_error(409, "tag-limit", _MESSAGES["tag-limit"]) + raise except Exception as e: self.logger.error(f"Apply autotag error: {e}") return web.json_response( {"success": False, "error": self._public_error(e)}, status=500 ) + added = receipt.get("added", 0) + return web.json_response( + { + "success": True, + "added_tags": receipt["tags"][-added:] if added else [], + "total_tags": len(receipt["tags"]), + } + ) async def unload_autotag_model(self, request): """Manually unload the AutoTag model from memory.""" diff --git a/tests/browser/legacy-autotag.spec.mjs b/tests/browser/legacy-autotag.spec.mjs index 1140a887..ff96f681 100644 --- a/tests/browser/legacy-autotag.spec.mjs +++ b/tests/browser/legacy-autotag.spec.mjs @@ -95,8 +95,14 @@ async function watchProgress(page) { await page.evaluate(() => { window.progressSeen = []; const node = document.getElementById('autoTagProgressPercent'); - new MutationObserver(() => window.progressSeen.push(node.textContent)) - .observe(node, { childList: true, characterData: true, subtree: true }); + // Each change from its own record: frames that land in one task share one + // callback, and the node's text by then is only the last of them. + new MutationObserver(records => { + for (const record of records) { + if (record.type === 'characterData') window.progressSeen.push(record.target.data); + for (const added of record.addedNodes) window.progressSeen.push(added.textContent); + } + }).observe(node, { childList: true, characterData: true, subtree: true }); }); } @@ -108,25 +114,58 @@ async function runGalleryBatch(page) { await page.locator('#startAutoTagBtn').click(); await done; await expect(page.locator('#autoTagProgressModal')).toBeHidden(); - await expect(page.getByText(/^Auto tagging complete! Applied tags to \d+ prompts\.$/)).toBeVisible(); + // #379: the batch counts images, never prompts. + await expect(page.getByText(/^Auto tagging complete! Tagged \d+ images?\.$/)).toBeVisible(); +} + +const CLOSE_NOTE = 'Closing this window stops tagging. Tags already added are kept. Prompt Manager\'s Activity page shows the ' + + 'time left.'; + +async function serverJobs(request) { + const response = await request.get('/prompt_manager/jobs'); + await expect(response).toBeOK(); + return (await response.json()).jobs; +} + +async function runNativeBatch(request) { + const started = await request.post('/prompt_manager/jobs/autotag', { data: { model_type: 'wd14-swinv2', existing: 'skip', + options: { general_threshold: 0.35, character_threshold: 0.85 } } }); + expect(started.status()).toBe(202); + const { job } = await started.json(); + await expect.poll(async () => (await serverJobs(request)).find(item => item.id === job.id)?.state).toBe('finished'); } test('gallery batch modal counts progress and closes on complete', async ({ page, request }) => { await page.goto('/prompt_manager/gallery'); await runGalleryBatch(page); const seen = await page.evaluate(() => window.progressSeen); - // The fixture's prompts are already tagged, so the batch skips them; its progress frames still move the bar. - expect(seen).toContain('95%'); + // The phase frames (model 5%, listing 8%) then the per-image frames (from 10%) move the bar. + expect(seen).toContain('8%'); + expect(seen.some(value => parseInt(value, 10) >= 10)).toBe(true); + await expect(page.locator('#autoTagCloseNote')).toHaveText(CLOSE_NOTE); + // The batch ran as the autotag job of the legacy page. + expect((await serverJobs(request)).filter(job => job.kind === 'autotag').map(job => [job.title, job.state])) + .toEqual([['Tag output images (legacy page)', 'finished']]); expect((await downloads(request)).length).toBe(0); }); +test('admin batch copy names images, not prompts (#379)', async ({ page }) => { + await page.goto('/prompt_manager/admin'); + await page.locator('#autoTagBtn').click(); + await expect(page.locator('input[name="autoTagMode"][value="skip"] + span')).toHaveText('Skip images already auto-tagged'); + await expect(page.getByText('"auto-scanned" is not counted as a real tag')).toHaveCount(0); + await expect(page.locator('#autoTagCloseNote')).toHaveText(CLOSE_NOTE); + await expect(page.locator('#retagConfirmModal h3')).toHaveText('Image Already Auto-tagged'); + await expect(page.locator('#retagConfirmModal')).toContainText('A tagging model already tagged this image. Generate new tags?'); +}); + test('admin review opens the first image', async ({ page }) => { await page.goto('/prompt_manager/admin'); await page.locator('#autoTagBtn').click(); await page.locator('input[name="autoTagModel"][value="wd14-swinv2"]').check(); await page.locator('#startReviewBtn').click(); - // The first image's prompt is already tagged: confirm a new pass, then the review opens on that image. - await page.locator('#retagConfirmBtn').click(); + // #379: no image is auto-tagged yet (prompt tags no longer count), so the review opens on the first image. + await expect(page.locator('#retagConfirmModal')).toBeHidden(); await expect(page.locator('#autoTagReviewModal')).toBeVisible(); await expect(page.locator('#reviewImageName')).not.toHaveText(/^(image_name\.png|Unknown|)$/); await expect(page.getByText('Failed to start review mode')).toHaveCount(0); @@ -150,3 +189,70 @@ test('gallery unload works and is refused while the gate is held', async ({ page await unload.click(); await expect(page.getByText('WD14-SWINV2 model unloaded successfully')).toBeVisible(); }); + +/** The admin review lists every row, videos too: Skip any image that is not one until the confirmation shows. */ +async function untilAutoTagged(page, confirm) { + // Each image's skip check answers before its confirmation or its error shows. + const failed = page.locator('#reviewTagsVisible .text-pm-error'); + for (let tries = 0; tries < 5; tries++) { + await expect.poll(async () => (await confirm.isVisible()) || (await failed.count()) > 0).toBe(true); + if (await confirm.isVisible()) return; + const answered = page.waitForResponse(item => item.url().endsWith('/prompt_manager/autotag/single')); + await page.locator('#skipReviewBtn').click(); + await answered; + } + await expect(confirm).toBeVisible(); +} + +// #379: the admin review asks before re-tagging an image a model already tagged, and Apply & Next writes that image's +// tags (its bytes, never its prompt). +test('admin review: an auto-tagged image asks first; Re-tag then Apply & Next tags the image', async ({ page, request }) => { + await runNativeBatch(request); + await page.goto('/prompt_manager/admin'); + await page.locator('#autoTagBtn').click(); + await page.locator('input[name="autoTagModel"][value="wd14-swinv2"]').check(); + await page.locator('#startReviewBtn').click(); + const confirm = page.locator('#retagConfirmModal'); + await untilAutoTagged(page, confirm); + await expect(confirm.locator('#retagExistingTags span').first()).toHaveText('sandstone'); + await page.locator('#retagConfirmBtn').click(); + await expect(page.locator('#autoTagReviewModal')).toBeVisible(); + await expect(page.locator('#reviewTagsVisible .tag-chip').first()).toContainText('sandstone'); + const applied = page.waitForRequest(item => item.url().endsWith('/prompt_manager/autotag/apply')); + const answer = page.waitForResponse(item => item.url().endsWith('/prompt_manager/autotag/apply')); + await page.locator('#applyReviewBtn').click(); + const body = (await applied).postDataJSON(); + expect(body.prompt_id).toBeUndefined(); + expect(body.sha256).toMatch(/^[0-9a-f]{64}$/); + expect((await answer).status()).toBe(200); +}); + +test('admin review: Skip All Tagged passes every auto-tagged image', async ({ page, request }) => { + await runNativeBatch(request); + await page.goto('/prompt_manager/admin'); + await page.locator('#autoTagBtn').click(); + await page.locator('input[name="autoTagModel"][value="wd14-swinv2"]').check(); + await page.locator('#startReviewBtn').click(); + const confirm = page.locator('#retagConfirmModal'); + await untilAutoTagged(page, confirm); + await page.locator('#retagSkipAllBtn').click(); + await expect(page.locator('#retagConfirmModal')).toBeHidden(); + await expect(page.getByText('Review complete!')).toBeVisible(); +}); + +test('gallery review: Apply & Next tags the image by its path and bytes', async ({ page }) => { + await page.goto('/prompt_manager/gallery'); + await page.locator('#autoTagBtn').click(); + await page.locator('input[name="autoTagModel"][value="wd14-swinv2"]').check(); + const suggested = page.waitForRequest(item => item.url().endsWith('/prompt_manager/autotag/single')); + await page.locator('#startReviewBtn').click(); + expect((await suggested).postDataJSON().existing).toBe('merge'); + await expect(page.locator('#reviewTagsContainer .tag-chip').first()).toContainText('sandstone'); + const applied = page.waitForRequest(item => item.url().endsWith('/prompt_manager/autotag/apply')); + const answer = page.waitForResponse(item => item.url().endsWith('/prompt_manager/autotag/apply')); + await page.locator('#applyReviewBtn').click(); + const body = (await applied).postDataJSON(); + expect(Object.keys(body).sort()).toEqual(['path', 'root_index', 'sha256', 'tags'].filter(key => key in body).sort()); + expect(body.prompt_id).toBeUndefined(); + expect((await answer).status()).toBe(200); +}); diff --git a/tests/browser/workflows.spec.mjs b/tests/browser/workflows.spec.mjs index 62d86454..e822d998 100644 --- a/tests/browser/workflows.spec.mjs +++ b/tests/browser/workflows.spec.mjs @@ -132,7 +132,8 @@ test('the gallery cache clear keeps preferences and other tools caches', async ( }))).toEqual({ theme: 'light', other: 'keep', prefixed: 'keep', notice: '3.2.0', caches: ['other-tool-gallery-thumbnails'] }); }); -test('actual gallery review generates and persists tags for the selected root', async ({ page, request }) => { +// #379: the reviewed tags are that image's own tags; no prompt changes. +test('actual gallery review generates and persists tags for the selected root\'s image', async ({ page, request }) => { await page.goto('/prompt_manager/gallery'); await expect(page.locator('#galleryGrid .image-item')).toHaveCount(2); await page.locator('#autoTagBtn').click(); @@ -140,13 +141,15 @@ test('actual gallery review generates and persists tags for the selected root', await page.locator('#startReviewBtn').click(); const generation = await generated; expect(generation.request().postDataJSON().root_index).toBe(0); - await expect(page.locator('#reviewTagsContainer')).toContainText('generated'); + await expect(page.locator('#reviewTagsContainer')).toContainText('sandstone'); const result = page.waitForResponse(response => response.url().endsWith('/prompt_manager/autotag/apply')); await page.locator('#applyReviewBtn').click(); expect((await (await result).json()).success).toBe(true); const data = await (await request.get('/prompt_manager/recent?limit=20')).json(); - expect(data.results.find(prompt => prompt.text === 'Cat portrait').tags).toContain('generated'); - expect(data.results.find(prompt => prompt.text === 'Dog landscape').tags).not.toContain('generated'); + expect(data.results.find(prompt => prompt.text === 'Cat portrait').tags).toEqual(['cats']); + expect(data.results.find(prompt => prompt.text === 'Dog landscape').tags).toEqual(['dogs']); + const tagged = await (await request.get('/prompt_manager/images/output?mode=workspace&tags=sandstone')).json(); + expect(tagged.images.map(image => `${image.root_index}:${image.path}`)).toEqual(['0:shared/same.png']); }); test('actual Tags page reloads the selected filter and opens its image viewer', async ({ page }) => { diff --git a/tests/js/gallery-review.test.js b/tests/js/gallery-review.test.js index 32e27c17..52825f7b 100644 --- a/tests/js/gallery-review.test.js +++ b/tests/js/gallery-review.test.js @@ -36,33 +36,48 @@ function fixture() { const reply = (index, data) => requests[index].resolve({ json: async () => data }); return { manager, requests, reply, notifications }; } -test('gallery review retains root identity and applies tags with the returned prompt ID', async () => { +const SHA = 'a'.repeat(64); +test('gallery review retains root identity and applies the tags to the image by its sha256 (#379)', async () => { const { manager, requests, reply } = fixture(); const load = manager.loadNextReviewImage(); assert.equal(requests[0].body.root_index, 1); - reply(0, { success: true, prompt_id: 42, tags: ['smile'] }); + assert.equal(requests[0].body.existing, 'merge'); + reply(0, { success: true, prompt_id: 42, tags: ['smile'], sha256: SHA, image_tags: [] }); await load; manager.loadNextReviewImage = () => {}; const apply = manager.applyReviewTags(); - assert.deepEqual(requests[1].body, { prompt_id: 42, tags: ['smile'] }); + assert.equal(requests[1].url, '/prompt_manager/autotag/apply'); + assert.deepEqual(requests[1].body, ImageHelpers.autotagApplyBody( + { id: 'digest', path: 'same.png', root_index: 1 }, SHA, ['smile'])); + assert.equal('prompt_id' in requests[1].body, false); reply(1, { success: true }); await apply; }); -test('unlinked images do not submit an invalid apply request or advance', async () => { +test('an unlinked image applies to the image like a linked one', async () => { const { manager, requests, reply } = fixture(); const load = manager.loadNextReviewImage(); - reply(0, { success: true, prompt_id: null, tags: ['smile'] }); + reply(0, { success: true, prompt_id: null, tags: ['smile'], sha256: SHA, image_tags: [] }); await load; + manager.loadNextReviewImage = () => {}; const apply = manager.applyReviewTags(); - if (requests[1]) reply(1, { success: false, error: "Unexpected apply" }); + assert.equal(requests[1].body.sha256, SHA); + assert.deepEqual(requests[1].body.tags, ['smile']); + reply(1, { success: true }); await apply; - assert.equal(requests.length, 1); - assert.equal(manager.autoTagState.reviewIndex, 0); +}); +test('an answer without a captured sha256 never sends an apply', async () => { + const { manager, requests, reply } = fixture(); + const load = manager.loadNextReviewImage(); + reply(0, { success: true, prompt_id: 42, tags: ['smile'] }); + await load; + manager.loadNextReviewImage = () => {}; + await manager.applyReviewTags(); + assert.equal(requests.filter(r => r.url === '/prompt_manager/autotag/apply').length, 0); }); test('a failed apply keeps the current image available for retry', async () => { const { manager, requests, reply } = fixture(); const load = manager.loadNextReviewImage(); - reply(0, { success: true, prompt_id: 42, tags: ['smile'] }); + reply(0, { success: true, prompt_id: 42, tags: ['smile'], sha256: SHA, image_tags: [] }); await load; const apply = manager.applyReviewTags(); reply(1, { success: false, error: 'Try again' }); diff --git a/tests/js/image-helpers.test.js b/tests/js/image-helpers.test.js index 0439330e..a353109f 100644 --- a/tests/js/image-helpers.test.js +++ b/tests/js/image-helpers.test.js @@ -5,6 +5,8 @@ const { formatImageCaption, workflowDocumentHtml, autotagSingleBody, + autotagApplyBody, + autotagCompleteMessage, fallbackImageSource, } = require("../../web/js/image-helpers.js"); @@ -121,3 +123,27 @@ test('metadata lookup preserves encoded filename characters and works behind a p assert.equal(metadataLookupUrl('/prompt_manager/images/serve/a.png?v=123'), '/prompt_manager/images/prompt/a.png'); assert.equal(metadataLookupUrl('/elsewhere/a.png'), null); }); + +test("autotagSingleBody asks to skip auto-tagged images only when told (#379)", () => { + assert.equal(autotagSingleBody({ id: 1 }, WD14).existing, "merge"); + assert.equal(autotagSingleBody({ id: 1 }, { ...WD14, existing: "skip" }).existing, "skip"); + assert.equal(autotagSingleBody({ id: 1 }, { ...WD14, existing: "odd" }).existing, "merge"); +}); + +test("autotagApplyBody names the image, its bytes and the tags, never a prompt (#379)", () => { + const sha = "a".repeat(64); + assert.deepEqual(autotagApplyBody({ id: 4, prompt_id: 7 }, sha, ["bird"]), { image_id: 4, sha256: sha, tags: ["bird"] }); + assert.deepEqual(autotagApplyBody({ id: "digest", path: "sub/a.png", root_index: 1 }, sha, []), + { path: "sub/a.png", root_index: 1, sha256: sha, tags: [] }); + assert.deepEqual(autotagApplyBody({ id: 2 }, undefined, null), { image_id: 2, sha256: "", tags: [] }); + const tags = ["x"]; + autotagApplyBody({ id: 1 }, sha, tags).tags.push("y"); + assert.deepEqual(tags, ["x"]); +}); + +test("autotagCompleteMessage counts images (#379)", () => { + assert.equal(autotagCompleteMessage(2), "Auto tagging complete! Tagged 2 images."); + assert.equal(autotagCompleteMessage(1), "Auto tagging complete! Tagged 1 image."); + assert.equal(autotagCompleteMessage(0), "Auto tagging complete! Tagged 0 images."); + assert.equal(autotagCompleteMessage(undefined), "Auto tagging complete! Tagged 0 images."); +}); diff --git a/tests/js/sse-stream.test.js b/tests/js/sse-stream.test.js index 63e2f994..13cd052c 100644 --- a/tests/js/sse-stream.test.js +++ b/tests/js/sse-stream.test.js @@ -97,6 +97,30 @@ test("connect reports a non-2xx response through onerror and never onmessage", a assert.equal(source.readyState, source.CLOSED); }); +test("a non-2xx JSON answer's error sentence reaches onerror as serverMessage", async () => { + let error = null; + const fetchImpl = async () => new Response( + JSON.stringify({ success: false, error: "Auto-tagging is already running" }), + { status: 409, headers: { "Content-Type": "application/json" } }, + ); + const source = connect("/x", { fetchImpl }); + source.onerror = (err) => { error = err; }; + await until(() => error !== null); + assert.match(error.message, /409/); + assert.equal(error.status, 409); + assert.equal(error.serverMessage, "Auto-tagging is already running"); +}); + +test("a non-2xx answer without a JSON error has no serverMessage", async () => { + let error = null; + const fetchImpl = async () => new Response("nope", { status: 500 }); + const source = connect("/x", { fetchImpl }); + source.onerror = (err) => { error = err; }; + await until(() => error !== null); + assert.equal(error.status, 500); + assert.equal(error.serverMessage, undefined); +}); + test("connect reports a network failure through onerror", async () => { let error = null; const source = connect("/x", { fetchImpl: async () => { throw new TypeError("Failed to fetch"); } }); diff --git a/tests/test_api_annotation_suggest.py b/tests/test_api_annotation_suggest.py index 29727ee2..de097d61 100644 --- a/tests/test_api_annotation_suggest.py +++ b/tests/test_api_annotation_suggest.py @@ -363,23 +363,20 @@ class TestModelGate(SuggestTestCase): prompt_id = self._save_prompt("batch prompt") self.api.db.link_image_to_prompt(prompt_id, str(self.output_dir / "sub/a.png")) started, proceed = threading.Event(), threading.Event() - - def slow_generate(image_path, **kwargs): - started.set() - proceed.wait(WAIT) - return ["batch"] - - self.tagger.generate_tags = slow_generate + # The legacy batch is the autotag job (#379): it suggests per image. + self.tagger.block = (started, proceed) batch = asyncio.ensure_future( self.client.post("/prompt_manager/autotag/start?model_type=wd14-vit") ) try: await asyncio.get_running_loop().run_in_executor(None, started.wait, WAIT) + calls = len(self.tagger.calls) status, body = await self.suggest(self.suggest_body(observed, "gguf")) self.assertEqual((status, body["code"]), (409, "model-busy")) - self.assertEqual(self.tagger.calls, []) + self.assertEqual(len(self.tagger.calls), calls) self.assertEqual(self.tagger.loaded, "wd14-vit") # not swapped finally: + self.tagger.block = None proceed.set() response = await batch await response.read() diff --git a/tests/test_api_autotag.py b/tests/test_api_autotag.py index da420590..e2d2dffc 100644 --- a/tests/test_api_autotag.py +++ b/tests/test_api_autotag.py @@ -5,6 +5,7 @@ no network request is made. """ import asyncio +import hashlib import json import os import shutil @@ -34,6 +35,7 @@ from aiohttp.test_utils import AioHTTPTestCase from database.operations import PromptDatabase from py.api import PromptManagerAPI +from py.api.autotag_routes import MODEL_BUSY_ERROR from py.api.jobs import JobBusy from utils.hashing import generate_prompt_hash @@ -70,6 +72,7 @@ class FakeAutotagService: self.status_calls = 0 self.status_threads = [] self.status_paths = {} + self.suggest_calls = 0 def get_models_status(self): self.status_calls += 1 @@ -115,6 +118,25 @@ class FakeAutotagService: raise self.generate_error return list(self.tags) + def generate_tag_suggestions( + self, + image_path, + *, + limit, + prompt=None, + general_threshold=None, + character_threshold=None, + ): + """The suggestion entry point (#379): the fixed tags, at most ``limit``.""" + self.suggest_calls += 1 + tags = self.generate_tags( + image_path, + prompt=prompt, + general_threshold=general_threshold, + character_threshold=character_threshold, + ) + return [(name, None) for name in tags[:limit]] + class AutotagAPITestCase(AioHTTPTestCase): """Stands up the PromptManager routes with a temp DB, temp output dir and a @@ -175,8 +197,17 @@ class AutotagAPITestCase(AioHTTPTestCase): ) def _make_image(self, directory, name="img.png"): + """A small, valid image (the legacy review captures and probes it).""" + from PIL import Image + path = directory / name - path.write_bytes(b"\x89PNG\r\n\x1a\n") + path.parent.mkdir(parents=True, exist_ok=True) + shade = sum(name.encode()) % 256 + fmt = {".jpg": "JPEG", ".jpeg": "JPEG", ".webp": "WEBP"}.get( + path.suffix.lower(), "PNG" + ) + with Image.new("RGB", (4, 4), (shade, 64, 128)) as image: + image.save(path, fmt) return path def _link_image(self, prompt_id, path): @@ -216,10 +247,9 @@ class TestAutotagSingleContainment(AutotagAPITestCase): self.assertEqual(data["tags"], ["1girl", "smile"]) self.assertEqual(data["prompt_id"], prompt_id) self.assertEqual(len(self.service.generate_calls), 1) - self.assertEqual( - os.path.realpath(self.service.generate_calls[0][0]), - os.path.realpath(str(image)), - ) + # #379: the model reads a private capture of the image, never the file. + self.assertNotIn(str(self.output_dir), self.service.generate_calls[0][0]) + self.assertEqual(data["sha256"], hashlib.sha256(image.read_bytes()).hexdigest()) async def test_path_inside_output_dir_is_tagged(self): image = self._make_image(self.output_dir) @@ -449,238 +479,6 @@ class TestAutotagSideEffectRoutesArePostOnly(AutotagAPITestCase): self.assertTrue((await resp.json())["success"]) -class TestAutotagBatch(AutotagAPITestCase): - """Batch tagging over the DB image list with SSE progress reporting.""" - - def _seed(self): - """Three prompts: A (one image), B (two images), C (missing file).""" - a = self._save_prompt("prompt a") - b = self._save_prompt("prompt b") - c = self._save_prompt("prompt c") - self._link_image(a, self._make_image(self.output_dir, "a1.png")) - self._link_image(b, self._make_image(self.output_dir, "b1.png")) - self._link_image(b, self._make_image(self.output_dir, "b2.png")) - self._link_image(c, self.output_dir / "missing.png") - return a, b, c - - async def _start(self, query="model_type=gguf", body=None): - if body is None: - resp = await self.client.request( - "POST", f"/prompt_manager/autotag/start?{query}" - ) - else: - resp = await self._post_json(f"/prompt_manager/autotag/start?{query}", body) - return resp, self._sse_events(await resp.text()) - - async def test_batch_tags_each_prompt_once_and_reports_counts(self): - a, b, c = self._seed() - - resp, events = await self._start() - - self.assertEqual(resp.status, 200) - done = events[-1] - self.assertEqual(done["type"], "complete") - self.assertEqual(done["processed"], 2) - self.assertEqual(done["tagged"], 2) - self.assertEqual(done["skipped"], 2) - self.assertEqual(done["errors"], 0) - self.assertEqual(len(self.service.generate_calls), 2) - self.assertEqual(self.api.db.get_prompt_by_id(a)["tags"], ["1girl", "smile"]) - self.assertEqual(self.api.db.get_prompt_by_id(b)["tags"], ["1girl", "smile"]) - self.assertEqual(self.api.db.get_prompt_by_id(c)["tags"], []) - progress = [e for e in events if e["type"] == "progress"] - self.assertTrue(all(0 <= e["progress"] <= 100 for e in progress)) - self.assertTrue(any("processed" in e for e in progress)) - - async def test_rows_outside_output_dirs_are_skipped_and_counted(self): - inside = self._save_prompt("inside") - outside = self._save_prompt("outside") - self._link_image(inside, self._make_image(self.output_dir, "in.png")) - secret = self._make_image(self.secret_dir, "password.png") - self._link_image(outside, secret) - - _, events = await self._start() - - done = events[-1] - self.assertEqual(done["type"], "complete") - self.assertEqual(done["processed"], 1) - self.assertEqual(done["tagged"], 1) - self.assertEqual(done["skipped"], 1) - self.assertEqual(done["skipped_outside_roots"], 1) - self.assertEqual(len(self.service.generate_calls), 1) - self.assertEqual( - os.path.realpath(self.service.generate_calls[0][0]), - os.path.realpath(str(self.output_dir / "in.png")), - ) - self.assertEqual(self.api.db.get_prompt_by_id(outside)["tags"], []) - progress = [e for e in events if e["type"] == "progress" and "skipped" in e] - self.assertTrue(progress) - self.assertTrue(all("skipped_outside_roots" in e for e in progress)) - body = json.dumps(events) - self.assertNotIn(str(self.secret_dir), body) - - async def test_second_start_while_running_is_409(self): - self._seed() - started = threading.Event() - release = threading.Event() - - def slow_generate(image_path, **kwargs): - started.set() - release.wait(5) - return ["slow"] - - self.service.generate_tags = slow_generate - first = asyncio.ensure_future(self._start()) - try: - await asyncio.get_event_loop().run_in_executor(None, started.wait, 5) - self.assertTrue(started.is_set()) - - resp = await self.client.request( - "POST", "/prompt_manager/autotag/start?model_type=gguf" - ) - - self.assertEqual(resp.status, 409) - self.assertEqual( - await resp.json(), - {"success": False, "error": "Auto-tagging is already running"}, - ) - finally: - release.set() - resp1, events = await first - self.assertEqual(resp1.status, 200) - self.assertEqual(events[-1]["type"], "complete") - - async def test_lock_is_released_after_a_run(self): - self._seed() - - resp1, _ = await self._start() - resp2, events = await self._start() - - self.assertEqual(resp1.status, 200) - self.assertEqual(resp2.status, 200) - self.assertEqual(events[-1]["type"], "complete") - - async def test_lock_is_released_when_the_run_fails(self): - self._seed() - self.service.load_error = RuntimeError("no model") - - _, events = await self._start() - self.assertEqual(events[-1]["type"], "error") - - self.service.load_error = None - resp, events = await self._start() - self.assertEqual(resp.status, 200) - self.assertEqual(events[-1]["type"], "complete") - - async def test_already_tagged_prompts_are_skipped_unless_asked(self): - p = self._save_prompt("tagged", tags=["cat"]) - self._link_image(p, self._make_image(self.output_dir, "t.png")) - - _, events = await self._start("model_type=gguf&skip_tagged=true") - self.assertEqual(events[-1]["skipped"], 1) - self.assertEqual(self.service.generate_calls, []) - - _, events = await self._start("model_type=gguf&skip_tagged=false") - self.assertEqual(events[-1]["tagged"], 1) - self.assertEqual( - self.api.db.get_prompt_by_id(p)["tags"], ["cat", "1girl", "smile"] - ) - - async def test_prompt_already_holding_the_tags_counts_as_skipped(self): - p = self._save_prompt("same", tags=["1girl", "smile"]) - self._link_image(p, self._make_image(self.output_dir, "s.png")) - - _, events = await self._start("model_type=gguf&skip_tagged=false") - - self.assertEqual(events[-1]["skipped"], 1) - self.assertEqual(events[-1]["tagged"], 0) - - async def test_generation_errors_are_counted_and_do_not_abort(self): - self._seed() - self.service.generate_error = RuntimeError("boom") - - _, events = await self._start() - - self.assertEqual(events[-1]["type"], "complete") - # a1, b1 and b2 each fail: a failed prompt is retried on its next image - self.assertEqual(events[-1]["errors"], 3) - self.assertEqual(events[-1]["tagged"], 0) - - async def test_json_body_parameters_are_accepted(self): - self._seed() - - _, events = await self._start( - "", {"model_type": "wd14-vit", "general_threshold": 0.5, "prompt": "p"} - ) - - self.assertEqual(events[-1]["type"], "complete") - self.assertEqual(self.service.load_calls, [("wd14-vit", True)]) - self.assertEqual(self.service.generate_calls[0][2], 0.5) - self.assertEqual(self.service.custom_prompt, "p") - - async def test_keep_in_memory_false_unloads_after_batch(self): - self._seed() - - _, events = await self._start("model_type=gguf&keep_in_memory=false") - - self.assertEqual(events[-1]["model_loaded"], False) - self.assertEqual(self.service.unload_calls, 1) - - async def test_model_not_downloaded_is_an_error_event(self): - self.service.downloaded = False - - _, events = await self._start() - - self.assertEqual(events[-1]["type"], "error") - self.assertEqual(self.service.load_calls, []) - - async def test_load_failure_is_an_error_event_without_details(self): - self.service.load_error = RuntimeError("/srv/models/secret.gguf is corrupt") - - _, events = await self._start() - - self.assertEqual(events[-1]["type"], "error") - self.assertNotIn("/srv/models", events[-1]["message"]) - - async def test_non_numeric_threshold_is_400(self): - resp = await self.client.request( - "POST", "/prompt_manager/autotag/start?general_threshold=abc" - ) - self.assertEqual(resp.status, 400) - - async def test_invalid_json_body_is_400(self): - resp = await self.client.request( - "POST", - "/prompt_manager/autotag/start", - data="{not json", - headers={"Content-Type": "application/json"}, - ) - self.assertEqual(resp.status, 400) - - async def test_client_disconnect_stops_the_batch_and_unloads(self): - self._seed() - real_write = web.StreamResponse.write - state = {"writes": 0} - - async def flaky_write(response, data): - state["writes"] += 1 - if b"Processing " in data: - raise ConnectionResetError("Cannot write to closing transport") - return await real_write(response, data) - - with patch.object(web.StreamResponse, "write", flaky_write): - try: - resp = await self.client.request( - "POST", "/prompt_manager/autotag/start?keep_in_memory=false" - ) - await resp.read() - except Exception: - pass - - self.assertEqual(len(self.service.generate_calls), 1) - self.assertEqual(self.service.unload_calls, 1) - - class TestAutotagModelsAndDownloadErrors(AutotagAPITestCase): async def test_models_error_is_500(self): @@ -834,7 +632,15 @@ class TestAutotagSingleEdgeCases(AutotagAPITestCase): self.assertEqual(resp.status, 200) self.assertEqual(self.service.load_calls, [("wd14-vit", True)]) - self.assertEqual(self.service.custom_prompt, "terse") + # #379: the request's prompt is for this call only. + self.assertEqual(self.service.custom_prompt, "") + resp = await self._post_json( + "/prompt_manager/autotag/single", + {"path": str(image), "model_type": "gguf", "prompt": "terse"}, + ) + self.assertEqual(resp.status, 200) + self.assertEqual(self.service.generate_calls[-1][1], "terse") + self.assertEqual(self.service.custom_prompt, "") async def test_image_id_without_stored_path_is_404(self): prompt_id = self._save_prompt() @@ -850,64 +656,6 @@ class TestAutotagSingleEdgeCases(AutotagAPITestCase): self.assertEqual(resp.status, 404) -class TestApplyAutotag(AutotagAPITestCase): - - async def test_missing_prompt_id_is_400(self): - resp = await self._post_json("/prompt_manager/autotag/apply", {"tags": ["a"]}) - self.assertEqual(resp.status, 400) - - async def test_no_tags_is_a_noop(self): - resp = await self._post_json("/prompt_manager/autotag/apply", {"prompt_id": 1}) - data = await resp.json() - self.assertTrue(data["success"]) - self.assertEqual(data["message"], "No tags to apply") - - async def test_unknown_prompt_is_404(self): - resp = await self._post_json( - "/prompt_manager/autotag/apply", {"prompt_id": 999, "tags": ["a"]} - ) - self.assertEqual(resp.status, 404) - - async def test_tags_are_merged_without_duplicates(self): - prompt_id = self._save_prompt(tags=["cat"]) - - resp = await self._post_json( - "/prompt_manager/autotag/apply", - {"prompt_id": prompt_id, "tags": ["cat", "dog"]}, - ) - - data = await resp.json() - self.assertEqual(data["added_tags"], ["dog"]) - self.assertEqual(data["total_tags"], 2) - self.assertEqual( - self.api.db.get_prompt_by_id(prompt_id)["tags"], ["cat", "dog"] - ) - - async def test_string_tags_from_db_are_split(self): - prompt_id = self._save_prompt() - with patch.object( - self.api.db, - "get_prompt_by_id", - return_value={"id": prompt_id, "tags": "cat, dog"}, - ): - resp = await self._post_json( - "/prompt_manager/autotag/apply", - {"prompt_id": prompt_id, "tags": ["dog", "owl"]}, - ) - data = await resp.json() - self.assertEqual(data["added_tags"], ["owl"]) - self.assertEqual(data["total_tags"], 3) - - async def test_db_error_is_500(self): - with patch.object( - self.api.db, "get_prompt_by_id", side_effect=RuntimeError("db") - ): - resp = await self._post_json( - "/prompt_manager/autotag/apply", {"prompt_id": 1, "tags": ["a"]} - ) - self.assertEqual(resp.status, 500) - - class TestAutotagErrorBodiesHideServerPaths(AutotagAPITestCase): """Every autotag handler reports exceptions through the shared public helper.""" @@ -932,9 +680,12 @@ class TestAutotagErrorBodiesHideServerPaths(AutotagAPITestCase): await self._assert_hidden(resp) async def test_apply_error(self): - with patch.object(self.api.db, "get_prompt_by_id", side_effect=self._locked()): + image = self._make_image(self.output_dir) + digest = hashlib.sha256(image.read_bytes()).hexdigest() + with patch.object(self.api.db, "add_image_tags", side_effect=self._locked()): resp = await self._post_json( - "/prompt_manager/autotag/apply", {"prompt_id": 1, "tags": ["a"]} + "/prompt_manager/autotag/apply", + {"path": str(image), "sha256": digest, "tags": ["a"]}, ) await self._assert_hidden(resp) @@ -1008,10 +759,13 @@ class TestLegacyRoutesShareTheModelGate(AutotagAPITestCase): "POST", "/prompt_manager/autotag/download/gguf" ) ) - await self._busy( - await self.client.request( - "POST", "/prompt_manager/autotag/start?model_type=gguf" - ) + # The legacy batch is read as a stream: one error frame (#379). + resp = await self.client.request( + "POST", "/prompt_manager/autotag/start?model_type=gguf" + ) + self.assertEqual( + self._sse_events(await resp.text()), + [{"type": "error", "status": "error", "message": MODEL_BUSY_ERROR}], ) await self._busy( await self.client.request("POST", "/prompt_manager/autotag/unload") @@ -1026,7 +780,7 @@ class TestLegacyRoutesShareTheModelGate(AutotagAPITestCase): self.assertEqual(self.service.unload_calls, 0) self.assertEqual(self.service.get_loaded_model_type(), "wd14-vit") - # A refused start released the batch lock: the next start runs. + # A refused start left no job behind: the next start runs. resp = await self.client.request( "POST", "/prompt_manager/autotag/start?model_type=gguf" ) @@ -1238,7 +992,7 @@ if __name__ == "__main__": class TestGalleryReviewContract(AutotagAPITestCase): - async def test_root_relative_gallery_targets_generate_and_apply_to_their_prompt( + async def test_root_relative_gallery_targets_generate_and_apply_to_their_image( self, ): second = self.tmp_root / "second-output" @@ -1255,16 +1009,22 @@ class TestGalleryReviewContract(AutotagAPITestCase): self.assertEqual(response.status, 200) data = await response.json() self.assertEqual(data["prompt_id"], pid) - self.assertEqual( - self.service.generate_calls[-1][0], str((root / "same.png").resolve()) - ) response = await self._post_json( "/prompt_manager/autotag/apply", - {"prompt_id": data["prompt_id"], "tags": data["tags"]}, + { + "path": "same.png", + "root_index": index, + "sha256": data["sha256"], + "tags": data["tags"], + }, ) self.assertEqual(response.status, 200) + # The image's own tags; the prompt is never written (#379). + self.assertEqual(self.api.db.get_prompt_by_id(pid)["tags"], []) + locator = os.path.normcase(os.path.realpath(str(root / "same.png"))) self.assertEqual( - set(self.api.db.get_prompt_by_id(pid)["tags"]), set(data["tags"]) + self.api.db.read_image_tags(locator, data["sha256"])["tags"], + data["tags"], ) async def test_gallery_unlinked_image_returns_no_prompt_id(self): @@ -2058,108 +1818,6 @@ class TestTagOutputImagesJob(BatchJobTestCase): self.assertTrue(inside) -# The legacy batch's per-image frames over BatchJobTestCase's library, newest -# first, with every throttle check passing (pinned from main's generator). -GOLDEN_LEGACY_ROWS = [ - { - "type": "progress", - "progress": 24, - "status": "Processing 1/6...", - "processed": 1, - "tagged": 1, - "skipped": 0, - "skipped_outside_roots": 0, - }, - { - "type": "progress", - "progress": 38, - "status": "Skipping 2/6 (outside allowed directories)...", - "processed": 1, - "tagged": 1, - "skipped": 1, - "skipped_outside_roots": 1, - }, - { - "type": "progress", - "progress": 52, - "status": "Skipping 3/6 (file missing)...", - "processed": 1, - "tagged": 1, - "skipped": 2, - "skipped_outside_roots": 1, - }, - { - "type": "progress", - "progress": 66, - "status": "Processing 4/6...", - "processed": 2, - "tagged": 2, - "skipped": 2, - "skipped_outside_roots": 1, - }, - { - "type": "progress", - "progress": 80, - "status": "Skipping 5/6 (prompt already processed)...", - "processed": 2, - "tagged": 2, - "skipped": 3, - "skipped_outside_roots": 1, - }, - { - "type": "progress", - "progress": 95, - "status": "Skipping 6/6 (already tagged)...", - "processed": 2, - "tagged": 2, - "skipped": 4, - "skipped_outside_roots": 1, - }, -] - - -class TestLegacyBatchFrames(BatchJobTestCase): - - async def test_legacy_batch_events_unchanged(self): - """Golden frames of the legacy batch over the seeded library.""" - self._seed() - ticks = iter(range(1000)) # every throttle check sees a second pass - with patch("py.api.autotag_routes._monotonic", side_effect=lambda: next(ticks)): - resp = await self.client.post( - "/prompt_manager/autotag/start?model_type=wd14-vit&skip_tagged=true" - ) - frames = self._sse_events(await resp.text()) # the stream runs here - self.assertEqual( - frames, - [ - {"type": "progress", "progress": 0, "status": "Loading model..."}, - { - "type": "progress", - "progress": 5, - "status": "Model loaded. Fetching all images from database...", - }, - { - "type": "progress", - "progress": 10, - "status": "Found 6 images. Processing...", - }, - *GOLDEN_LEGACY_ROWS, - { - "type": "complete", - "progress": 100, - "processed": 2, - "tagged": 2, - "skipped": 4, - "errors": 0, - "skipped_outside_roots": 1, - "status": "Complete", - "model_status": "Model kept in memory", - "model_loaded": True, - }, - ], - ) - - class TestTagOutputImagesEdges(BatchJobTestCase): async def test_cancel_during_backup_loads_no_model(self): diff --git a/tests/test_autotag_legacy.py b/tests/test_autotag_legacy.py new file mode 100644 index 00000000..126ef670 --- /dev/null +++ b/tests/test_autotag_legacy.py @@ -0,0 +1,617 @@ +"""The legacy pages and the Tag image dialog tag images, never prompts (#379, PR 2). + +- ``POST /autotag/start`` (legacy admin and gallery batch) starts the + ``autotag`` job titled "Tag output images (legacy page)" and relays it as + the legacy frames; closing the stream cancels it. +- ``POST /autotag/single`` suggests on a capture with the saved cap and + answers the captured ``sha256``; ``existing: "skip"`` answers + ``already_tagged`` for an auto-tagged image without the model gate. +- ``POST /autotag/apply`` unions into that image's tags and marks it; a + ``prompt_id`` body is refused. +- A Tag image dialog save after Generate suggestions marks the image. + +No real model runs: the per-image fake from test_autotag_batch answers. +""" + +import asyncio +import hashlib +import json +import sqlite3 +import threading +from unittest.mock import patch + +from aiohttp import web + +try: + from test_autotag_batch import BatchTestCase, _absolute, _strings +except ImportError: + from tests.test_autotag_batch import BatchTestCase, _absolute, _strings + +from database.execution_history_schema import rotate_history_epoch +from py import model_gate +from py.api import autotag_routes, image_tag_jobs +from py.config import AutotagConfig + +START = "/prompt_manager/autotag/start" +SINGLE = "/prompt_manager/autotag/single" +APPLY = "/prompt_manager/autotag/apply" +SUGGEST = "/prompt_manager/image-annotations/suggest" +OBSERVE = "/prompt_manager/image-annotations/observe" +TAGS = "/prompt_manager/image-annotations/tags" +WAIT = 30.0 +LEGACY_TITLE = "Tag output images (legacy page)" + + +def frames_of(text): + return [ + json.loads(line[len("data: ") :]) + for line in text.splitlines() + if line.startswith("data: ") + ] + + +class LegacyTestCase(BatchTestCase): + async def legacy(self, query="model_type=wd14-vit&skip_tagged=true"): + response = await self.client.post(f"{START}?{query}") + return response, frames_of(await response.text()) + + def digest(self, path): + return hashlib.sha256((self.output_dir / path).read_bytes()).hexdigest() + + async def single(self, body): + response = await self.client.post(SINGLE, json=body) + return response.status, await response.json() + + async def apply(self, body): + response = await self.client.post(APPLY, json=body) + return response.status, await response.json() + + def snapshot_prompts(self): + conn = sqlite3.connect(self._temp_db.name) + try: + return [ + conn.execute(f"SELECT * FROM {t} ORDER BY 1,2").fetchall() + for t in ("prompts", "prompt_tags") + ] + finally: + conn.close() + + +class TestLegacyBatch(LegacyTestCase): + async def test_the_legacy_batch_tags_each_image_and_relays_frames(self): + prompt = self.prompt("one prompt", tags=["beach"]) + a = self.image("a.png", (1, 2, 3), sees=["bird"]) + b = self.image("b.png", (4, 5, 6), sees=["waves"]) + self.link(prompt, a) + self.link(prompt, b) + before = self.snapshot_prompts() + ticks = iter(range(1000)) + with patch.object( + autotag_routes, "_monotonic", side_effect=lambda: next(ticks) + ): + response, frames = await self.legacy() + self.assertEqual(response.status, 200) + self.assertEqual(self.stored("a.png")["tags"], ["bird"]) + self.assertEqual(self.stored("b.png")["tags"], ["waves"]) + self.assertEqual(self.snapshot_prompts(), before) + phases = [f["status"] for f in frames if f.get("progress") in (0, 5, 8)] + self.assertEqual( + phases, + sorted( + phases, + key=[ + "Taking a safety backup…", + "Loading model…", + "Listing output images…", + ].index, + ), + ) + self.assertIn("Listing output images…", phases) + rows = [f for f in frames if f["type"] == "progress" and "processed" in f] + self.assertTrue(rows) + for row in rows: + self.assertEqual( + set(row), + {"type", "progress", "status", "processed", "tagged", "skipped"}, + ) + self.assertTrue(10 <= row["progress"] <= 95, row) + self.assertEqual( + frames[-1], + { + "type": "complete", + "progress": 100, + "processed": 2, + "tagged": 2, + "skipped": 0, + "errors": 0, + "skipped_outside_roots": 0, + "status": "Complete", + "model_status": "Model kept in memory", + "model_loaded": True, + }, + ) + job = self.api._jobs.list()[0] + self.assertEqual( + (job.title, job.kind, job.state), (LEGACY_TITLE, "autotag", "finished") + ) + self.assertEqual([s for s in _strings(frames) if _absolute(s)], []) + + # Skip: the marked images are skipped before the model runs. + self.tagger.calls.clear() + _, again = await self.legacy() + self.assertEqual(again[-1]["skipped"], 2) + self.assertEqual(again[-1]["processed"], 0) + self.assertEqual(self.tagger.calls, []) + + async def test_no_still_images_and_keep_in_memory_false(self): + _, frames = await self.legacy("model_type=wd14-vit&keep_in_memory=false") + self.assertEqual(frames[-1]["status"], "No still images found") + self.assertEqual(frames[-1]["model_loaded"], False) + self.assertEqual(frames[-1]["model_status"], "Model unloaded") + self.assertEqual(self.tagger.unload_calls, 1) + + async def test_language_model_options_come_from_the_page_or_the_template(self): + self.image("a.png", (1, 1, 1)) + await self.legacy("model_type=gguf&prompt=Tags%20only") + self.assertEqual(self.tagger.calls[-1]["prompt"], "Tags only") + self.assertEqual(self.tagger.calls[-1]["limit"], 10) + self.image("b.png", (2, 2, 2)) + await self.relist() + await self.legacy("model_type=gguf&prompt=%20") + self.assertEqual(self.tagger.calls[-1]["prompt"], AutotagConfig.TEMPLATE) + self.assertEqual(self.tagger.custom_prompt, "legacy prompt") + + async def test_each_refusal_is_one_error_frame(self): + self.tagger.downloaded["wd14-vit"] = False + _, frames = await self.legacy() + self.assertEqual(len(frames), 1) + self.assertEqual(frames[0]["type"], "error") + self.assertEqual(frames[0]["status"], "error") + self.assertIn("WD14 ViT is not downloaded", frames[0]["message"]) + self.tagger.downloaded["wd14-vit"] = True + + _, frames = await self.legacy("model_type=wd14-vit&general_threshold=2") + self.assertEqual( + frames, + [ + { + "type": "error", + "status": "error", + "message": "general_threshold must be between 0 and 1", + } + ], + ) + + owner = model_gate.new_owner("image-annotations") + self.assertTrue(model_gate.try_acquire(owner)) + try: + _, frames = await self.legacy() + finally: + model_gate.release(owner) + self.assertEqual(frames[0]["message"], autotag_routes.MODEL_BUSY_ERROR) + self.assertEqual(self.api._jobs.list(), []) + self.assertIsNone(model_gate.holder()) + + response = await self.client.post(f"{START}?general_threshold=abc") + self.assertEqual(response.status, 400) + + async def test_busy_jobs_refuse_the_legacy_batch(self): + self.image("a.png", (1, 1, 1)) + entered, proceed = threading.Event(), threading.Event() + self.tagger.block = (entered, proceed) + status, data = await self.start_batch() + self.assertEqual(status, 202, data) + job = self.api._jobs.get(data["job"]["id"]) + try: + await self.wait_for(entered) + _, frames = await self.legacy() + self.assertEqual( + frames, + [ + { + "type": "error", + "status": "error", + "message": "Tag output images already running", + } + ], + ) + finally: + self.tagger.block = None + proceed.set() + await asyncio.wait_for(asyncio.shield(job.task), WAIT) + + # A second legacy batch keeps its JSON 409. + self.image("b.png", (2, 2, 2)) + await self.relist() + entered, proceed = threading.Event(), threading.Event() + self.tagger.block = (entered, proceed) + first = asyncio.ensure_future( + self.legacy("model_type=wd14-vit&skip_tagged=false") + ) + try: + await self.wait_for(entered) + second = await self.client.post(f"{START}?model_type=wd14-vit") + self.assertEqual(second.status, 409) + self.assertEqual( + await second.json(), + {"success": False, "error": "Auto-tagging is already running"}, + ) + finally: + self.tagger.block = None + proceed.set() + _, frames = await first + self.assertEqual(frames[-1]["type"], "complete") + + async def test_a_legacy_start_that_loses_the_race_gets_the_json_409(self): + # Two starts can both pass the title check before either registers its + # job; the loser's busy refusal names the legacy batch and answers the + # same JSON 409 as a second legacy batch. + refusal = image_tag_jobs.Refusal( + 409, + "busy", + f"{autotag_routes.LEGACY_BATCH_TITLE} already running", + web.json_response({}, status=409), + ) + + async def refused(**_kwargs): + return None, refusal + + with patch.object(self.api, "_start_autotag", side_effect=refused): + response = await self.client.post(f"{START}?model_type=wd14-vit") + self.assertEqual(response.status, 409) + self.assertEqual( + await response.json(), + {"success": False, "error": "Auto-tagging is already running"}, + ) + + async def test_closing_the_stream_cancels_the_job(self): + for n in range(3): + self.image(f"{n}.png", (n, n, n)) + real_write = web.StreamResponse.write + + async def flaky_write(response, data): + if b'"processed"' in data: + raise ConnectionResetError("Cannot write to closing transport") + return await real_write(response, data) + + with patch.object(web.StreamResponse, "write", flaky_write): + try: + response = await self.client.post(f"{START}?model_type=wd14-vit") + await response.read() + except Exception: # nosec B110 - the client sees the dropped stream + pass + job = self.api._jobs.list()[0] + await asyncio.wait_for(asyncio.shield(job.task), WAIT) + self.assertEqual(job.state, "cancelled") + self.assertLess(job.counters["tagged"], 3) + + async def test_a_quiet_job_repeats_its_last_frame(self): + self.image("a.png", (1, 1, 1)) + entered, proceed = threading.Event(), threading.Event() + self.tagger.block = (entered, proceed) + with patch.object(autotag_routes, "LEGACY_REPEAT_AFTER", 0.05): + run = asyncio.ensure_future(self.legacy()) + await self.wait_for(entered) + await asyncio.sleep(0.3) + self.tagger.block = None + proceed.set() + _, frames = await run + statuses = [f.get("status") for f in frames] + repeated = [s for s in statuses if s and statuses.count(s) > 1] + self.assertTrue(repeated, statuses) + self.assertEqual(frames[-1]["type"], "complete") + + async def test_a_cancel_from_activity_ends_with_a_cancelled_frame(self): + for n in range(2): + self.image(f"{n}.png", (n, n, n)) + entered, proceed = threading.Event(), threading.Event() + self.tagger.block = (entered, proceed) + run = asyncio.ensure_future(self.legacy()) + try: + await self.wait_for(entered) + job = self.api._jobs.active("autotag") + await self.client.post(f"/prompt_manager/jobs/{job.id}/cancel") + finally: + self.tagger.block = None + proceed.set() + _, frames = await run + self.assertEqual( + frames[-1], + { + "type": "cancelled", + "message": "Cancelled. Tags already added were kept.", + }, + ) + + +class TestLegacySingleAndApply(LegacyTestCase): + async def test_single_suggests_on_a_capture_and_apply_tags_only_that_image(self): + prompt = self.prompt("shared prompt", tags=["keep"]) + a = self.image("a.png", (1, 2, 3), sees=["bird", "sky"]) + b = self.image("b.png", (4, 5, 6), sees=["cat"]) + image_a = self.link(prompt, a) + self.link(prompt, b) + self.saved_set("b.png", b.read_bytes(), ["hand"]) + before = self.snapshot_prompts() + status, data = await self.single( + {"image_id": image_a, "model_type": "wd14-vit"} + ) + self.assertEqual(status, 200, data) + self.assertEqual(data["tags"], ["bird", "sky"]) + self.assertEqual(data["sha256"], self.digest("a.png")) + self.assertEqual(data["image_tags"], []) + self.assertEqual(data["prompt_id"], prompt) + self.assertEqual(self.tagger.custom_prompt, "legacy prompt") + # Inference read the private capture, not the file. + self.assertNotIn(str(self.output_dir), self.tagger.read_paths[-1]) + status, applied = await self.apply( + {"image_id": image_a, "sha256": data["sha256"], "tags": ["bird"]} + ) + self.assertEqual( + (status, applied), + (200, {"success": True, "added_tags": ["bird"], "total_tags": 1}), + ) + self.assertEqual(self.stored("a.png")["tags"], ["bird"]) + self.assertEqual(self.stored("b.png")["tags"], ["hand"]) + self.assertEqual(self.snapshot_prompts(), before) + self.assertEqual(self.marked(), {self.locator("a.png")}) + + async def test_skip_answers_already_tagged_without_the_gate(self): + a = self.image("a.png", (1, 2, 3), sees=["bird"]) + await self.run_batch() + self.tagger.calls.clear() + owner = model_gate.new_owner("image-tag-job") + self.assertTrue(model_gate.try_acquire(owner)) + try: + status, data = await self.single( + { + "path": "a.png", + "root_index": 0, + "existing": "skip", + "model_type": "wd14-vit", + } + ) + finally: + model_gate.release(owner) + self.assertEqual(status, 200, data) + self.assertEqual(data["already_tagged"], True) + self.assertEqual(data["image_tags"], ["bird"]) + self.assertEqual(self.tagger.calls, []) + # Hand tags alone do not count; merge always suggests. + b = self.image("b.png", (9, 9, 9)) + self.saved_set("b.png", b.read_bytes(), ["hand"]) + status, data = await self.single( + {"path": str(b), "existing": "skip", "model_type": "wd14-vit"} + ) + self.assertNotIn("already_tagged", data) + self.assertEqual(data["image_tags"], ["hand"]) + status, data = await self.single( + {"path": str(a), "existing": "merge", "model_type": "wd14-vit"} + ) + self.assertNotIn("already_tagged", data) + + async def test_single_uses_the_saved_cap_and_the_refusal_table(self): + self.image("a.png", (1, 2, 3)) + AutotagConfig.SUGGESTION_LIMIT = 3 + status, data = await self.single( + {"path": "a.png", "root_index": 0, "model_type": "wd14-vit"} + ) + self.assertEqual((status, len(data["tags"])), (200, 3)) + (self.output_dir / "broken.png").write_bytes(b"not an image") + await self.relist() + status, data = await self.single( + {"path": "broken.png", "root_index": 0, "model_type": "wd14-vit"} + ) + self.assertEqual((status, data["code"]), (415, "not-a-still")) + self.assertIn(self.locator("broken.png"), self.marked()) + self.tagger.downloaded["gguf"] = False + status, data = await self.single( + {"path": "a.png", "root_index": 0, "model_type": "gguf"} + ) + self.assertEqual((status, data["code"]), (409, "model-unavailable")) + self.api._capture_store().max_file = 16 + status, data = await self.single( + {"path": "a.png", "root_index": 0, "model_type": "wd14-vit"} + ) + self.assertEqual((status, data["code"]), (413, "too-large")) + self.api._capture_store().max_file = 2**20 + owner = model_gate.new_owner("image-annotations") + self.assertTrue(model_gate.try_acquire(owner)) + try: + status, data = await self.single( + {"path": "a.png", "root_index": 0, "model_type": "wd14-vit"} + ) + finally: + model_gate.release(owner) + self.assertEqual((status, data["code"]), (409, "model-busy")) + + async def test_apply_body_and_answer_table(self): + a = self.image("a.png", (1, 2, 3)) + digest = self.digest("a.png") + status, data = await self.apply({"prompt_id": 1, "tags": ["a"]}) + self.assertEqual( + (status, data["error"]), (400, "This page is out of date. Reload it.") + ) + status, _ = await self.apply( + {"path": "a.png", "root_index": 0, "sha256": "x", "tags": ["a"]} + ) + self.assertEqual(status, 400) + status, _ = await self.apply( + {"path": "a.png", "root_index": 0, "sha256": digest, "tags": ["bad,name"]} + ) + self.assertEqual(status, 400) + status, _ = await self.apply({"path": "a.png", "sha256": digest, "tags": []}) + self.assertEqual(status, 403) # a bare relative path is not inside a root + # An empty reviewed list only marks the image. + status, data = await self.apply({"path": str(a), "sha256": digest, "tags": []}) + self.assertEqual((status, data["added_tags"]), (200, [])) + self.assertEqual(self.marked(), {self.locator("a.png")}) + # The file changed since its tags were generated. + status, data = await self.apply( + {"path": str(a), "sha256": "0" * 64, "tags": ["x"]} + ) + self.assertEqual((status, data["code"]), (409, "file-changed")) + # Too many tags. + full = self.image("full.png", (7, 7, 7)) + self.saved_set("full.png", full.read_bytes(), [f"t{i}" for i in range(100)]) + status, data = await self.apply( + {"path": str(full), "sha256": self.digest("full.png"), "tags": ["one_more"]} + ) + self.assertEqual((status, data["code"]), (409, "tag-limit")) + # Gone. + status, data = await self.apply( + {"path": "gone.png", "root_index": 0, "sha256": digest, "tags": ["x"]} + ) + self.assertIn(status, (403, 404)) + + async def test_apply_after_a_restore_writes_nothing(self): + a = self.image("a.png", (1, 2, 3)) + digest = self.digest("a.png") + real = self.api.db.get_execution_epoch + + def stale_epoch(): + epoch = real() + conn = sqlite3.connect(self._temp_db.name) + try: + rotate_history_epoch(conn) + finally: + conn.close() + return epoch + + with patch.object(self.api.db, "get_execution_epoch", side_effect=stale_epoch): + status, data = await self.apply( + {"path": str(a), "sha256": digest, "tags": ["x"]} + ) + self.assertEqual((status, data["code"]), (409, "storage-changed")) + self.assertEqual(self.stored("a.png")["tags"], []) + + async def test_image_id_and_path_bodies_reach_the_same_bytes(self): + prompt = self.prompt("p") + a = self.image("sub/a.png", (1, 2, 3), sees=["x"]) + image_id = self.link(prompt, a) + _, by_id = await self.single({"image_id": image_id, "model_type": "wd14-vit"}) + _, by_path = await self.single( + {"path": "sub/a.png", "root_index": 0, "model_type": "wd14-vit"} + ) + self.assertEqual(by_id["sha256"], by_path["sha256"]) + status, _ = await self.apply( + {"image_id": image_id, "sha256": by_id["sha256"], "tags": ["x"]} + ) + self.assertEqual(status, 200) + status, data = await self.apply( + { + "path": "sub/a.png", + "root_index": 0, + "sha256": by_path["sha256"], + "tags": ["y"], + } + ) + self.assertEqual((status, data["total_tags"]), (200, 2)) + + +class TestDialogMarks(LegacyTestCase): + """DEC4: a Tag image dialog save after Generate suggestions marks the image.""" + + def target(self, path): + return { + "root_index": 0, + "root_scope": self.api._gallery_root_scope(self.output_dir, 0), + "path": path, + } + + async def observe(self, path): + response = await self.client.post(OBSERVE, json={"target": self.target(path)}) + data = await response.json() + self.assertEqual(response.status, 200, data) + return data + + async def suggest(self, observed, path): + response = await self.client.post( + SUGGEST, + json={ + "target": self.target(path), + "capture_token": observed["capture"]["token"], + "expected_epoch": observed["library_epoch"], + "model_type": "wd14-vit", + "options": {"general_threshold": 0.35, "character_threshold": 0.85}, + }, + ) + self.assertEqual(response.status, 200, await response.text()) + + async def save(self, observed, path, tags, revision=None): + response = await self.client.put( + TAGS, + json={ + "target": self.target(path), + "capture_token": observed["capture"]["token"], + "sha256": observed["capture"]["sha256"], + "expected_epoch": observed["library_epoch"], + "expected_tag_revision": ( + observed["tag_revision"] if revision is None else revision + ), + "tags": tags, + }, + ) + return response.status, await response.json() + + async def test_save_after_generate_marks_and_a_skip_batch_leaves_it(self): + self.image("d.png", (1, 2, 3), sees=["bird", "sky"]) + observed = await self.observe("d.png") + await self.suggest(observed, "d.png") + status, _ = await self.save(observed, "d.png", ["sky"]) # bird removed + self.assertEqual(status, 200) + self.assertEqual(self.marked(), {self.locator("d.png")}) + job = await self.run_batch(existing="skip") + self.assertEqual(self.rows(job, "skipped"), [("d.png", "Already auto-tagged")]) + self.assertEqual(self.stored("d.png")["tags"], ["sky"]) + + async def test_save_without_generate_does_not_mark(self): + self.image("e.png", (1, 2, 3)) + observed = await self.observe("e.png") + status, _ = await self.save(observed, "e.png", ["hand"]) + self.assertEqual(status, 200) + self.assertEqual(self.marked(), set()) + + async def test_a_conflict_then_a_saved_retry_marks(self): + f = self.image("f.png", (1, 2, 3)) + observed = await self.observe("f.png") + await self.suggest(observed, "f.png") + self.saved_set("f.png", f.read_bytes(), ["other"]) # another writer + status, data = await self.save(observed, "f.png", ["mine"]) + self.assertEqual((status, data["code"]), (409, "tag-conflict")) + self.assertEqual(self.marked(), set()) + status, _ = await self.save( + observed, "f.png", ["mine"], revision=data["tag_revision"] + ) + self.assertEqual(status, 200) + self.assertEqual(self.marked(), {self.locator("f.png")}) + + async def test_another_capture_does_not_inherit_the_mark(self): + self.image("g.png", (1, 2, 3)) + self.image("h.png", (4, 5, 6)) + first = await self.observe("g.png") + await self.suggest(first, "g.png") + second = await self.observe("h.png") + status, _ = await self.save(second, "h.png", ["x"]) + self.assertEqual(status, 200) + self.assertEqual(self.marked(), set()) + self.assertIn(first["capture"]["token"], self.api._suggested_tokens) + + async def test_gone_tokens_are_dropped_on_the_next_insert_without_touching_clocks( + self, + ): + self.image("i.png", (1, 2, 3)) + self.image("j.png", (4, 5, 6)) + first = await self.observe("i.png") + await self.suggest(first, "i.png") + store = self.api._capture_store() + store.release(first["capture"]["token"]) + second = await self.observe("j.png") + seen = [] + real_get = store.get + with patch.object( + store, "get", side_effect=lambda t: seen.append(t) or real_get(t) + ): + await self.suggest(second, "j.png") + self.assertEqual(seen, []) + self.assertEqual(self.api._suggested_tokens, {second["capture"]["token"]}) diff --git a/tests/test_database.py b/tests/test_database.py index 09fe6e97..92935ce1 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -1398,50 +1398,6 @@ class TestModelHousekeeping(DatabaseTestCase): self.assertIsNot(self.db.model.get_connection(), failing) -class TestAutotagBatchCandidates(DatabaseTestCase): - """#328: the batch's candidate read never loads workflow documents.""" - - def test_autotag_candidates_skip_workflow_data(self): - from utils.hashing import generate_prompt_hash - - first = self.db.save_prompt( - text="a", tags=["one", "two"], prompt_hash=generate_prompt_hash("a") - ) - second = self.db.save_prompt(text="b", prompt_hash=generate_prompt_hash("b")) - old = self.db.link_image_to_prompt( - first, "/out/a.png", {"workflow": {"big": "x" * 1000}} - ) - new = self.db.link_image_to_prompt(second, "/out/b.png") - statements = [] - conn = self.db.model.get_connection() # this thread's own connection - conn.set_trace_callback(statements.append) - try: - rows = self.db.autotag_batch_candidates() - finally: - conn.set_trace_callback(None) - self.assertEqual( - rows[0], - { - "id": new, - "image_path": "/out/b.png", - "prompt_id": second, - "prompt_tags": [], - }, - ) - self.assertEqual( - {**rows[1], "prompt_tags": sorted(rows[1]["prompt_tags"])}, - { - "id": old, - "image_path": "/out/a.png", - "prompt_id": first, - "prompt_tags": ["one", "two"], - }, - ) - self.assertEqual(len(rows), 2) - self.assertTrue(statements) - self.assertFalse(any("workflow_data" in sql for sql in statements)) - - if __name__ == "__main__": unittest.main() diff --git a/tests/test_job_admission.py b/tests/test_job_admission.py index 44d47ff4..4e8e5584 100644 --- a/tests/test_job_admission.py +++ b/tests/test_job_admission.py @@ -535,7 +535,9 @@ class TestHeldJobStates(_LegacyHoldCase): self.assertIn("event: error", text) self.assertEqual(await self._listed(), [("thumbnails", "failed")]) - async def test_legacy_autotag_that_ends_in_error_is_failed(self): + async def test_legacy_autotag_refused_at_start_leaves_no_job(self): + # #379: the legacy batch is the autotag job; a model that is not + # downloaded refuses the start (one error frame) and registers nothing. service = MagicMock() service.get_models_status.return_value = {} # model not downloaded with patch("py.autotag.get_autotag_service", return_value=service): @@ -545,7 +547,7 @@ class TestHeldJobStates(_LegacyHoldCase): text = await resp.text() self.assertEqual(resp.status, 200) self.assertIn('"type": "error"', text) - self.assertEqual(await self._listed(), [("autotag", "failed")]) + self.assertEqual(await self._listed(), []) if __name__ == "__main__": diff --git a/web/admin.html b/web/admin.html index dee1d8a9..f80b3565 100644 --- a/web/admin.html +++ b/web/admin.html @@ -1187,14 +1187,13 @@ No sentences, no commentary, no captions. Only tags. Keep length short but descr <div class="space-y-1.5"> <label class="flex items-center p-2.5 bg-pm-input rounded-pm-md cursor-pointer hover:bg-pm-hover border border-pm"> <input type="radio" name="autoTagMode" value="skip" checked class="w-3.5 h-3.5 accent-[var(--pm-accent)]"> - <span class="ml-2.5 text-pm text-xs">Skip images with existing tags</span> + <span class="ml-2.5 text-pm text-xs">Skip images already auto-tagged</span> </label> <label class="flex items-center p-2.5 bg-pm-input rounded-pm-md cursor-pointer hover:bg-pm-hover border border-pm"> <input type="radio" name="autoTagMode" value="retag" class="w-3.5 h-3.5 accent-[var(--pm-accent)]"> <span class="ml-2.5 text-pm text-xs">Re-tag all images</span> </label> </div> - <p class="text-pm-muted text-[11px] mt-1">Note: "auto-scanned" is not counted as a real tag</p> </div> <!-- Action Buttons --> @@ -1293,6 +1292,7 @@ No sentences, no commentary, no captions. Only tags. Keep length short but descr <div class="w-full bg-pm-input rounded-full h-1.5"> <div id="autoTagProgressBar" class="bg-pm-accent h-1.5 rounded-full transition-all duration-300" style="width: 0%"></div> </div> + <p id="autoTagCloseNote" class="text-pm-muted text-[11px] mt-1.5">Closing this window stops tagging. Tags already added are kept. Prompt Manager's Activity page shows the time left.</p> </div> <div class="grid grid-cols-3 gap-3 mb-4"> @@ -1332,7 +1332,7 @@ No sentences, no commentary, no captions. Only tags. Keep length short but descr <!-- Re-tag Confirmation Modal --> <div id="retagConfirmModal" class="fixed inset-0 bg-black/60 backdrop-blur-[2px] hidden items-center justify-center z-50"> <div class="bg-pm-surface rounded-pm-lg p-4 max-w-lg w-full mx-4 border border-pm shadow-pm"> - <h3 class="text-sm font-semibold text-pm mb-3">Image Already Tagged</h3> + <h3 class="text-sm font-semibold text-pm mb-3">Image Already Auto-tagged</h3> <div class="mb-3"> <img id="retagPreviewImage" src="" alt="Preview" class="w-full h-40 object-contain bg-pm-primary rounded-pm-md"> @@ -1345,7 +1345,7 @@ No sentences, no commentary, no captions. Only tags. Keep length short but descr </div> </div> - <p class="text-pm-secondary text-xs mb-4">This image already has tags. Would you like to generate new tags?</p> + <p class="text-pm-secondary text-xs mb-4">A tagging model already tagged this image. Generate new tags?</p> <div class="flex gap-2 justify-end"> <button id="retagSkipBtn" diff --git a/web/gallery.html b/web/gallery.html index 00b890a5..5613555f 100644 --- a/web/gallery.html +++ b/web/gallery.html @@ -682,6 +682,7 @@ No sentences, no commentary, no captions. Only tags. Keep length short but descr <div class="w-full bg-pm-input rounded-full h-2"> <div id="autoTagProgressBar" class="bg-pm-accent h-2 rounded-full transition-all duration-300" style="width: 0%"></div> </div> + <p id="autoTagCloseNote" class="text-pm-muted text-xs mt-1.5">Closing this window stops tagging. Tags already added are kept. Prompt Manager's Activity page shows the time left.</p> </div> <!-- Stats --> diff --git a/web/js/admin.js b/web/js/admin.js index 200e462a..e48c0985 100644 --- a/web/js/admin.js +++ b/web/js/admin.js @@ -3801,7 +3801,7 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`; } else if (data.type === 'complete') { this.autoTagState.eventSource.close(); this.hideModal("autoTagProgressModal"); - this.showNotification(`Auto tagging complete! Applied tags to ${data.tagged || 0} prompts.`, 'success'); + this.showNotification(ImageHelpers.autotagCompleteMessage(data.tagged), 'success'); this.refreshList(); // Refresh the prompt list } else if (data.type === 'error') { this.autoTagState.eventSource.close(); @@ -3819,7 +3819,7 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`; console.error('EventSource readyState:', this.autoTagState.eventSource.readyState); this.autoTagState.eventSource.close(); this.hideModal("autoTagProgressModal"); - this.showNotification('Connection lost during auto tagging', 'error'); + this.showNotification((error && error.serverMessage) || 'Connection lost during auto tagging', 'error'); }; } catch (error) { console.error('Auto tag error:', error); @@ -3894,10 +3894,14 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`; } } - // Check if tags array has "real" tags (excluding auto-scanned and prepend:* metadata) - getRealTags(tags) { - if (!tags || !Array.isArray(tags)) return []; - return tags.filter(tag => tag !== 'auto-scanned' && !tag.startsWith('prepend:') && !tag.startsWith('append:')); + /** POST /autotag/single for `image` (#379: `existing` "skip" answers already_tagged for an auto-tagged image). */ + async requestReviewTags(image, existing) { + const response = await ApiClient.fetchUrl('/prompt_manager/autotag/single', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(ImageHelpers.autotagSingleBody(image, { ...this.autoTagState, existing })) + }); + return response.json(); } // Show the re-tag confirmation modal @@ -3949,26 +3953,48 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`; // always provides a servable url; never rebuild it from the path. const imageUrl = this.getImageUrl(image); - // Check if image already has real tags (excluding auto-scanned) - const realTags = this.getRealTags(image.prompt_tags); + this.autoTagState.currentImage = image; + this.autoTagState.currentSha256 = null; + this.autoTagState.tagsExpanded = false; // Reset accordion for new image - if (realTags.length > 0) { - // Skip if "Skip All Tagged" was selected - if (this.autoTagState.skipAllTagged) { - this.autoTagState.reviewIndex++; - return this.loadNextReviewImage(); + let data; + try { + // Each image is first asked with "skip": an image a tagging model already tagged answers + // already_tagged (its image tags, never its prompt's) without running the model (#379). + data = await this.requestReviewTags(image, 'skip'); + if (data.success && data.already_tagged) { + if (this.autoTagState.skipAllTagged) { + this.autoTagState.reviewIndex++; + return this.loadNextReviewImage(); + } + const choice = await this.showRetagConfirmation(image, imageUrl, data.image_tags || []); + if (choice === 'skip' || choice === 'skipAll') { + this.autoTagState.reviewIndex++; + return this.loadNextReviewImage(); + } + // choice === 'retag': generate new tags for it + this.showReviewImage(image, imageUrl); + data = await this.requestReviewTags(image, 'merge'); + } else { + this.showReviewImage(image, imageUrl); } - - // Show confirmation modal - const choice = await this.showRetagConfirmation(image, imageUrl, realTags); - - if (choice === 'skip' || choice === 'skipAll') { - this.autoTagState.reviewIndex++; - return this.loadNextReviewImage(); + if (data.success) { + this.autoTagState.currentTags = data.tags; + this.autoTagState.currentSha256 = data.sha256 || null; + this.renderReviewTags(); + } else { + document.getElementById('reviewTagsVisible').innerHTML = + `<div class="text-pm-error">Error: ${escapeHtml(data.error)}</div>`; } - // choice === 'retag' - continue with tagging + } catch (error) { + console.error('Error generating tags:', error); + this.showReviewImage(image, imageUrl); + document.getElementById('reviewTagsVisible').innerHTML = + '<div class="text-pm-error">Failed to generate tags</div>'; } + } + showReviewImage(image, imageUrl) { document.getElementById('reviewImage').src = imageUrl; // The record's file name, else the last segment of its path (#328: `filename` was never declared). const filename = image.filename || String(image.image_path || image.path || '').split(/[\\/]/).pop(); @@ -3977,37 +4003,6 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`; document.getElementById('reviewTagsHidden').innerHTML = ''; document.getElementById('reviewTagsAccordion').classList.add('hidden'); document.getElementById('reviewTagsToggle').classList.add('hidden'); - - // Store prompt_id from database for applying tags later - this.autoTagState.currentPromptId = image.prompt_id; - this.autoTagState.tagsExpanded = false; // Reset accordion for new image - - try { - // Review rows come from /images/all (database ids), so this resolves to image_id - const requestBody = ImageHelpers.autotagSingleBody(image, this.autoTagState); - - const response = await ApiClient.fetchUrl('/prompt_manager/autotag/single', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(requestBody) - }); - - const data = await response.json(); - - if (data.success) { - this.autoTagState.currentTags = data.tags; - // Use prompt_id from database, fallback to API response - this.autoTagState.currentPromptId = image.prompt_id || data.prompt_id; - this.renderReviewTags(); - } else { - document.getElementById('reviewTagsVisible').innerHTML = - `<div class="text-pm-error">Error: ${escapeHtml(data.error)}</div>`; - } - } catch (error) { - console.error('Error generating tags:', error); - document.getElementById('reviewTagsVisible').innerHTML = - '<div class="text-pm-error">Failed to generate tags</div>'; - } } /** /prompt_manager/images/all is capped per call; collect every page. */ @@ -4126,15 +4121,9 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`; } async applyReviewTags() { - if (this.autoTagState.currentTags.length === 0) { - this.skipReviewImage(); - return; - } - - const promptId = this.autoTagState.currentPromptId; - - if (!promptId) { - this.showNotification('No linked prompt found for this image', 'warning'); + // #379: the reviewed tags go to this image (linked or not), never its prompt; an empty + // list still marks the image as reviewed by a tagging model. + if (!this.autoTagState.currentSha256) { this.skipReviewImage(); return; } @@ -4143,10 +4132,8 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`; const response = await ApiClient.fetchUrl('/prompt_manager/autotag/apply', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - prompt_id: promptId, - tags: this.autoTagState.currentTags - }) + body: JSON.stringify(ImageHelpers.autotagApplyBody( + this.autoTagState.currentImage, this.autoTagState.currentSha256, this.autoTagState.currentTags)) }); const data = await response.json(); diff --git a/web/js/gallery.js b/web/js/gallery.js index 51838c69..0c4e90d2 100644 --- a/web/js/gallery.js +++ b/web/js/gallery.js @@ -3560,19 +3560,24 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`; } else if (data.type === 'complete') { this.autoTagState.eventSource.close(); this.hideModal("autoTagProgressModal"); - this.showNotification(`Auto tagging complete! Applied tags to ${data.tagged || 0} prompts.`, 'success'); + this.showNotification(ImageHelpers.autotagCompleteMessage(data.tagged), 'success'); this.loadImages(); // Refresh the gallery } else if (data.type === 'error') { this.autoTagState.eventSource.close(); this.hideModal("autoTagProgressModal"); this.showNotification(`Auto tag error: ${data.message}`, 'error'); + } else if (data.type === 'cancelled') { + // Cancelled in Prompt Manager's Activity (#379): as its own cancel. + this.autoTagState.eventSource.close(); + this.hideModal("autoTagProgressModal"); + this.showNotification('Auto tagging cancelled', 'info'); } }; - this.autoTagState.eventSource.onerror = () => { + this.autoTagState.eventSource.onerror = (error) => { this.autoTagState.eventSource.close(); this.hideModal("autoTagProgressModal"); - this.showNotification('Connection lost during auto tagging', 'error'); + this.showNotification((error && error.serverMessage) || 'Connection lost during auto tagging', 'error'); }; } catch (error) { console.error('Auto tag error:', error); @@ -3629,6 +3634,8 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`; this.reviewRequest = request; this.autoTagState.currentTags = []; this.autoTagState.reviewPromptId = null; + this.autoTagState.reviewImage = null; + this.autoTagState.reviewSha256 = null; if (this.autoTagState.reviewIndex >= this.autoTagState.reviewImages.length) { this.hideModal("autoTagReviewModal"); this.showNotification('Review complete!', 'success'); @@ -3643,8 +3650,9 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`; document.getElementById('reviewTagsContainer').innerHTML = '<div class="text-pm-secondary">Generating tags...</div>'; try { - // Gallery entries come from /images/output: id is a path digest, so address by path - const requestBody = ImageHelpers.autotagSingleBody(image, this.autoTagState); + // Gallery entries come from /images/output: id is a path digest, so address by path. + // The gallery review always suggests (it never skipped tagged images, #379). + const requestBody = ImageHelpers.autotagSingleBody(image, { ...this.autoTagState, existing: 'merge' }); const response = await ApiClient.fetchUrl('/prompt_manager/autotag/single', { method: 'POST', @@ -3657,6 +3665,8 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`; if (this.reviewRequest !== request) return; if (data.success) { this.autoTagState.reviewPromptId = data.prompt_id ?? null; + this.autoTagState.reviewImage = image; + this.autoTagState.reviewSha256 = data.sha256 || null; this.autoTagState.currentTags = data.tags; this.renderReviewTags(); } else { @@ -3697,16 +3707,12 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`; } async applyReviewTags() { - if (this.autoTagState.currentTags.length === 0) { + // #379: the reviewed tags go to this image (linked or not), never its prompt; an empty + // list still marks the image as reviewed by a tagging model. + if (!this.autoTagState.reviewSha256) { this.skipReviewImage(); return; } - - const promptId = this.autoTagState.reviewPromptId; - if (!promptId) { - this.showNotification('This image has no linked prompt. Scan the output folder first.', 'warning'); - return; - } const request = this.reviewRequest; if (this.reviewApplying === request) return; this.reviewApplying = request; @@ -3715,10 +3721,8 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`; const response = await ApiClient.fetchUrl('/prompt_manager/autotag/apply', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - prompt_id: promptId, - tags: this.autoTagState.currentTags - }) + body: JSON.stringify(ImageHelpers.autotagApplyBody( + this.autoTagState.reviewImage, this.autoTagState.reviewSha256, this.autoTagState.currentTags)) }); const data = await response.json(); diff --git a/web/js/image-helpers.js b/web/js/image-helpers.js index fc019d7c..7ab5047f 100644 --- a/web/js/image-helpers.js +++ b/web/js/image-helpers.js @@ -70,10 +70,28 @@ * path digest as id, which would 404, so those are addressed by path. * * @param {{id?: unknown, path?: unknown}} image - * @param {{modelType: string, generalThreshold?: number, characterThreshold?: number, prompt?: string}} settings + * `existing` (#379): "skip" answers `already_tagged` for an image a model + * already tagged; anything else is "merge" (always suggest). + * + * @param {{id?: unknown, path?: unknown}} image + * @param {{modelType: string, generalThreshold?: number, characterThreshold?: number, prompt?: string, existing?: string}} settings * @returns {object} JSON-serialisable request body */ function autotagSingleBody(image, settings) { + const target = autotagTarget(image); + const existing = settings.existing === "skip" ? "skip" : "merge"; + const modelType = String(settings.modelType || ""); + const params = modelType.startsWith("wd14") + ? { + general_threshold: settings.generalThreshold, + character_threshold: settings.characterThreshold, + } + : { prompt: settings.prompt }; + return { ...target, model_type: modelType, existing, ...params }; + } + + /** The image a legacy single/apply body names: its row id, or its path (and root). */ + function autotagTarget(image) { const entry = image || {}; const target = typeof entry.path === "string" && entry.path && !Number.isInteger(entry.id) @@ -82,14 +100,22 @@ if ("path" in target && Number.isInteger(entry.root_index)) { target.root_index = entry.root_index; } - const modelType = String(settings.modelType || ""); - const params = modelType.startsWith("wd14") - ? { - general_threshold: settings.generalThreshold, - character_threshold: settings.characterThreshold, - } - : { prompt: settings.prompt }; - return { ...target, model_type: modelType, ...params }; + return target; + } + + /** + * Body for POST /prompt_manager/autotag/apply (#379): the image, the + * `sha256` /autotag/single answered and the reviewed tags. The tags go to + * that image only, never to its prompt; an empty list marks it reviewed. + */ + function autotagApplyBody(image, sha256, tags) { + return { ...autotagTarget(image), sha256: String(sha256 || ""), tags: Array.isArray(tags) ? [...tags] : [] }; + } + + /** The legacy batch's completion toast (#379): images, never prompts. */ + function autotagCompleteMessage(tagged) { + const count = Number.isSafeInteger(tagged) && tagged > 0 ? tagged : 0; + return `Auto tagging complete! Tagged ${count} ${count === 1 ? "image" : "images"}.`; } /** @@ -122,7 +148,15 @@ } } - const api = { formatImageCaption, workflowDocumentHtml, autotagSingleBody, fallbackImageSource, metadataLookupUrl }; + const api = { + formatImageCaption, + workflowDocumentHtml, + autotagSingleBody, + autotagApplyBody, + autotagCompleteMessage, + fallbackImageSource, + metadataLookupUrl, + }; if (typeof module !== "undefined" && module.exports) module.exports = api; else root.ImageHelpers = api; })(typeof window !== "undefined" ? window : globalThis); diff --git a/web/js/sse-stream.js b/web/js/sse-stream.js index 9ee98ba0..ee3372d9 100644 --- a/web/js/sse-stream.js +++ b/web/js/sse-stream.js @@ -59,6 +59,20 @@ * @returns {{onmessage: Function|null, onerror: Function|null, onclose: Function|null, * readyState: number, close: Function, CONNECTING: number, OPEN: number, CLOSED: number}} */ + // A non-2xx answer as an Error carrying its status and, when the body is + // the API's JSON {error}, that sentence as serverMessage for the page. + async function httpError(response) { + const error = new Error(`HTTP ${response.status}`); + error.status = response.status; + try { + const data = await response.json(); + if (data && typeof data.error === "string" && data.error) error.serverMessage = data.error; + } catch (_) { + // Not JSON: the status alone. + } + return error; + } + function connect(url, { method = "GET", body, fetchImpl } = {}) { const doFetch = fetchImpl || ((...args) => globalThis.fetch(...args)); const controller = new AbortController(); @@ -103,7 +117,7 @@ if (body !== undefined) init.body = body; const requestUrl = root.ApiClient ? root.ApiClient.resolveUrl(url) : url; const response = await doFetch(requestUrl, init); - if (!response.ok) throw new Error(`HTTP ${response.status}`); + if (!response.ok) throw await httpError(response); if (source.readyState === CLOSED) return; source.readyState = OPEN; From 44177974cb2ea12d4ea6473783f023e34ab474e9 Mon Sep 17 00:00:00 2001 From: Vito <vito.sansevero@gmail.com> Date: Wed, 28 Oct 2026 00:32:30 +0000 Subject: [PATCH 03/11] fix(scan): read sampler-less graphs from the one wired text, retire the heuristic (#252) (#397) * fix(scan): read sampler-less graphs from the one wired text, retire the heuristic (#252) A graph where no node takes a positive or negative input fell back to the pre-redesign _extract_readable_prompt heuristic: the last node with any string text input won, and with none it read stale UI workflow widgets. It stored LoRA syntax, display-node reports, install notes, model file names and numbers. The scan now asks sampler_less_prompt: only the executed API graph counts; a candidate is a node with a text input that another node consumes; blank and numeric texts are never prompts; this repository's own positive nodes are authoritative; one distinct text links, several (or an unreadable wired text that no candidate builds) stay unlinked and try the recorded-run attribution. The heuristic and its helpers are removed; A1111 files read the parsed positive prompt directly. * fix(scan): refuse loader side outputs, seeded templates and keep A1111 text (#252) Review findings: an unreadable text link counts as built from a candidate only through that node's first output, so a LoRA loader's trigger-word output is unknown, not its text. A node with a seed input holds a template, never the prompt that ran. A sampler-less graph the rule cannot decide still links the A1111 positive a metadata saver wrote beside it, as the old scan did. Only plain numbers are numeric; words such as nan or inf are prompts. * fix(scan): linear plain-number check for sampler-less texts (#252) The numeric pattern had overlapping quantifiers that backtrack quadratically on a long digit run ending in a non-digit; it now has none. --- py/api/__init__.py | 193 ------------------------------ py/api/admin.py | 27 +++-- tests/test_api_admin.py | 34 ++---- tests/test_api_core.py | 92 -------------- tests/test_pair_scan.py | 159 +++++++++++++++++++++++- tests/test_sampler_less_prompt.py | 112 +++++++++++++++++ utils/prompt_pair_resolver.py | 112 +++++++++++++++++ 7 files changed, 414 insertions(+), 315 deletions(-) create mode 100644 tests/test_sampler_less_prompt.py diff --git a/py/api/__init__.py b/py/api/__init__.py index 485578c7..def2186f 100644 --- a/py/api/__init__.py +++ b/py/api/__init__.py @@ -1107,196 +1107,3 @@ class PromptManagerAPI( result["parameters"][field] = metadata[field] return result - - def _extract_readable_prompt(self, parsed_data): - """Extract human-readable prompt text from ComfyUI/A1111 data.""" - - def safe_to_string(value): - if isinstance(value, str): - return value - elif isinstance(value, list): - return " ".join(str(item) for item in value if item) - elif value is not None: - return str(value) - return None - - # First check if we already extracted a positive prompt (A1111 format) - if parsed_data.get("positive_prompt"): - return parsed_data["positive_prompt"] - - # Check if prompt is already a string - if isinstance(parsed_data.get("prompt"), str): - return parsed_data["prompt"] - - # Check if prompt is a simple value that can be converted - if parsed_data.get("prompt") and not isinstance( - parsed_data.get("prompt"), dict - ): - return safe_to_string(parsed_data["prompt"]) - - prompt_data = parsed_data.get("prompt") - if isinstance(prompt_data, dict): - # Use the enhanced logic from parse-metadata.py - positive_prompt = self._extract_positive_prompt_from_comfyui_data( - prompt_data - ) - if positive_prompt: - return positive_prompt - - # Check workflow data if available - workflow_data = parsed_data.get("workflow") - if isinstance(workflow_data, dict): - positive_prompt = self._extract_positive_prompt_from_comfyui_data( - workflow_data - ) - if positive_prompt: - return positive_prompt - - # Check parameters for positive prompt - if parsed_data.get("parameters", {}).get("positive"): - return safe_to_string(parsed_data["parameters"]["positive"]) - - return None - - def _get_node_inputs(self, node): - """Safely get inputs from a node, handling both dict and list formats. - - Old format: inputs is a dict with direct key-value pairs - inputs = {"text": "my prompt", "seed": 123} - - New format: inputs is a list of connection objects - inputs = [ - {"name": "text", "type": "STRING", "link": null, - "widget": {"name": "text"}}, - {"name": "clip", "type": "CLIP", "link": 11} - ] - """ - if not isinstance(node, dict): - return {} - - inputs = node.get("inputs", {}) - - # If inputs is already a dict, return it - if isinstance(inputs, dict): - return inputs - - # If inputs is a list, convert to dict format - if isinstance(inputs, list): - inputs_dict = {} - for input_item in inputs: - if isinstance(input_item, dict) and "name" in input_item: - name = input_item["name"] - inputs_dict[name] = input_item - return inputs_dict - - return {} - - def _find_text_in_node(self, node): - """Try to find text content in a node using various strategies. - - Handles both old and new workflow formats. - """ - if not isinstance(node, dict): - return None - - # Strategy 1: Check normalized inputs for 'text' field - inputs = self._get_node_inputs(node) - if "text" in inputs and isinstance(inputs["text"], str): - return inputs["text"] - - # Strategy 2: For text encoder nodes, check widgets_values - class_type = node.get("class_type", node.get("type", "")) - text_encoder_types = [ - "CLIPTextEncode", - "CLIPTextEncodeSDXL", - "CLIPTextEncodeSDXLRefiner", - "CLIPTextEncodeFlux", - "PromptManager", - "PromptManagerText", - "BNK_CLIPTextEncoder", - "Text Encoder", - "CLIP Text Encode", - ] - - if any( - encoder_type.lower() in class_type.lower() - for encoder_type in text_encoder_types - ): - widgets_values = node.get("widgets_values", []) - if widgets_values and len(widgets_values) > 0: - if isinstance(widgets_values[0], str) and widgets_values[0].strip(): - return widgets_values[0] - - return None - - def _extract_positive_prompt_from_comfyui_data(self, data): - """Extract positive prompt from ComfyUI data. - - Handles both old and new formats. - """ - if not isinstance(data, dict): - return None - - # Build nodes dictionary - nodes_by_id = {} - if "nodes" in data: - # Handle nodes array format - for node in data["nodes"]: - if isinstance(node, dict): - nid = node.get("id") - if nid is not None: - nodes_by_id[nid] = node - else: - # Handle flat dictionary format (node_id -> node_data) - for nid_str, node in data.items(): - try: - nid = int(nid_str) - except (ValueError, TypeError): - nid = nid_str - if isinstance(node, dict): - if "id" in node: - nid = node["id"] - nodes_by_id[nid] = node - - if not nodes_by_id: - return None - - # First, try to find positive/negative connection pattern - pos_id = None - for node in nodes_by_id.values(): - if isinstance(node, dict): - inputs = self._get_node_inputs(node) - if "positive" in inputs and "negative" in inputs: - try: - pos_input = inputs["positive"] - if isinstance(pos_input, list) and len(pos_input) > 0: - pos_id = int(pos_input[0]) - break - except (ValueError, TypeError, IndexError): - continue - - # Get text from the positive node - if pos_id is not None and pos_id in nodes_by_id: - text_val = self._find_text_in_node(nodes_by_id[pos_id]) - if text_val: - return text_val - - # Fallback: find any text encoder node with text content - text_nodes = [] - for node in nodes_by_id.values(): - if isinstance(node, dict): - text_val = self._find_text_in_node(node) - if text_val: - # Try to determine if this is positive or negative - node_title = node.get("title", "").lower() - if "neg" not in node_title and "negative" not in node_title: - # Prioritize non-negative prompts - text_nodes.insert(0, text_val) - else: - text_nodes.append(text_val) - - # Return the first positive-looking prompt - if text_nodes: - return text_nodes[0] - - return None diff --git a/py/api/admin.py b/py/api/admin.py index 4ebf8eea..ec3f6db7 100644 --- a/py/api/admin.py +++ b/py/api/admin.py @@ -47,7 +47,7 @@ try: extract_generation_params, parse_page_json, ) - from ...utils.prompt_pair_resolver import resolve_prompt_pair + from ...utils.prompt_pair_resolver import resolve_prompt_pair, sampler_less_prompt from ...utils.review_capture import reviewable except ImportError: from database.pair_ingestion import blank_positive, ingest_resolved_pair @@ -62,7 +62,7 @@ except ImportError: extract_generation_params, parse_page_json, ) - from utils.prompt_pair_resolver import resolve_prompt_pair + from utils.prompt_pair_resolver import resolve_prompt_pair, sampler_less_prompt from utils.review_capture import reviewable @@ -1754,13 +1754,24 @@ class AdminRoutesMixin: canonical.get("parameters"), str ): return resolution["state"] + elif canonical.get("prompt"): + # No node takes a positive or negative input (a guider or wrapper + # graph): the one wired text, never a guess (#252). + found = sampler_less_prompt(canonical["prompt"]) + prompt_text = found["text"] + if prompt_text is None and isinstance(canonical.get("parameters"), str): + # A metadata saver also wrote A1111 text: its positive prompt. + parsed = self._parse_comfyui_prompt(metadata)["positive_prompt"] + prompt_text = parsed.strip() if parsed and parsed.strip() else None + if prompt_text is None: + linked = self._link_recorded_run( + media_file, canonical, detailed, history + ) + if linked is not None: + return linked + return "unresolved" if found["state"] == "ambiguous" else "found" else: - parsed = self._parse_comfyui_prompt(metadata) - if not (parsed.get("prompt") or parsed.get("parameters")): - return None - prompt_text = self._extract_readable_prompt(parsed) - if prompt_text is not None and not isinstance(prompt_text, str): - prompt_text = str(prompt_text) + prompt_text = self._parse_comfyui_prompt(metadata)["positive_prompt"] if not (prompt_text and prompt_text.strip()): return "found" prompt_text = prompt_text.strip() diff --git a/tests/test_api_admin.py b/tests/test_api_admin.py index a5b85ba0..120c1e93 100644 --- a/tests/test_api_admin.py +++ b/tests/test_api_admin.py @@ -846,6 +846,14 @@ class TestSettingsReadMonitorThroughModule(AdminAPITestCase): self.assertEqual(data["settings"]["gallery_root_paths"], ["output"]) +def _wired_prompt(text): + """A sampler-less graph whose one encoder feeds a guider (#252).""" + return { + "1": {"class_type": "CLIPTextEncode", "inputs": {"text": text}}, + "2": {"class_type": "BasicGuider", "inputs": {"conditioning": ["1", 0]}}, + } + + def _raise(*args, **kwargs): raise RuntimeError("boom") @@ -2186,11 +2194,7 @@ class TestOutputScanInternals(AdminAPITestCase): def test_ingest_outcomes(self): f = Path("gen.png") - meta = { - "prompt": json.dumps( - {"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "a cat"}}} - ) - } + meta = {"prompt": json.dumps(_wired_prompt("a cat"))} self.assertIsNone(self.api._ingest_scanned_file(f, {})) self.assertIsNone(self.api._ingest_scanned_file(f, {"prompt": "{}"})) self.assertEqual( @@ -2202,31 +2206,19 @@ class TestOutputScanInternals(AdminAPITestCase): def test_ingest_link_failures_are_tolerated(self): f = Path("gen.png") - meta = { - "prompt": json.dumps( - {"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "a dog"}}} - ) - } + meta = {"prompt": json.dumps(_wired_prompt("a dog"))} self.api.db.link_image_to_prompt = _raise self.assertEqual(self.api._ingest_scanned_file(f, meta), "added") self.assertEqual(self.api._ingest_scanned_file(f, meta), "found") def test_ingest_save_failure_counts_as_found(self): f = Path("gen.png") - meta = { - "prompt": json.dumps( - {"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "a bird"}}} - ) - } + meta = {"prompt": json.dumps(_wired_prompt("a bird"))} self.api.db.save_prompt = lambda *a, **k: None self.assertEqual(self.api._ingest_scanned_file(f, meta), "found") def test_batch_counts_and_per_file_errors(self): - meta = { - "prompt": json.dumps( - {"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "a fish"}}} - ) - } + meta = {"prompt": json.dumps(_wired_prompt("a fish"))} self.api.db.get_prompt_by_hash = _raise counts = self.api._ingest_scan_batch_sync( [(Path("a.png"), meta), (Path("b.png"), {})] @@ -2915,7 +2907,7 @@ class TestScanReadsVideos(AdminAPITestCase): shutil.which("ffmpeg") and shutil.which("ffprobe"), "ffmpeg not installed" ) async def test_scan_adds_prompts_found_in_videos(self): - graph = {"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "a cat"}}} + graph = _wired_prompt("a cat") _write_mp4( self.output_dir / "clip.mp4", json.dumps({"prompt": json.dumps(graph)}) ) diff --git a/tests/test_api_core.py b/tests/test_api_core.py index 07e6ffa7..bf46c323 100644 --- a/tests/test_api_core.py +++ b/tests/test_api_core.py @@ -835,7 +835,6 @@ class TestMetadataParsing(unittest.TestCase): self.assertEqual(parsed["positive_prompt"], "a cat") self.assertEqual(parsed["negative_prompt"], "dog") self.assertEqual(parsed["parameters"]["parameters"], meta["parameters"]) - self.assertEqual(self.api._extract_readable_prompt(parsed), "a cat") def test_parse_comfyui_fields(self): meta = { @@ -856,97 +855,6 @@ class TestMetadataParsing(unittest.TestCase): self.assertEqual(parsed["prompt"], "plain text") self.assertEqual(parsed["workflow"], {"nodes": []}) - def test_extract_readable_prompt_variants(self): - extract = self.api._extract_readable_prompt - self.assertEqual(extract({"prompt": "plain"}), "plain") - self.assertEqual(extract({"prompt": ["a", "", "b"]}), "a b") - self.assertEqual(extract({"prompt": 42}), "42") - self.assertEqual( - extract({"prompt": None, "parameters": {"positive": ["x", "y"]}}), "x y" - ) - self.assertIsNone(extract({"prompt": {}, "parameters": {}})) - - graph = { - "1": {"class_type": "CLIPTextEncode", "inputs": {"text": "pos"}}, - "2": {"class_type": "CLIPTextEncode", "inputs": {"text": "neg"}}, - "3": { - "class_type": "KSampler", - "inputs": {"positive": ["1", 0], "negative": ["2", 0]}, - }, - } - self.assertEqual(extract({"prompt": graph}), "pos") - self.assertEqual(extract({"prompt": {"x": 1}, "workflow": graph}), "pos") - - def test_node_inputs_and_text_lookup(self): - self.assertEqual(self.api._get_node_inputs("nope"), {}) - self.assertEqual(self.api._get_node_inputs({"inputs": 5}), {}) - self.assertEqual(self.api._get_node_inputs({"inputs": {"a": 1}}), {"a": 1}) - listed = self.api._get_node_inputs( - {"inputs": [{"name": "text", "link": None}, {"nope": 1}, "junk"]} - ) - self.assertEqual(list(listed), ["text"]) - - self.assertIsNone(self.api._find_text_in_node(None)) - self.assertEqual(self.api._find_text_in_node({"inputs": {"text": "hi"}}), "hi") - self.assertEqual( - self.api._find_text_in_node( - {"type": "CLIPTextEncode", "widgets_values": ["w"]} - ), - "w", - ) - self.assertIsNone( - self.api._find_text_in_node( - {"type": "CLIPTextEncode", "widgets_values": [" "]} - ) - ) - self.assertIsNone(self.api._find_text_in_node({"type": "KSampler"})) - - def test_positive_prompt_from_nodes_array_and_fallbacks(self): - extract = self.api._extract_positive_prompt_from_comfyui_data - self.assertIsNone(extract("x")) - self.assertIsNone(extract({})) - self.assertIsNone(extract({"nodes": []})) - - workflow = { - "nodes": [ - {"id": 1, "type": "CLIPTextEncode", "widgets_values": ["pos"]}, - { - "id": 2, - "type": "CLIPTextEncode", - "widgets_values": ["neg"], - "title": "Negative", - }, - {"id": 3, "type": "KSampler", "inputs": [{"name": "seed", "link": 4}]}, - "junk", - ] - } - self.assertEqual(extract(workflow), "pos") - - only_negative = { - "nodes": [ - { - "id": 2, - "type": "CLIPTextEncode", - "widgets_values": ["neg"], - "title": "neg", - }, - ] - } - self.assertEqual(extract(only_negative), "neg") - - bad_link = { - "1": {"class_type": "CLIPTextEncode", "inputs": {"text": "pos"}}, - "3": { - "class_type": "KSampler", - "inputs": {"positive": "oops", "negative": []}, - }, - "4": { - "class_type": "KSampler", - "inputs": {"positive": ["zz", 0], "negative": []}, - }, - } - self.assertEqual(extract(bad_link), "pos") - if __name__ == "__main__": unittest.main() diff --git a/tests/test_pair_scan.py b/tests/test_pair_scan.py index 1597b1f9..2ee70151 100644 --- a/tests/test_pair_scan.py +++ b/tests/test_pair_scan.py @@ -115,7 +115,12 @@ class TestPairScan(AdminAPITestCase): self.assertEqual(second["pairs_added"], 0) def test_legacy_single_prompt_second_pass_not_reported_as_new_link(self): - meta = {"prompt": json.dumps({"1": graph()["1"]})} + # A sampler-less graph: one encoder wired into a guider (#252). + single = { + "1": graph()["1"], + "2": {"class_type": "BasicGuider", "inputs": {"conditioning": ["1", 0]}}, + } + meta = {"prompt": json.dumps(single)} path = self.output_dir / "old.png" self.assertEqual(self.api._ingest_scanned_file(path, meta), "added") self.assertEqual(self.api._ingest_scanned_file(path, meta), "unchanged") @@ -292,3 +297,155 @@ class TestPairScan(AdminAPITestCase): self.api.db.get_image_prompt_info(str(path))["workflow_data"], workflow ) self.assertEqual(self.api._ingest_scanned_file(path, metadata), "unchanged") + + +def flux_graph(text="a red fox", encoder="CLIPTextEncode"): + """A sampler-less Flux graph: the prompt reaches a guider's conditioning (#252).""" + return { + "1": {"class_type": "UNETLoader", "inputs": {"unet_name": "flux.safetensors"}}, + "2": {"class_type": "DualCLIPLoader", "inputs": {"clip_name1": "t5"}}, + "6": {"class_type": encoder, "inputs": {"text": text, "clip": ["2", 0]}}, + "22": { + "class_type": "BasicGuider", + "inputs": {"model": ["1", 0], "conditioning": ["6", 0]}, + }, + "13": { + "class_type": "SamplerCustomAdvanced", + "inputs": {"guider": ["22", 0], "sigmas": ["17", 0]}, + }, + "17": {"class_type": "BasicScheduler", "inputs": {"steps": 20}}, + "9": {"class_type": "SaveImage", "inputs": {"images": ["13", 0]}}, + } + + +class TestSamplerLessScan(AdminAPITestCase): + """Graphs where no node takes a positive or negative input (#252).""" + + def _scanned(self, name, prompt_graph, workflow=None): + meta = {"prompt": json.dumps(prompt_graph)} + if workflow is not None: + meta["workflow"] = json.dumps(workflow) + return self.api._ingest_scanned_file(self.output_dir / name, meta) + + def _texts(self): + return [p["text"] for p in self.api.db.get_recent_prompts()["prompts"]] + + def test_the_one_wired_prompt_is_linked(self): + self.assertEqual(self._scanned("flux.png", flux_graph()), "added") + self.assertEqual(self._texts(), ["a red fox"]) + + def test_a_text_node_feeding_the_encoder_is_one_prompt(self): + g = flux_graph(text=["30", 0]) + g["30"] = {"class_type": "TextInput", "inputs": {"text": "a red fox"}} + self.assertEqual(self._scanned("textinput.png", g), "added") + self.assertEqual(self._texts(), ["a red fox"]) + + def test_a_display_node_that_feeds_nothing_is_never_the_prompt(self): + # A real scan stored a "=== GIT CLONE REPORT ===" display text this way. + g = flux_graph() + g["40"] = { + "class_type": "easy showAnything", + "inputs": {"text": "=== GIT CLONE REPORT ===\nTotal repositories: 1"}, + } + self.assertEqual(self._scanned("report.png", g), "added") + self.assertEqual(self._texts(), ["a red fox"]) + + def test_only_unconsumed_texts_store_nothing(self): + g = { + "40": {"class_type": "easy showAnything", "inputs": {"text": "report"}}, + "41": {"class_type": "Textbox", "inputs": {"text": "install note"}}, + } + self.assertEqual(self._scanned("notes.png", g), "found") + self.assertEqual(self._texts(), []) + + def test_two_wired_texts_are_ambiguous_and_stay_unlinked(self): + # A LoRA loader's text input is wired too: the scan stored "<lora:...>". + g = flux_graph() + g["50"] = { + "class_type": "Lora Loader (LoraManager)", + "inputs": { + "model": ["1", 0], + "clip": ["2", 0], + "text": "<lora:style:1.00>", + }, + } + g["22"]["inputs"]["model"] = ["50", 0] + g["6"]["inputs"]["clip"] = ["50", 1] + self.assertEqual(self._scanned("lora.png", g), "unresolved") + self.assertEqual(self._texts(), []) + + def test_an_untraceable_wired_text_makes_the_graph_ambiguous(self): + # The encoder's text comes from a node no rule reads: never guess the other. + g = flux_graph(text=["60", 0]) + g["60"] = {"class_type": "SomeLLMNode", "inputs": {"seed": 1}} + g["50"] = { + "class_type": "Lora Loader (LoraManager)", + "inputs": {"model": ["1", 0], "text": "<lora:style:1.00>"}, + } + g["22"]["inputs"]["model"] = ["50", 0] + self.assertEqual(self._scanned("llm.png", g), "unresolved") + self.assertEqual(self._texts(), []) + + def test_this_repositorys_prompt_node_is_authoritative(self): + g = flux_graph(text="a red fox", encoder="PromptManager") + g["50"] = { + "class_type": "Lora Loader (LoraManager)", + "inputs": {"model": ["1", 0], "text": "<lora:style:1.00>"}, + } + g["22"]["inputs"]["model"] = ["50", 0] + self.assertEqual(self._scanned("own.png", g), "added") + self.assertEqual(self._texts(), ["a red fox"]) + + def test_a_numeric_text_is_never_stored_as_a_prompt(self): + self.assertEqual(self._scanned("number.png", flux_graph(text="2048")), "found") + self.assertEqual(self._texts(), []) + + def test_workflow_widget_values_are_never_read(self): + # The API text is linked from a node no rule reads; the UI workflow + # still holds a stale widget value, which the old scan stored. + g = flux_graph(text=["70", 0]) + g["70"] = {"class_type": "PromptBuilder", "inputs": {"prompt": "x", "n": 2}} + workflow = { + "nodes": [ + {"id": 6, "type": "CLIPTextEncode", "widgets_values": ["stale text"]} + ] + } + self.assertEqual(self._scanned("stale.png", g, workflow), "unresolved") + self.assertEqual(self._texts(), []) + + def test_a_lora_loaders_trigger_word_output_is_not_its_text(self): + # The encoder reads the loader's trigger-word output (slot 2), not its + # text input: the prompt is unknown, never the "<lora:...>" syntax. + g = flux_graph(text=["50", 2]) + g["50"] = { + "class_type": "Lora Loader (LoraManager)", + "inputs": {"model": ["1", 0], "text": "<lora:style:1.00>"}, + } + g["22"]["inputs"]["model"] = ["50", 0] + self.assertEqual(self._scanned("trigger.png", g), "unresolved") + self.assertEqual(self._texts(), []) + + def test_a_seeded_text_node_holds_a_template_not_the_prompt(self): + g = flux_graph(text=["80", 0]) + g["80"] = { + "class_type": "RandomPrompt", + "inputs": {"text": "a {red|blue} fox", "seed": 5}, + } + self.assertEqual(self._scanned("random.png", g), "unresolved") + self.assertEqual(self._texts(), []) + + def test_a1111_parameters_beside_an_undecided_graph_are_read(self): + # Metadata savers write both; the old scan read the A1111 positive. + g = flux_graph() + g["50"] = { + "class_type": "Lora Loader (LoraManager)", + "inputs": {"model": ["1", 0], "text": "<lora:style:1.00>"}, + } + g["22"]["inputs"]["model"] = ["50", 0] + meta = { + "prompt": json.dumps(g), + "parameters": "a red fox\nNegative prompt: blur\nSteps: 20", + } + path = self.output_dir / "saver.png" + self.assertEqual(self.api._ingest_scanned_file(path, meta), "added") + self.assertEqual(self._texts(), ["a red fox"]) diff --git a/tests/test_sampler_less_prompt.py b/tests/test_sampler_less_prompt.py new file mode 100644 index 00000000..62ee6fbe --- /dev/null +++ b/tests/test_sampler_less_prompt.py @@ -0,0 +1,112 @@ +"""``sampler_less_prompt``: the one wired text of a graph with no sampler (#252).""" + +import time +import unittest + +from utils.prompt_pair_resolver import sampler_less_prompt + + +def _guided(**nodes): + """``nodes`` plus a guider consuming node ``"1"``.""" + graph = {"9": {"class_type": "BasicGuider", "inputs": {"conditioning": ["1", 0]}}} + graph.update(nodes) + return graph + + +class TestSamplerLessPrompt(unittest.TestCase): + def test_not_a_graph_or_too_large_is_absent(self): + for graph in (None, [], {}, "text"): + self.assertEqual(sampler_less_prompt(graph)["state"], "absent") + big = {str(i): {"class_type": "X", "inputs": {}} for i in range(3)} + self.assertEqual(sampler_less_prompt(big, max_visits=2)["state"], "absent") + + def test_one_literal_text(self): + graph = _guided(**{"1": {"class_type": "Enc", "inputs": {"text": " a fox "}}}) + self.assertEqual( + sampler_less_prompt(graph), + {"state": "resolved", "text": "a fox", "texts": ["a fox"]}, + ) + + def test_a_number_literal_is_never_a_prompt(self): + for value in (2048, "2048", "1.5", " 7 "): + graph = _guided(**{"1": {"class_type": "Enc", "inputs": {"text": value}}}) + self.assertEqual(sampler_less_prompt(graph)["state"], "absent", value) + + def test_a_long_digit_run_with_a_letter_is_text_and_fast(self): + text = "9" * 50_000 + "x" + graph = _guided(**{"1": {"class_type": "Enc", "inputs": {"text": text}}}) + started = time.monotonic() + self.assertEqual(sampler_less_prompt(graph)["text"], text) + self.assertLess(time.monotonic() - started, 1.0) + + def test_number_like_words_are_prompts(self): + for value in ("nan", "inf", "Infinity", "1_000"): + graph = _guided(**{"1": {"class_type": "Enc", "inputs": {"text": value}}}) + self.assertEqual(sampler_less_prompt(graph)["text"], value) + + def test_an_own_node_with_unreadable_text_beside_another_text_is_ambiguous(self): + graph = _guided( + **{ + "1": {"class_type": "PromptManager", "inputs": {"text": ["5", 0]}}, + "5": {"class_type": "LLM", "inputs": {"seed": 3}}, + "2": {"class_type": "Enc", "inputs": {"text": "<lora:x:1>"}}, + "3": {"class_type": "Sink", "inputs": {"c": ["2", 0]}}, + } + ) + self.assertEqual(sampler_less_prompt(graph)["state"], "ambiguous") + + def test_unhashable_class_type_is_an_unknown_node_not_a_crash(self): + graph = _guided(**{"1": {"class_type": ["x"], "inputs": {"text": "a fox"}}}) + self.assertEqual(sampler_less_prompt(graph)["text"], "a fox") + + def test_own_negative_node_is_not_authoritative(self): + graph = _guided( + **{ + "1": {"class_type": "Enc", "inputs": {"text": "a fox"}}, + "2": { + "class_type": "PromptManagerNegative", + "inputs": {"text": "blur"}, + }, + "3": {"class_type": "Sink", "inputs": {"c": ["2", 0]}}, + } + ) + found = sampler_less_prompt(graph) + self.assertEqual(found["state"], "ambiguous") + self.assertEqual(found["texts"], ["a fox", "blur"]) + + def test_own_node_text_wins_over_an_untraceable_text(self): + graph = _guided( + **{ + "1": {"class_type": "PromptManagerText", "inputs": {"text": "a fox"}}, + "2": {"class_type": "Enc", "inputs": {"text": ["5", 0]}}, + "5": {"class_type": "LLM", "inputs": {"seed": 3}}, + "6": {"class_type": "Sink", "inputs": {"c": ["2", 0]}}, + } + ) + self.assertEqual(sampler_less_prompt(graph)["text"], "a fox") + + def test_two_own_texts_are_ambiguous(self): + graph = _guided( + **{ + "1": {"class_type": "PromptManager", "inputs": {"text": "a fox"}}, + "2": {"class_type": "PromptManager", "inputs": {"text": "a cat"}}, + "3": {"class_type": "Sink", "inputs": {"c": ["2", 0]}}, + } + ) + self.assertEqual(sampler_less_prompt(graph)["state"], "ambiguous") + + def test_only_an_untraceable_text_is_ambiguous_with_nothing_to_show(self): + graph = _guided( + **{ + "1": {"class_type": "Enc", "inputs": {"text": ["5", 0]}}, + "5": {"class_type": "LLM", "inputs": {"seed": 3}}, + } + ) + self.assertEqual( + sampler_less_prompt(graph), + {"state": "ambiguous", "text": None, "texts": []}, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/utils/prompt_pair_resolver.py b/utils/prompt_pair_resolver.py index b45c948c..15200481 100644 --- a/utils/prompt_pair_resolver.py +++ b/utils/prompt_pair_resolver.py @@ -2578,3 +2578,115 @@ def resolve_text_input( return resolver.text(inputs.get(input_name), "string")[0] except _Unresolved: return None + + +_NUMBER = re.compile(r"[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?") + + +def _numeric(text: str) -> bool: + """Whether ``text`` is a plain number (a widget value such as ``2048``).""" + return _NUMBER.fullmatch(text) is not None + + +def sampler_less_prompt(graph: Any, *, max_visits: int = 500) -> Dict[str, Any]: + """The positive prompt of an API graph with no sampler pair, if one (#252). + + For graphs ``_no_sampler`` finds ``absent`` (no node takes a ``positive`` + or ``negative`` input, such as a Flux guider or a video wrapper). Only the + executed API graph is read, never the UI workflow. A candidate is a node + with a ``text`` input whose output another node consumes: a text that + feeds nothing did not make the image. Its text is traced as a STRING + (``resolve_text_input``); a linked text no rule reads is unknown unless + it is built from a candidate's text (``_covered``). Blank and + numeric texts are never prompts, and a node with a seed input holds a + template, not the prompt (unknown). This repository's own positive-text + nodes are authoritative: when they give a text, only they count. + + Returns ``{"state", "text", "texts"}``: ``resolved`` with the one distinct + text; ``ambiguous`` when several texts or an unknown one could be the + prompt (``texts`` lists the readable ones); ``absent`` when none. + """ + absent = {"state": "absent", "text": None, "texts": []} + if not isinstance(graph, dict) or not graph or len(graph) > max_visits: + return absent + nodes = {str(key): value for key, value in graph.items()} + consumed = { + str(value[0]) + for node in nodes.values() + if isinstance(node, dict) and isinstance(node.get("inputs"), dict) + for value in node["inputs"].values() + if _link(value) + } + own: list = [] + other: list = [] + unknown = False + for node_id, node in nodes.items(): + inputs = node.get("inputs") if isinstance(node, dict) else None + if node_id not in consumed or not _has_text_input(node): + continue + if _seeded(inputs): + # Generated at run time (a wildcard or a language model): its + # text input is a template, not the prompt that ran. + unknown = True + continue + value = inputs["text"] + text = resolve_text_input(nodes, node_id, "text", max_visits=max_visits) + if text is None: + # A literal that is not a string (a number) is never a prompt. + if _link(value) and not _covered(nodes, value): + unknown = True + continue + text = text.strip() + if not text or _numeric(text): + continue + kind = node.get("class_type") + if isinstance(kind, str) and kind in _OWN_CLASSES and "Negative" not in kind: + own.append(text) + else: + other.append(text) + texts = list(dict.fromkeys(own or other)) + if len(texts) == 1 and (own or not unknown): + return {"state": "resolved", "text": texts[0], "texts": texts} + if texts or unknown: + return {"state": "ambiguous", "text": None, "texts": texts} + return absent + + +def _has_text_input(node: Any) -> bool: + inputs = node.get("inputs") if isinstance(node, dict) else None + return isinstance(inputs, dict) and "text" in inputs + + +def _seeded(inputs: dict) -> bool: + """A seed input: the node's string output is a run-time value (the + resolver's ``runtime-value`` rule).""" + return any(isinstance(name, str) and name.endswith("seed") for name in inputs) + + +def _covered(nodes: dict, link: list) -> bool: + """Whether an unreadable text link is built from a candidate's text. + + True when a node upstream of ``link`` is read through its first output + (slot 0, the legacy encoder rule's text) and has an unseeded ``text`` + input: that candidate is read on its own, and whatever else is joined to + it at run time (a caption, a language model) is no stored prompt to + choose. Another output of such a node (a loader's trigger words) is not + its text. False when the text comes only from inputs no candidate holds + (a builder's ``prompt`` field): the prompt is unknown then. + """ + seen, stack = set(), [(str(link[0]), link[1])] + while stack: + node_id, slot = stack.pop() + if (node_id, slot) in seen: + continue + seen.add((node_id, slot)) + node = nodes.get(node_id) + inputs = node.get("inputs") if isinstance(node, dict) else None + if not isinstance(inputs, dict): + continue + if "text" in inputs and slot == 0 and not _seeded(inputs): + return True + stack.extend( + (str(value[0]), value[1]) for value in inputs.values() if _link(value) + ) + return False From e472a8bf747a23189a3c21c581a02856232d0cbe Mon Sep 17 00:00:00 2001 From: Vito <vito.sansevero@gmail.com> Date: Wed, 28 Oct 2026 02:03:22 +0000 Subject: [PATCH 04/11] fix(system): Tagging selects show their whole chosen option at 390 px (#398) (#399) * fix(system): Tagging selects show their whole chosen option at 390 px (#398) The concept's phone cap (120px) cut Existing tags ("Replace after review"). #tag-existing now never shrinks for its row text and keeps its natural width at phone width, beside the text. The comparison expects selectsFit for every select, and a new test walks every option of both selects at 390 and 320 px in both themes. TESTING.md D132 records the decision. * fix(system): bound Existing tags by its row at phone width (#398 review) --- TESTING.md | 16 ++++--- tests/browser/system-tagging-concept.spec.mjs | 45 ++++++++++++------- web/js/workspace/workspace.css | 5 ++- 3 files changed, 44 insertions(+), 22 deletions(-) diff --git a/TESTING.md b/TESTING.md index ce4bd6b2..b21f77c9 100644 --- a/TESTING.md +++ b/TESTING.md @@ -3200,12 +3200,16 @@ the per-section hand-off panel, was retired by S12 (#329): every section is nati again"; an Unload that found nothing loaded only reads the status again. At phone width the concept's 120px select cap cuts every engine label, so `#tag-model` alone keeps its natural width there and its row wraps the select under the text (`workspace.css` - `/* D132 */`); Existing tags stays as the concept draws it, cut at 390 as there. The - comparison maps `selectsFit` only for the engine select and measures each row's - `flexWrap` and whether its control sits under the text. Pinned by `auto-tagging matches - the concept` (both themes and sizes), `auto-tagging at 390 wraps only the engine row`, - `auto-tagging engine select fits at 320 wide` and the mutation check (it includes a - wrapped row) (system-tagging-concept.spec), and the model, Download, Unload and Preview + `/* D132 */`). The concept's cap cuts Existing tags too ("Replace after review"); by the + maintainer's decision (#398) that parity is not kept: `#tag-existing` never shrinks for + its row text and at phone width keeps its natural width beside the text (its row does + not wrap; the text column narrows). 1600 is as the concept draws it. The comparison + expects `selectsFit` true for every select and measures each row's `flexWrap` and + whether its control sits under the text. Pinned by `auto-tagging matches the concept` + (both themes and sizes), `auto-tagging at 390 wraps only the engine row`, `auto-tagging + selects fit every option at 390/320 wide` (every option of both selects, both themes, no + page scroll) and the mutation check (a wrapped row, a squeezed engine and Existing tags + select) (system-tagging-concept.spec), and the model, Download, Unload and Preview tests of workspace-system-tagging.spec. - **D133** (Auto-tagging, S11): a "Suggestions per image" row (`#tag-suggestions`, number 1–100, host key `autotag_suggestion_limit`) follows the two confidence rows: diff --git a/tests/browser/system-tagging-concept.spec.mjs b/tests/browser/system-tagging-concept.spec.mjs index b02f5a83..ff79595c 100644 --- a/tests/browser/system-tagging-concept.spec.mjs +++ b/tests/browser/system-tagging-concept.spec.mjs @@ -127,9 +127,9 @@ function mapTaggingConcept(reference, { template, phone }) { actions: { ...reference.actions, buttons: [preview, ['Download again', ''], ['Unload', '']] }, // D134: the prompt-tag batch, reviewed first. operations: [{ tree: ['div', 'div.row-actions'], title: 'Tag output images', description: BATCH, actions: [['Review', '']] }], - // D132: the engine select shows its whole chosen option at every width (at 390 the concept's 120px cap cuts it; - // `#tag-model` alone may grow there). Existing tags keeps the concept's own fit, cut or not. - selectsFit: reference.selectsFit.map((fit, index) => (index === 0 ? true : fit)), + // D132 (#398): every select shows its whole chosen option at every width (at 390 the concept's 120px cap cuts + // them; production lets each grow past it). + selectsFit: reference.selectsFit.map(() => true), }; } @@ -168,16 +168,28 @@ for (const theme of THEMES) for (const [width, height] of SIZES) { }); } -// D132 below the concept's smallest size: on a 320px phone the engine select still shows its whole option. -for (const theme of THEMES) test(`auto-tagging engine select fits at 320 wide ${theme}`, async ({ page }) => { - await page.setViewportSize({ width: 320, height: 700 }); - const production = await productionTagging(page, theme); - for (const model of ['hf', 'gguf', 'wd14-swinv2']) { - await page.locator('#workspace-main #tag-model').selectOption(model); - expect((await page.evaluate(measureTagging, '#workspace-main')).selectsFit[0], model).toBe(true); - } - expect(production.pageScroll).toBe(false); -}); +// D132 (#398) at phone width and below the concept's smallest size: every option of every select, once chosen, shows +// whole, with no page scroll. +for (const theme of THEMES) for (const width of [390, 320]) { + test(`auto-tagging selects fit every option at ${width} wide ${theme}`, async ({ page }, testInfo) => { + await page.setViewportSize({ width, height: 844 }); + await productionTagging(page, theme); + await page.locator('#workspace-main #tag-existing').selectOption('replace_after_review'); + await page.locator('#workspace-main section.surface').screenshot({ path: testInfo.outputPath(`production-tagging-selects-${theme}-${width}.png`) }); + const selects = page.locator('#workspace-main section.surface > .setting-row > select'); + await expect(selects).toHaveCount(2); + for (let index = 0; index < 2; index += 1) { + const select = selects.nth(index); + const values = await select.evaluate(node => [...node.options].map(option => option.value)); + for (const value of values) { + await select.selectOption(value); + const measured = await page.evaluate(measureTagging, '#workspace-main'); + expect(measured.selectsFit, `${await select.getAttribute('id')}=${value}`).toEqual([true, true]); + expect(measured.pageScroll, value).toBe(false); + } + } + }); +} test('auto-tagging comparison detects mutations', async ({ page, request }) => { await page.setViewportSize({ width: 1600, height: 1100 }); @@ -222,6 +234,9 @@ test('auto-tagging comparison detects mutations', async ({ page, request }) => { const undoSqueeze = await mutate(page, { css: '#workspace-main #tag-model { max-width: 60px !important; min-width: 0 !important; }' }); expect(await paths()).toEqual(expect.arrayContaining(['selectsFit.0'])); await undoSqueeze(); + const undoSqueezeExisting = await mutate(page, { css: '#workspace-main #tag-existing { max-width: 60px !important; min-width: 0 !important; }' }); + expect(await paths()).toEqual(expect.arrayContaining(['selectsFit.1'])); + await undoSqueezeExisting(); // A row that wraps where the concept's does not. const undoWrap = await mutate(page, { css: '#workspace-main .setting-row:has(> #tag-existing) { flex-wrap: wrap; }' }); expect(await paths()).toEqual(['rows.1.flexWrap']); @@ -233,14 +248,14 @@ test('auto-tagging comparison detects mutations', async ({ page, request }) => { expect(diffInventory(expected, await measure())).toEqual([]); }); -// D132 at phone width, unmapped: only the engine row wraps; Existing tags keeps the concept's cut select. +// D132 at phone width: only the engine row wraps; Existing tags grows beside its text instead (#398). test('auto-tagging at 390 wraps only the engine row', async ({ page, request }) => { await page.setViewportSize({ width: 390, height: 844 }); const concept = await conceptTagging(page, 'dark'); const production = await productionTagging(page, 'dark'); expect(diffInventory(mapTaggingConcept(concept, await savedTemplate(request, page)), production)).toEqual([]); expect(production.rows.map(row => row.flexWrap)).toEqual(['wrap', 'nowrap', 'nowrap', 'nowrap', 'nowrap']); - expect(production.selectsFit[1]).toBe(concept.selectsFit[1]); + expect(production.selectsFit).toEqual([true, true]); }); /** Runs in the page: the open review dialog: its title, body and the concept's tokens. */ diff --git a/web/js/workspace/workspace.css b/web/js/workspace/workspace.css index 8d29e3f2..fedd3ed6 100644 --- a/web/js/workspace/workspace.css +++ b/web/js/workspace/workspace.css @@ -433,10 +433,13 @@ pre.prompt-text { font: inherit; font-size: 12px; line-height: 1.75; margin-bloc } /* D132: at phone width the concept's 120px cap cuts every engine label ("WD14 SwinV2 · ~400 MB"); the Tagging model select alone keeps its natural width there (its widest engine label) and, with no room beside the row text, wraps - under it. */ + under it. Existing tags (#398) never shrinks for its row text and, past the cap, keeps its natural width beside the + text ("Replace after review"), bounded by its row like the engine select. */ +.setting-row #tag-existing { flex-shrink: 0; /* D132 */ } @media (max-width: 700px) { #workspace-main .setting-row:has(> #tag-model) { flex-wrap: wrap; /* D132 */ } #workspace-main .setting-row #tag-model { flex-shrink: 0; max-width: 100% !important; /* D132 */ } + #workspace-main .setting-row #tag-existing { max-width: 100% !important; /* D132 */ } } /* D109: System › Diagnostics & logs' added Lines field sits in the concept's toolbar like its Level select, and a value From 83fa80c330bce995fbe0c3d60553e62bbb0685f9 Mon Sep 17 00:00:00 2001 From: Vito <vito.sansevero@gmail.com> Date: Wed, 28 Oct 2026 03:48:11 +0000 Subject: [PATCH 05/11] fix(restore,threads): a stopped restore waits for its worker; exit-window thread refusals are quiet (#360, #375) (#404) * fix(restore): a restore cut off by shutdown waits for its worker thread (#360) A cancelled restore handler or job no longer removes the stage or the upload, or ends its exclusive registry admission, while the executor thread still stages or publishes. The cancel waits for the worker; a publish that finished is adopted before the cancel goes on, and a stage finished after the cancel is removed. * fix(threads): a thread or pool refused at interpreter exit is quiet and leaves nothing pending (#375) During interpreter exit, threading._shutdown shuts every thread pool before the package teardown runs while our daemon threads keep working. map_parallel now runs the items a refused pool did not start on the calling thread, schedule() drops the path and returns False when its worker cannot start, and the preview queue skips the file. Each refusal is logged once at DEBUG without a traceback; other start failures behave as before. * fix(restore): keep the cancel and the adoption when a stopped restore's publish ends (#360) A publish that fails after the cancel now ends in CancelledError (after its prune) instead of a 500, and the restored library is adopted to its end even when a second cancel arrives. The blocking calls go through _run_in_executor again, as the executor contract test expects. Exit-window refusals: the comments state what was measured (a pool is refused during threading._shutdown; Thread.start only once the interpreter is finalizing), and the tests assert the one DEBUG line. --- py/api/backup_restore.py | 76 ++++++++- py/api/thumbnail_auto.py | 34 ++-- tests/test_exit_window_threads.py | 266 ++++++++++++++++++++++++++++++ tests/test_restore_shutdown.py | 233 ++++++++++++++++++++++++++ tests/test_thumbnail_auto.py | 21 +++ utils/image_monitor.py | 16 +- utils/parallel.py | 69 +++++++- 7 files changed, 696 insertions(+), 19 deletions(-) create mode 100644 tests/test_exit_window_threads.py create mode 100644 tests/test_restore_shutdown.py diff --git a/py/api/backup_restore.py b/py/api/backup_restore.py index edd60d14..a83ecbfa 100644 --- a/py/api/backup_restore.py +++ b/py/api/backup_restore.py @@ -483,13 +483,59 @@ class BackupRestoreRoutesMixin: self.logger.info(f"Removed {len(removed)} old safety backups") async def _publish(self, model, staged): - """Publish ``staged``; on a page-copy failure prune and re-raise.""" + """Publish ``staged``; on a page-copy failure prune and re-raise. + + A cancel (server stop) waits for the publishing thread (#360): the + caller's cleanup and its registry admission must outlive it. When the + copy then succeeded the library was replaced, so it is adopted + (``_after_publish``, itself run to its end) before the cancel goes + on; a copy that failed is handled as usual, then the cancel goes on. + """ + future = self._in_executor_task(model.publish_staged, staged) + cancelled = await _outlive_cancel(future) try: - return await self._run_in_executor(model.publish_staged, staged) + backup_path = future.result() except sqlite3.OperationalError as exc: if getattr(exc, "_prompt_manager_restore_backup_created", False) is True: await self._prune_after(model.db_path, failed=True) + if cancelled: + raise asyncio.CancelledError from exc raise + except Exception as exc: + if cancelled: + raise asyncio.CancelledError from exc + raise + if cancelled: + self.logger.info("The restore finished while the server was stopping") + # A second cancel (the loop's own shutdown) must not cut it short. + adopted = asyncio.ensure_future(self._after_publish(model.db_path)) + await _outlive_cancel(adopted) + if not adopted.cancelled() and adopted.exception() is not None: + self.logger.error( + "Could not adopt the restored library", + exc_info=adopted.exception(), + ) + raise asyncio.CancelledError + return backup_path + + async def _stage_to_end(self, model, temp_path): + """Stage ``temp_path``; a cancel waits for the worker (#360). + + The upload stays until the staging thread is done with it, and a + stage it finished after the cancel is removed before the cancel goes + on. + """ + future = self._in_executor_task(model.stage_restore, temp_path) + if await _outlive_cancel(future): + if not future.cancelled() and future.exception() is None: + await self._discard_held([_Orphan(future.result())]) + raise asyncio.CancelledError + return future.result() + + def _in_executor_task(self, func, *args): + """``_run_in_executor(func, *args)`` as a task ``_outlive_cancel`` can + wait for (shielded, the executor call is never cancelled).""" + return asyncio.ensure_future(self._run_in_executor(func, *args)) async def _after_publish(self, db_path): """Everything after the copy replaced the library: best effort, logged. @@ -609,7 +655,7 @@ class BackupRestoreRoutesMixin: {"success": False, "error": REQUESTS_BUSY}, status=409 ) model = self.db.model - staged = await self._run_in_executor(model.stage_restore, temp_path) + staged = await self._stage_to_end(model, temp_path) try: backup_path = await self._publish(model, staged) finally: @@ -927,6 +973,30 @@ class BackupRestoreRoutesMixin: job.update(progress=100, status_line=line) +async def _outlive_cancel(future): + """Await ``future`` (an executor call's task) to its end, even through + cancels (#360). + + A cancelled task cannot stop the worker thread, so the files it uses and + the registry admission it runs under must outlive it: the cancel is held + until the worker returned. Returns whether the task was cancelled + meanwhile; the caller reads the outcome from ``future`` and re-raises + ``CancelledError``. A future cancelled itself (an executor shut down) is + not a cancel of the task: ``future.result()`` raises it. Never catches + ``GeneratorExit``: a closed loop still closes the coroutine. + """ + cancelled = False + while not future.done(): + try: + await asyncio.shield(future) + except asyncio.CancelledError: + if not future.cancelled(): + cancelled = True + except Exception: + pass # the caller reads it from the future + return cancelled + + class _Orphan: """A stage with no store entry, in the shape ``_discard_held`` takes.""" diff --git a/py/api/thumbnail_auto.py b/py/api/thumbnail_auto.py index bfe38315..0c9095cf 100644 --- a/py/api/thumbnail_auto.py +++ b/py/api/thumbnail_auto.py @@ -88,9 +88,9 @@ from .thumbnail_encode import shutdown_encoders, writer_settings from .thumbnail_jobs import ThumbnailOptions try: - from ...utils.parallel import map_parallel + from ...utils.parallel import map_parallel, note_refusal, refused_at_shutdown except ImportError: # tests import the package top-level - from utils.parallel import map_parallel + from utils.parallel import map_parallel, note_refusal, refused_at_shutdown # The placeholder frame a video without a preview is answered with (JPEG # bytes, never written to disk); written previews follow the settings. @@ -410,7 +410,8 @@ class AutoThumbnailer: def enqueue(self, src, dst, is_video): """Queue *dst*'s preview for the worker; False when already queued, - full or closed. Never blocks on a write.""" + full, closed or its worker is refused at interpreter exit. Never + blocks on a write.""" key = _dst_key(dst) with self._changed: if self._closed or key in self._queued: @@ -420,30 +421,41 @@ class AutoThumbnailer: self._queued.add(key) self._queue.append((key, Path(src), Path(dst), bool(is_video))) self._busy = True + started = True if self._start_worker and not self._worker_running: - self._start_worker_locked() + started = self._start_worker_locked() + if not started: + # Refused at interpreter exit: no worker will take it (#375). + self._queue.pop() + self._queued.discard(key) self._changed.notify_all() - return True + return started def _start_worker_locked(self): - """Start the worker (lock held). A start that fails leaves the file - queued and the next enqueue tries again.""" + """Start the worker (lock held); False only when the interpreter is + exiting (#375: logged once at DEBUG, the caller drops the file). + Another start failure leaves the file queued and the next enqueue + tries again.""" worker = threading.Thread(target=self._run, name=WORKER_NAME, daemon=True) worker.pm_preview_token = _MODULE_TOKEN worker.pm_preview_close = self.close self._worker_running = True try: worker.start() - except RuntimeError: + except RuntimeError as error: self._worker_running = False self._busy = False + if refused_at_shutdown(error): + note_refusal(self._logger, "The automatic preview worker") + return False self._logger.warning( "The automatic preview worker could not start; the next output " "tries again", exc_info=True, ) - return + return True self._worker = worker + return True def _run(self): try: @@ -790,8 +802,8 @@ class AutoThumbnailMixin: Returns ``queued``, ``fresh`` (its preview is current) or ``skipped`` (not a media file under a current root, a link, a preview itself, - already queued, or after the app's cleanup). Never writes on the - caller's thread. + already queued, after the app's cleanup, or when its worker is refused + at interpreter exit). Never writes on the caller's thread. """ if self._auto_thumbnails_shut: return "skipped" diff --git a/tests/test_exit_window_threads.py b/tests/test_exit_window_threads.py new file mode 100644 index 00000000..1323f4db --- /dev/null +++ b/tests/test_exit_window_threads.py @@ -0,0 +1,266 @@ +"""A thread or pool refused at interpreter shutdown is quiet (#375). + +When the interpreter exits, ``threading._shutdown`` first shuts every +``ThreadPoolExecutor`` (``submit`` then raises "cannot schedule new futures +after interpreter shutdown") and joins the non-daemon threads; only then do +the ``atexit`` callbacks (the package teardown) run. Our daemon threads keep +working meanwhile. ``Thread.start`` is refused ("can't create new thread at +interpreter shutdown") only later, once the interpreter is finalizing; the +unit tests cover that case, the child process the pool case. Such a refusal +is logged once at DEBUG, with no traceback, and leaves nothing pending. + +``threading.Thread`` is replaced only in the module under test (a copy of +the ``threading`` namespace), never process-wide. The child-process test runs +the real exit: a daemon thread queues previews while the main thread returns. +""" + +import json +import logging +import os +import subprocess +import sys +import tempfile +import textwrap +import threading +import types +import unittest +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from unittest import mock + +from py.api import thumbnail_auto +from utils import image_monitor, parallel + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +CHILD_CEILING_SECONDS = 60 # the hang watchdog allows 120 s per test +REPORT_PREFIX = "PM_EXIT_REPORT " +THREAD_REFUSAL = "can't create new thread at interpreter shutdown" +POOL_REFUSAL = "cannot schedule new futures after interpreter shutdown" + + +def _refusing_threading(): + """A ``threading`` namespace whose ``Thread.start`` is refused.""" + + class RefusedThread(threading.Thread): + def start(self): + raise RuntimeError(THREAD_REFUSAL) + + namespace = types.SimpleNamespace(**vars(threading)) + namespace.Thread = RefusedThread + return namespace + + +class _ShutPool(ThreadPoolExecutor): + def submit(self, *args, **kwargs): + raise RuntimeError(POOL_REFUSAL) + + +class _Capture(logging.Handler): + def __init__(self): + super().__init__(logging.DEBUG) + self.records = [] + + def emit(self, record): + self.records.append(record) + + +def _capture(testcase, name): + parallel._noted.clear() # "once" is per process: start each test afresh + logger = logging.getLogger(name) + handler = _Capture() + previous = logger.level + logger.addHandler(handler) + logger.setLevel(logging.DEBUG) + + def restore(): + logger.removeHandler(handler) + logger.setLevel(previous) + + testcase.addCleanup(restore) + return handler.records + + +def _assert_one_debug_line(testcase, records): + """Exactly one refusal line, at DEBUG and without a traceback.""" + refusals = [r for r in records if "interpreter is exiting" in r.getMessage()] + testcase.assertEqual(len(refusals), 1, [r.getMessage() for r in records]) + testcase.assertEqual(refusals[0].levelno, logging.DEBUG) + testcase.assertEqual([r for r in records if r.levelno > logging.DEBUG], []) + testcase.assertTrue(all(r.exc_info is None for r in records)) + + +class TestScheduleRefusedAtShutdown(unittest.TestCase): + def setUp(self): + self.handler = image_monitor.ImageGenerationHandler( + mock.MagicMock(), mock.MagicMock() + ) + self.addCleanup(self.handler.close) + + def test_schedule_returns_false_and_leaves_nothing_pending(self): + records = _capture(self, "prompt_manager.image_monitor") + with mock.patch.object(image_monitor, "threading", _refusing_threading()): + self.assertFalse(self.handler.schedule("/out/a.png")) + self.assertFalse(self.handler.schedule("/out/b.png")) + self.assertEqual(self.handler.pending_count(), 0) + _assert_one_debug_line(self, records) + + def test_another_start_failure_still_raises_and_leaves_nothing_pending(self): + class Failing(threading.Thread): + def start(self): + raise RuntimeError("can't start new thread") + + namespace = types.SimpleNamespace(**vars(threading)) + namespace.Thread = Failing + with mock.patch.object(image_monitor, "threading", namespace): + with self.assertRaises(RuntimeError): + self.handler.schedule("/out/a.png") + self.assertEqual(self.handler.pending_count(), 0) + + def test_schedule_before_shutdown_is_unchanged(self): + self.handler.processing_delay = 3600 + self.assertTrue(self.handler.schedule("/out/a.png")) + self.assertEqual(self.handler.pending_count(), 1) + + +class TestPreviewQueueRefusedAtShutdown(unittest.TestCase): + def thumbnailer(self, logger): + made = thumbnail_auto.AutoThumbnailer( + lambda *a, **k: {"action": "generated"}, lambda: True, logger + ) + self.addCleanup(made.close) + return made + + def test_enqueue_is_refused_quietly_and_drops_the_file(self): + logger = logging.getLogger("prompt_manager.test_exit_window") + records = _capture(self, logger.name) + thumbnailer = self.thumbnailer(logger) + src = Path("/out/a.png") + with mock.patch.object(thumbnail_auto, "threading", _refusing_threading()): + self.assertFalse(thumbnailer.enqueue(src, Path("/out/t/a.webp"), False)) + self.assertFalse(thumbnailer.enqueue(src, Path("/out/t/b.webp"), False)) + self.assertFalse(thumbnailer._worker_running) + self.assertEqual(len(thumbnailer._queue), 0) + self.assertEqual(thumbnailer._queued, set()) + self.assertTrue(thumbnailer.wait_idle(0)) + _assert_one_debug_line(self, records) + + +class TestMapParallelRefusedAtShutdown(unittest.TestCase): + def test_a_shut_pool_runs_the_items_on_the_calling_thread(self): + records = _capture(self, "prompt_manager.parallel") + with mock.patch.object(parallel, "ThreadPoolExecutor", _ShutPool): + self.assertEqual( + parallel.map_parallel(lambda n: n * n, [1, 2, 3], 4), [1, 4, 9] + ) + _assert_one_debug_line(self, records) + + def test_another_pool_error_still_raises(self): + class Broken(ThreadPoolExecutor): + def submit(self, *args, **kwargs): + raise RuntimeError("something else") + + with mock.patch.object(parallel, "ThreadPoolExecutor", Broken): + with self.assertRaises(RuntimeError): + parallel.map_parallel(lambda n: n, [1, 2], 2) + + +CHILD = textwrap.dedent(""" + import threading + + # Registered before concurrent.futures is imported, so it runs after the + # pool's own exit hook: from here on, a pool refuses new work. + exiting = threading.Event() + threading._register_atexit(exiting.set) + + import json + import logging + import os + import sys + from pathlib import Path + + root, tmp = sys.argv[1], sys.argv[2] + sys.path.insert(0, root) + + from py.api.thumbnail_auto import AutoThumbnailer + + logging.basicConfig(level=logging.WARNING, stream=sys.stderr) + logger = logging.getLogger("prompt_manager.exit_child") + + def generate(src, dst, is_video, *, stop, guard, track): + Path(dst).write_bytes(b"preview") + return {"action": "generated"} + + thumbnailer = AutoThumbnailer( + generate, lambda: True, logger, workers=lambda: 4 + ) + queued = threading.Event() + + def late_monitor(): + exiting.wait() + # One batch of two: the worker maps it on a pool. + with thumbnailer._changed: + for name in ("a", "b"): + src = Path(tmp, name + ".png") + src.write_bytes(b"x") + thumbnailer.enqueue(src, Path(tmp, name + "_thumb.webp"), False) + queued.set() + + def holder(): + # Non-daemon: the interpreter waits for it, inside the exit window. + exiting.wait() + queued.wait(30) + idle = thumbnailer.wait_idle(30) + written = sorted( + n for n in os.listdir(tmp) if n.endswith("_thumb.webp") + ) + print("PM_EXIT_REPORT " + json.dumps({"idle": idle, "written": written})) + sys.stdout.flush() + + threading.Thread(target=late_monitor, daemon=True).start() + threading.Thread(target=holder).start() + # Return normally, as ComfyUI does after Ctrl+C. + """) + + +class TestExitWindowChild(unittest.TestCase): + def test_previews_queued_in_the_exit_window_log_no_traceback(self): + with tempfile.TemporaryDirectory() as tmp: + script = os.path.join(tmp, "child.py") + with open(script, "w", encoding="utf-8") as handle: + handle.write(CHILD) + env = dict(os.environ) + env.pop("PROMPT_MANAGER_CONFIG_PATH", None) + env["PROMPT_MANAGER_DATA_DIR"] = tmp + env["PYTHONIOENCODING"] = "utf-8" + try: + completed = subprocess.run( + [sys.executable, script, ROOT, tmp], + cwd=ROOT, + env=env, + capture_output=True, + timeout=CHILD_CEILING_SECONDS, + encoding="utf-8", + errors="replace", + ) + except subprocess.TimeoutExpired as hung: + self.fail( + f"the child did not exit within {CHILD_CEILING_SECONDS} s\n" + f"stdout:\n{hung.stdout}\nstderr:\n{hung.stderr}" + ) + detail = f"stdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + self.assertEqual(completed.returncode, 0, detail) + reports = [ + line[len(REPORT_PREFIX) :] + for line in completed.stdout.splitlines() + if line.startswith(REPORT_PREFIX) + ] + self.assertEqual(len(reports), 1, detail) + report = json.loads(reports[0]) + self.assertTrue(report["idle"], detail) + # The batch still ran: on the worker's own thread. + self.assertEqual(report["written"], ["a_thumb.webp", "b_thumb.webp"], detail) + self.assertNotIn("Traceback", completed.stderr, detail) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_restore_shutdown.py b/tests/test_restore_shutdown.py new file mode 100644 index 00000000..2ebf787f --- /dev/null +++ b/tests/test_restore_shutdown.py @@ -0,0 +1,233 @@ +"""A restore cut off by a server stop waits for its worker thread (#360). + +A cancelled handler (or job) task cannot stop the executor thread that +stages or publishes. Its cleanup (the stage, the upload) and the registry's +``restore`` admission must outlive that thread, and the live library ends as +the old or the restored one, never partial, with the safety backup kept. + +Every positive wait is on an event. The one bounded wait (``SETTLE``) is a +negative check: the cancelled task must still be pending while its worker is +held. +""" + +import asyncio +import os +import shutil +import sqlite3 +import threading +from unittest.mock import patch + +from database import restore_stage +from database.models import PromptModel +from py.api import admin as admin_mod +from py.api.jobs import RESTORE_JOB_KIND + +try: + from tests.test_restore_plans import RestoreTestCase, _texts +except ImportError: # discovered with tests/ as the top-level directory + from test_restore_plans import RestoreTestCase, _texts + +WAIT = 60 # a ceiling only a hung test reaches +SETTLE = 0.3 # the negative check's bound: the old code's cleanup takes ms + + +class _Gate: + """Hold one blocking call on its worker thread until released.""" + + def __init__(self, real): + self.real = real + self.entered = threading.Event() + self.release = threading.Event() + self.done = threading.Event() + + def method(self): + """A function to patch onto the class (bound like the real one).""" + + def held(*args): + return self(*args) + + return held + + def __call__(self, *args): + self.entered.set() + self.release.wait(WAIT) + try: + return self.real(*args) + finally: + self.done.set() + + +class ShutdownRestoreCase(RestoreTestCase): + def gate(self, real): + gate = _Gate(real) + # A failed check never leaves the worker held until WAIT. + self._gates = getattr(self, "_gates", []) + [gate] + self.addCleanup(gate.release.set) + return gate + + def stages(self): + """The stage folders (the upload's prefix starts the same way).""" + return [ + name + for name in super().stages() + if os.path.isdir(os.path.join(self.tmpdir, name)) + ] + + def uploads(self): + """The upload files (not the -wal/-shm its readers may add).""" + return sorted( + name + for name in os.listdir(self.tmpdir) + if name.startswith(restore_stage.UPLOAD_PREFIX) and name.endswith(".tmp") + ) + + def fake_upload(self, texts): + """Make ``_receive_restore_upload`` hand over a prepared upload.""" + self.upload(texts, "source.db") + source = os.path.join(self.tmpdir, "uploads", "source.db") + target = os.path.join(self.tmpdir, restore_stage.UPLOAD_PREFIX + "x.tmp") + shutil.copyfile(source, target) + + async def receive(request, uploads): + uploads.append(target) + return target, None + + return patch.object(self.api, "_receive_restore_upload", receive) + + async def wait_thread(self, event): + loop = asyncio.get_running_loop() + self.assertTrue(await loop.run_in_executor(None, event.wait, WAIT)) + + async def assert_still_pending(self, task): + done, _ = await asyncio.wait({task}, timeout=SETTLE) + if done: + for gate in getattr(self, "_gates", ()): + gate.release.set() # let the worker end before teardown + self.fail("the cancelled task ended while its worker still ran") + + +class TestLegacyRestoreAtShutdown(ShutdownRestoreCase): + async def test_cancel_during_publish_waits_for_the_worker(self): + self.save("old") + old_db = self.api.db + gate = self.gate(PromptModel.publish_staged) + with ( + self.fake_upload(["restored"]), + patch.object(PromptModel, "publish_staged", gate.method()), + ): + task = asyncio.ensure_future(self.api.restore_database(None)) + await self.wait_thread(gate.entered) + self.assertEqual(len(self.stages()), 1) + task.cancel() + await self.assert_still_pending(task) + # The worker still has its stage and upload, and runs alone. + self.assertEqual(len(self.stages()), 1) + self.assertEqual(len(self.uploads()), 1) + self.assertIsNotNone(self.api._jobs.active(RESTORE_JOB_KIND)) + gate.release.set() + with self.assertRaises(asyncio.CancelledError): + await asyncio.wait_for(task, WAIT) + self.assertTrue(gate.done.is_set()) + self.assertEqual(_texts(self.db_path), ["restored"]) + self.assertIsNot(self.api.db, old_db) # the restored library adopted + self.assertEqual(len(self.backups()), 1) # the safety backup is kept + self.assertEqual(self.stages(), []) + self.assertEqual(self.uploads(), []) + self.assertIsNone(self.api._jobs.active(RESTORE_JOB_KIND)) + + async def test_cancel_during_staging_waits_and_changes_nothing(self): + self.save("old") + gate = self.gate(PromptModel.stage_restore) + with ( + self.fake_upload(["restored"]), + patch.object(PromptModel, "stage_restore", gate.method()), + ): + task = asyncio.ensure_future(self.api.restore_database(None)) + await self.wait_thread(gate.entered) + task.cancel() + await self.assert_still_pending(task) + self.assertEqual(len(self.uploads()), 1) # still being read + self.assertIsNotNone(self.api._jobs.active(RESTORE_JOB_KIND)) + gate.release.set() + with self.assertRaises(asyncio.CancelledError): + await asyncio.wait_for(task, WAIT) + self.assertEqual(_texts(self.db_path), ["old"]) + self.assertEqual(self.backups(), []) + self.assertEqual(self.stages(), []) # the finished stage is removed + self.assertEqual(self.uploads(), []) + self.assertIsNone(self.api._jobs.active(RESTORE_JOB_KIND)) + + async def test_a_publish_that_fails_after_the_cancel_still_cancels(self): + self.save("old") + + def refuse(model, staged): + raise sqlite3.OperationalError("refused") + + gate = self.gate(refuse) + with ( + self.fake_upload(["restored"]), + patch.object(PromptModel, "publish_staged", gate.method()), + ): + task = asyncio.ensure_future(self.api.restore_database(None)) + await self.wait_thread(gate.entered) + task.cancel() + await self.assert_still_pending(task) + gate.release.set() + # Not answered as a 500: the cancel goes on. + with self.assertRaises(asyncio.CancelledError): + await asyncio.wait_for(task, WAIT) + self.assertEqual(_texts(self.db_path), ["old"]) + self.assertEqual(self.stages(), []) + self.assertEqual(self.uploads(), []) + self.assertIsNone(self.api._jobs.active(RESTORE_JOB_KIND)) + + async def test_a_second_cancel_still_adopts_the_restored_library(self): + self.save("old") + old_db = self.api.db + publish = self.gate(PromptModel.publish_staged) + prune = self.gate(admin_mod._prune_safety_backups) + with ( + self.fake_upload(["restored"]), + patch.object(PromptModel, "publish_staged", publish.method()), + patch.object(admin_mod, "_prune_safety_backups", prune), + ): + task = asyncio.ensure_future(self.api.restore_database(None)) + await self.wait_thread(publish.entered) + task.cancel() + await self.assert_still_pending(task) + publish.release.set() + await self.wait_thread(prune.entered) # adopting the library + task.cancel() # the loop's own shutdown cancels again + await self.assert_still_pending(task) + prune.release.set() + with self.assertRaises(asyncio.CancelledError): + await asyncio.wait_for(task, WAIT) + self.assertEqual(_texts(self.db_path), ["restored"]) + self.assertIsNot(self.api.db, old_db) + self.assertEqual(len(self.backups()), 1) + self.assertEqual(self.stages(), []) + self.assertIsNone(self.api._jobs.active(RESTORE_JOB_KIND)) + + +class TestReviewedRestoreAtShutdown(ShutdownRestoreCase): + async def test_cancelled_job_waits_for_the_publish(self): + self.save("old") + old_db = self.api.db + body = await self.preview(self.upload(["restored"])) + gate = self.gate(PromptModel.publish_staged) + with patch.object(PromptModel, "publish_staged", gate.method()): + started = await self.apply(body["token"]) + job = self.api._jobs.get(started["job"]["id"]) + await self.wait_thread(gate.entered) + job.task.cancel() + await self.assert_still_pending(job.task) + self.assertEqual(len(self.stages()), 1) + self.assertTrue(job.running) # its exclusive admission holds + gate.release.set() + with self.assertRaises(asyncio.CancelledError): + await asyncio.wait_for(job.task, WAIT) + self.assertEqual(_texts(self.db_path), ["restored"]) + self.assertIsNot(self.api.db, old_db) + self.assertEqual(len(self.backups()), 1) + self.assertEqual(self.stages(), []) + self.assertFalse(job.running) diff --git a/tests/test_thumbnail_auto.py b/tests/test_thumbnail_auto.py index 71697131..18c92db1 100644 --- a/tests/test_thumbnail_auto.py +++ b/tests/test_thumbnail_auto.py @@ -978,6 +978,27 @@ class TestThumbnailOnCapture(AutoThumbnailCase): (self.output_dir / "thumbnails" / f"{name}_thumb.webp").is_file() ) + async def test_a_worker_refused_at_interpreter_exit_skips_quietly(self): + """#375: refused once the interpreter exits; nothing stays queued.""" + + class Refused(threading.Thread): + def start(self): + raise RuntimeError("can't create new thread at interpreter shutdown") + + namespace = type(sys)("threading_copy") + namespace.__dict__.update(vars(threading)) + namespace.Thread = Refused + thumbnailer = self.api._auto_thumbnails() + path = make_png(self.output_dir / "late.png", size=(32, 32)) + with ( + patch.object(auto, "threading", namespace), + self.assertNoLogs("prompt_manager.api", "INFO"), + ): + self.assertEqual(self.api.queue_capture_thumbnail(str(path)), "skipped") + self.assertFalse(thumbnailer._worker_running) + self.assertEqual(len(thumbnailer._queue), 0) + self.assertTrue(thumbnailer.wait_idle(0)) + def test_queue_is_bounded(self): thumbnailer = auto.AutoThumbnailer( lambda *a: {"action": "generated"}, diff --git a/utils/image_monitor.py b/utils/image_monitor.py index 6e13dede..7c70d876 100644 --- a/utils/image_monitor.py +++ b/utils/image_monitor.py @@ -35,6 +35,7 @@ from watchdog.events import FileSystemEventHandler from .hashing import generate_prompt_hash from .metadata_extractor import ComfyUIMetadataExtractor +from .parallel import note_refusal, refused_at_shutdown from .logging_config import get_logger from .prompt_graph import resolve_text, run_prompt_nodes from .prompt_pair_resolver import resolve_prompt_pair @@ -194,8 +195,9 @@ class ImageGenerationHandler(FileSystemEventHandler): and once its size has stopped changing. Returns: - False when the path is already pending, the pending set is full or - the handler is closed + False when the path is already pending, the pending set is full, + the handler is closed or the interpreter is exiting (its worker + cannot start) """ with self._pending_lock: if self._closed: @@ -211,7 +213,15 @@ class ImageGenerationHandler(FileSystemEventHandler): return False due = time.monotonic() + self.processing_delay self._pending[image_path] = (prompt_snapshot, due) - self._ensure_worker() + try: + self._ensure_worker() + except RuntimeError as error: + # No worker will ever take it: never leave it pending (#375). + self._pending.pop(image_path, None) + if not refused_at_shutdown(error): + raise + note_refusal(self.logger, "The image worker") + return False self._work.put(image_path) return True diff --git a/utils/parallel.py b/utils/parallel.py index 8b183dd8..65dcca44 100644 --- a/utils/parallel.py +++ b/utils/parallel.py @@ -5,17 +5,51 @@ a CUDA-initialised, multi-threaded process is unsafe and the spawn start method would re-import ComfyUI's main module in every child. Pillow releases the GIL while decoding, resizing and encoding, so threads still scale close to linearly for both jobs. + +Interpreter exit (#375): ``threading._shutdown`` shuts every thread pool +(``submit`` then raises "cannot schedule new futures after interpreter +shutdown") before it joins the non-daemon threads and long before the package +teardown runs, while our daemon threads keep working. Measured on 3.12 and +3.14, ``Thread.start`` still succeeds in that window; it is refused ("can't +create new thread at interpreter shutdown") only once the interpreter is +marked finalizing, after the ``atexit`` callbacks. ``refused_at_shutdown`` +recognises both; ``note_refusal`` logs one DEBUG line per place, with no +traceback. """ +import logging +import threading from concurrent.futures import ThreadPoolExecutor +SHUTDOWN_REFUSAL = "interpreter shutdown" +_logger = logging.getLogger("prompt_manager.parallel") +_noted = set() +_noted_lock = threading.Lock() + + +def refused_at_shutdown(error): + """Whether ``error`` is a thread or pool start refused at interpreter exit.""" + return isinstance(error, RuntimeError) and SHUTDOWN_REFUSAL in str(error) + + +def note_refusal(logger, what): + """Log once per ``what``, at DEBUG and without a traceback, that ``what`` + was refused because the interpreter is exiting.""" + key = (logger.name, what) + with _noted_lock: + if key in _noted: + return + _noted.add(key) + logger.debug(f"{what}: refused, the interpreter is exiting") + def map_parallel(func, items, workers): """Apply ``func`` to every item on up to ``workers`` threads, keeping order. ``workers`` of 1 (or fewer) runs on the calling thread with no pool. An exception in ``func`` propagates to the caller, so callers that must - survive bad input wrap ``func`` themselves. + survive bad input wrap ``func`` themselves. A pool refused at interpreter + exit runs the items it did not start on the calling thread (#375). """ items = list(items) workers = max(1, int(workers or 1)) @@ -23,4 +57,35 @@ def map_parallel(func, items, workers): return [func(item) for item in items] max_workers = min(workers, len(items)) with ThreadPoolExecutor(max_workers, thread_name_prefix="pm-worker") as pool: - return list(pool.map(func, items)) + futures = [] + try: + for item in items: + futures.append(pool.submit(func, item)) + except RuntimeError as error: + if not refused_at_shutdown(error): + raise + note_refusal(_logger, "A worker pool") + # Drop what was queued but not started; it runs here instead. + # (A refused Thread.start leaves its item queued without a future + # here: it is cancelled too, unless a busy pool thread took it in + # between, the one case where an item can run twice.) + pool.shutdown(wait=True, cancel_futures=True) + else: + return [future.result() for future in futures] + return _run_rest(func, items, futures) + + +def _run_rest(func, items, futures): + """The pool's finished results, and every item it did not run, run here. + + After ``shutdown(cancel_futures=True)`` each submitted future either ran + (its result or exception is kept) or was cancelled before it started. + """ + results = [] + for index, item in enumerate(items): + future = futures[index] if index < len(futures) else None + if future is not None and not future.cancelled(): + results.append(future.result()) + else: + results.append(func(item)) + return results From ef7e07c7658aad8e8095b1cb9f86855bb7ddeed1 Mon Sep 17 00:00:00 2001 From: Vito <vito.sansevero@gmail.com> Date: Wed, 28 Oct 2026 03:54:55 +0000 Subject: [PATCH 06/11] refactor(admin): open System for Diagnostics and logs (#330, PR 1) (#402) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(legacy): snapshot the routes and the settings response Pins every registered route and the key paths of GET /prompt_manager/settings while System S13 (#330) moves the legacy pages' tools to native System. Recorded on the unchanged main; cutover (#334) removes it. * test(legacy): prove each removed legacy control has native coverage TESTING.md gains the "Legacy System UI removed by S13" table (no rows yet). legacy-coverage.test.mjs fails when a removal cites a capability that is not covered, copies another native test, lists an id or method twice, or leaves the id or method on its page. legacy-system-links.spec.mjs holds the System link table and guards the kept admin flows. * refactor(admin): open System for Diagnostics and logs The admin page's Diagnostics and View Logs buttons become links that open System › Diagnostics & logs in a new tab. The legacy Diagnostics and System Logs dialogs and their methods go; TESTING.md lists them against their covered capabilities (rows B1–B7). * test(legacy): find a removed modal id passed to showModal and settle the links spec The removal check also finds an id passed as a string to a ...Modal() call and no longer reads data-id= as id=. The no-request test waits for the page's own start-up reads to finish, the sort check compares as the server sorts, and the kept-flow test checks the added prompt shows in the list. --- TESTING.md | 27 ++ tests/browser/legacy-system-links.spec.mjs | 163 +++++++ tests/fixtures/README.md | 16 + tests/fixtures/legacy_contract_snapshot.json | 200 +++++++++ tests/test_legacy_contract_snapshot.py | 136 ++++++ tests/vendor/legacy-coverage.test.mjs | 99 ++++- tools/vendor/legacy-coverage.mjs | 148 ++++++- web/admin.html | 169 +------ web/js/admin.js | 443 ------------------- 9 files changed, 788 insertions(+), 613 deletions(-) create mode 100644 tests/browser/legacy-system-links.spec.mjs create mode 100644 tests/fixtures/legacy_contract_snapshot.json create mode 100644 tests/test_legacy_contract_snapshot.py diff --git a/TESTING.md b/TESTING.md index b21f77c9..d66064e8 100644 --- a/TESTING.md +++ b/TESTING.md @@ -4103,6 +4103,33 @@ list stay unscoped: any row's control proves them. | API timeout `apiTimeout` | `gallery-settings.js:37` | Legacy gallery (S12-10; read by no request) | kept in legacy | — | defaults are a complete, sane settings object (gallery-settings.test.js) | | config.json-only keys | `py/config.py:212–234,722–739`; `py/api/diagnostics_report.py:185` | S12 report (read-only) | covered | `diagnostics` · button · "Download diagnostic report" | test_report_lists_config_only_keys (test_api_diagnostics_report.py) | +### Legacy System UI removed by S13 + +System S13 (#330) points the legacy pages' System buttons at native System (a link that +opens `workspace#/system/<id>` in a new tab) and removes the legacy copy of a capability +only when its row above is `covered`. One row per removed capability: Capability is the +exact Capability cell of its coverage row, Page is `admin` or `gallery`, Removed ids and +Removed methods list what went (`—` for none), and Native test copies that row's Test +cell. An id that stays on a System link is not listed, only its handler. + +`tests/vendor/legacy-coverage.test.mjs` fails when a row cites a capability that is not +`covered` or copies another test, when an id or method is listed twice for one page, or +when a listed id is still on its page (`id="…"` in the markup or the script, a +`getElementById`/`querySelector('#…')` lookup, an entry of a modal-id list) or a listed +method is still defined or called (`.name(`). A deletion listed nowhere is caught by the +kept-flow tests (`kept admin flows still work` in `legacy-system-links.spec.mjs` and the +legacy page workflows), not by this check. + +| Capability | Page | Removed ids | Removed methods | Native test | +| --- | --- | --- | --- | --- | +| Diagnostics run | admin | `#diagnosticsModal`, `#runDiagnosticsBtn`, `#diagnosticsContent` | `showDiagnosticsModal()`, `closeDiagnostics()`, `runDiagnostics()`, `renderDiagnostics()` | Run checks opens the shared Health report (workspace-system-diagnostics.spec.mjs) | +| Synthetic image-link test | admin | `#testImageLinkBtn` | `testImageLink()` | Create test link confirms first and names the prompt (workspace-system-diagnostics.spec.mjs) | +| View logs with level and limit | admin | `#logsModal`, `#logLevel`, `#logLimit`, `#refreshLogsBtn`, `#logsContainer` | `showLogsModal()`, `closeLogs()`, `refreshLogs()`, `displayLogs()` | level filter is minimum-level over all six choices (workspace-system-diagnostics.spec.mjs) | +| Log statistics | admin | `#logStatsTotal`, `#logStatsErrors`, `#logStatsWarnings`, `#logStatsLevel`, `#logStatsSize` | `loadLogStats()`, `formatBytes()` | log statistics show entries, errors, warnings, files and size (workspace-system-diagnostics.spec.mjs) | +| Log files list and download | admin | `#downloadLogsBtn`, `#logFilesList` | `loadLogFiles()`, `downloadLogs()`, `downloadLogFile()` | each listed file downloads by its returned name and a link has no Download (workspace-system-diagnostics.spec.mjs) | +| Clear/truncate logs | admin | `#clearLogsBtn` | `clearLogs()` | Clear all logs states both counts, confirms first and reads everything again (workspace-system-diagnostics.spec.mjs) | +| Set log level (incl. CRITICAL), console logging | admin | `#updateLogConfigBtn`, `#setLogLevel`, `#consoleLogging` | `loadLogConfig()`, `updateLogConfig()` | runtime level (including CRITICAL) and console logging save and say they apply until the server restarts (workspace-system-diagnostics.spec.mjs) | + ### Browser cache ownership (#222) The legacy gallery's Clear cache used to delete every localStorage key starting `pm_` or diff --git a/tests/browser/legacy-system-links.spec.mjs b/tests/browser/legacy-system-links.spec.mjs new file mode 100644 index 00000000..50b1c420 --- /dev/null +++ b/tests/browser/legacy-system-links.spec.mjs @@ -0,0 +1,163 @@ +// System S13 (#330): the legacy admin and gallery pages open native System for their System tools. Each replaced +// control is a plain link (`workspace#/system/<id>`, a new tab), and the legacy flows S13 keeps still work. +import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs'; +import { test, expect } from './fixtures.mjs'; + +const PAGES = Object.freeze({ admin: '/prompt_manager/admin', gallery: '/prompt_manager/gallery' }); +const LABELS = Object.freeze({ + overview: 'Overview', + appearance: 'Appearance', + sources: 'Sources & scans', + integrations: 'Integrations', + tagging: 'Auto-tagging', + maintenance: 'Maintenance', + backups: 'Backup & export', + diagnostics: 'Diagnostics & logs', +}); + +// One row per System link of the design's §2 map: the page, the control's label, how to find it, the section it opens +// and what a test must arrange before the control shows. +const LINKS = Object.freeze([ + { page: 'admin', label: 'Diagnostics', locate: page => page.locator('#diagnosticsBtn'), section: 'diagnostics' }, + { page: 'admin', label: 'View Logs', locate: page => page.locator('#logsBtn'), section: 'diagnostics' }, +]); + +async function post(request, url, data) { + await expect(await request.post(url, data === undefined ? {} : { data })).toBeOK(); +} + +test.beforeEach(async ({ page, request, browserName }) => { + await startCoverage(page, browserName); + await post(request, '/__test__/reset'); + await post(request, '/__test__/system/reset'); + await post(request, '/__test__/tagging/reset'); + // The 3.2.0 update notice would cover the admin page's toolbar. + await page.addInitScript(() => { localStorage.setItem('pm_update_notice_dismissed', '3.2.0'); }); + page.legacyErrors = []; + page.on('pageerror', error => page.legacyErrors.push(error.message)); +}); + +test.afterEach(async ({ page, request, browserName }) => { + await saveCoverage(page, browserName); + await post(request, '/__test__/tagging/reset'); + await post(request, '/__test__/system/reset'); + expect(page.legacyErrors).toEqual([]); +}); + +/** Opens the legacy page with the link showing; returns the link. */ +async function openLink(page, request, link) { + if (link.arrange) await link.arrange(page, request); + await page.goto(PAGES[link.page]); + if (link.reveal) await link.reveal(page); + const control = link.locate(page); + await expect(control).toBeVisible(); + return control; +} + +for (const link of LINKS) { + test(`legacy ${link.page} ${link.label} opens System ${link.section}`, async ({ page, context, request }) => { + const control = await openLink(page, request, link); + const legacyUrl = page.url(); + await expect(control).toHaveAttribute('href', `workspace#/system/${link.section}`); + await expect(control).toHaveAttribute('target', '_blank'); + await expect(control).toHaveAttribute('rel', 'noopener'); + const opened = context.waitForEvent('page'); + await control.click(); + const system = await opened; + await expect(system).toHaveURL(new RegExp(`/prompt_manager/workspace#/system/${link.section}$`)); + // System's page heading is "System"; the open section is the pressed button of its section switcher. + const main = system.locator('#workspace-main'); + await expect(main.getByRole('heading', { level: 1, name: 'System', exact: true })).toBeVisible(); + await expect(main.getByRole('group', { name: 'System sections', exact: true }) + .getByRole('button', { name: LABELS[link.section], exact: true })).toHaveAttribute('aria-pressed', 'true'); + expect(page.url()).toBe(legacyUrl); + await system.close(); + }); +} + +test('a System link sends no request from the legacy page', async ({ page, context, request }) => { + for (const link of LINKS) { + const control = await openLink(page, request, link); + // The page's own start-up reads (list, film strips, settings) finish first; then the click must add none. + await page.waitForLoadState('networkidle'); + const sent = []; + const record = item => sent.push(`${item.method()} ${item.url()}`); + page.on('request', record); + const opened = context.waitForEvent('page'); + await control.click(); + const system = await opened; + await system.waitForLoadState('domcontentloaded'); + page.off('request', record); + expect(sent, `${link.page} ${link.label}`).toEqual([]); + await system.close(); + } +}); + +test('kept admin flows still work', async ({ page, context }) => { + await page.goto(PAGES.admin); + const texts = page.locator('#resultsList .prompt-text'); + await expect(texts.first()).toBeVisible(); + + // K1: sorting reorders the list. + for (const [value, order] of [['text_asc', 1], ['text_desc', -1]]) { + const listed = page.waitForResponse(item => /\/prompt_manager\/(recent|search)\?/.test(item.url()) && item.url().includes(`sort=${value}`)); + await page.locator('#sortBy').selectOption(value); + await listed; + await expect.poll(async () => { + const shown = await texts.allTextContents(); + // The server sorts case-insensitively (SQLite NOCASE), not by locale. + const key = text => text.toLowerCase(); + const sorted = [...shown].sort((a, b) => order * (key(a) < key(b) ? -1 : key(a) > key(b) ? 1 : 0)); + return shown.length > 1 && shown.join('\n') === sorted.join('\n'); + }).toBe(true); + } + // K3: Add Prompt saves a prompt the list then shows. + await page.locator('#addPromptBtn').click(); + await page.locator('#addPromptText').fill('Kept flow lighthouse'); + const saved = page.waitForResponse(item => item.request().method() === 'POST' && item.url().endsWith('/prompt_manager/save')); + await page.locator('#saveNewPromptBtn').click(); + expect((await saved).ok()).toBe(true); + await expect(page.locator('#addPromptModal')).toBeHidden(); + await expect(texts.filter({ hasText: 'Kept flow lighthouse' })).toBeVisible(); + + // K1: a search narrows the list, and a selection enables the bulk buttons. + const all = await texts.allTextContents(); + const word = all[0].trim().split(/\s+/)[0]; + const searched = page.waitForResponse(item => item.url().includes('/prompt_manager/search?') && item.url().includes('text=')); + await page.locator('#searchText').fill(word); + await page.locator('#searchBtn').click(); + await searched; + await expect.poll(async () => { + const shown = await texts.allTextContents(); + return shown.length > 0 && shown.length < all.length && shown.every(text => text.toLowerCase().includes(word.toLowerCase())); + }).toBe(true); + await expect(page.locator('#bulkDeleteBtn')).toBeDisabled(); + await page.locator('#resultsList .prompt-checkbox').first().check(); + for (const id of ['#bulkDeleteBtn', '#bulkTagBtn', '#bulkCategoryBtn']) await expect(page.locator(id)).toBeEnabled(); + + // K4: Stats reads the statistics again; Export downloads the prompts. + const stats = page.waitForResponse(item => item.url().endsWith('/prompt_manager/stats') && item.ok()); + await page.locator('#statsBtn').click(); + await stats; + await expect(page.getByText('Stats refreshed successfully!')).toBeVisible(); + const download = page.waitForEvent('download'); + await page.locator('#exportBtn').click(); + expect((await download).suggestedFilename()).toMatch(/^prompts-\d{4}-\d{2}-\d{2}\.json$/); + + // K5: Scan Images opens its dialog (with the quick backup) without starting a scan. + await page.locator('#scanBtn').click(); + await expect(page.locator('#scanModal')).toBeVisible(); + await expect(page.locator('#quickBackupBtn')).toBeVisible(); + await page.locator('#cancelScan').click(); + await expect(page.locator('#scanModal')).toBeHidden(); + + // K7: Metadata and Gallery open their pages in a new tab. + for (const [id, path] of [['#metadataBtn', '/prompt_manager/gallery.html'], ['#galleryBtn', '/prompt_manager/gallery']]) { + const opened = context.waitForEvent('page'); + await page.locator(id).click(); + const tab = await opened; + await tab.waitForLoadState('domcontentloaded'); + expect(new URL(tab.url()).pathname).toBe(path); + await tab.close(); + } +}); diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md index 25b61bc9..7e1476a2 100644 --- a/tests/fixtures/README.md +++ b/tests/fixtures/README.md @@ -113,3 +113,19 @@ by their own suites. The expectations start as the observed baseline. A change that alters an outcome edits the vector in the same commit. `tests/js/legacy-metadata-characterization.test.js` pins what the legacy viewers show on the same files. + +# Legacy contract snapshot + +`legacy_contract_snapshot.json` records what the legacy admin and gallery pages +talk to while System S13 (#330) moves their tools to native System: every +registered route as `"<METHOD> <path>"` (`routes`) and the key paths of +`GET /prompt_manager/settings` to depth 2 (`settings_keys`; list items give the +union of their keys as `name[].key`). `tests/test_legacy_contract_snapshot.py` +compares a temp-database API against it. Record it again only for an intended +change, and say why in the PR body: + +```sh +PM_RECORD_LEGACY_SNAPSHOT=1 python3 -m unittest tests.test_legacy_contract_snapshot +``` + +Cutover (#334) removes the test and this fixture. diff --git a/tests/fixtures/legacy_contract_snapshot.json b/tests/fixtures/legacy_contract_snapshot.json new file mode 100644 index 00000000..50a48f53 --- /dev/null +++ b/tests/fixtures/legacy_contract_snapshot.json @@ -0,0 +1,200 @@ +{ + "routes": [ + "DELETE /prompt_manager/delete/{prompt_id}", + "DELETE /prompt_manager/image-annotations/capture/{token}", + "DELETE /prompt_manager/images/favorite", + "DELETE /prompt_manager/images/{image_id}", + "DELETE /prompt_manager/pairs/{pair_id}", + "DELETE /prompt_manager/prompts/{prompt_id}/tags", + "DELETE /prompt_manager/restore/preview/{token}", + "DELETE /prompt_manager/saved-views/{view_id}", + "DELETE /prompt_manager/tags/{tag_name}", + "GET /prompt_manager/admin", + "GET /prompt_manager/autotag/models", + "GET /prompt_manager/backup", + "GET /prompt_manager/backup/status", + "GET /prompt_manager/categories", + "GET /prompt_manager/diagnostics", + "GET /prompt_manager/export", + "GET /prompt_manager/gallery", + "GET /prompt_manager/gallery.html", + "GET /prompt_manager/gallery/subfolders", + "GET /prompt_manager/history/runs/{run_id}", + "GET /prompt_manager/history/runs/{run_id}/output/current/inspection", + "GET /prompt_manager/history/runs/{run_id}/output/current/media", + "GET /prompt_manager/history/runs/{run_id}/output/current/thumbnail", + "GET /prompt_manager/history/runs/{run_id}/outputs", + "GET /prompt_manager/image-annotations/capture/{token}", + "GET /prompt_manager/images/all", + "GET /prompt_manager/images/inspect/{image_path}", + "GET /prompt_manager/images/output", + "GET /prompt_manager/images/prompt/{image_path}", + "GET /prompt_manager/images/recent", + "GET /prompt_manager/images/search", + "GET /prompt_manager/images/serve/{filepath}", + "GET /prompt_manager/images/thumbnails/plan", + "GET /prompt_manager/images/thumbnails/status", + "GET /prompt_manager/images/{image_id}/file", + "GET /prompt_manager/jobs", + "GET /prompt_manager/jobs/{job_id}", + "GET /prompt_manager/jobs/{job_id}/events", + "GET /prompt_manager/jobs/{job_id}/report", + "GET /prompt_manager/js/{filepath}", + "GET /prompt_manager/lib/{filepath}", + "GET /prompt_manager/library", + "GET /prompt_manager/library/export", + "GET /prompt_manager/library/options", + "GET /prompt_manager/library/{positive_id}", + "GET /prompt_manager/library/{positive_id}/history", + "GET /prompt_manager/library/{positive_id}/history/pairs/{pair_id}/newest", + "GET /prompt_manager/library/{positive_id}/images", + "GET /prompt_manager/logs", + "GET /prompt_manager/logs/config", + "GET /prompt_manager/logs/download/{filename}", + "GET /prompt_manager/logs/files", + "GET /prompt_manager/logs/stats", + "GET /prompt_manager/lora/detect", + "GET /prompt_manager/lora/status", + "GET /prompt_manager/lora/trigger-words", + "GET /prompt_manager/maintenance/duplicates/{job_id}", + "GET /prompt_manager/pairs", + "GET /prompt_manager/pairs/export", + "GET /prompt_manager/pairs/{pair_id}", + "GET /prompt_manager/prompts/{prompt_id}/images", + "GET /prompt_manager/recent", + "GET /prompt_manager/review/events", + "GET /prompt_manager/review/items", + "GET /prompt_manager/review/items/{item_id}", + "GET /prompt_manager/saved-views", + "GET /prompt_manager/scan/status", + "GET /prompt_manager/scan_duplicates", + "GET /prompt_manager/scan_duplicates/status", + "GET /prompt_manager/scan_output_dir", + "GET /prompt_manager/search", + "GET /prompt_manager/settings", + "GET /prompt_manager/stats", + "GET /prompt_manager/subfolders", + "GET /prompt_manager/system/diagnostics", + "GET /prompt_manager/system/health", + "GET /prompt_manager/system/library", + "GET /prompt_manager/system/overview", + "GET /prompt_manager/system/statistics", + "GET /prompt_manager/tags", + "GET /prompt_manager/tags/filter", + "GET /prompt_manager/tags/stats", + "GET /prompt_manager/tags/suggest", + "GET /prompt_manager/tags/{tag_name}/prompts", + "GET /prompt_manager/test", + "GET /prompt_manager/vocabulary/categories", + "GET /prompt_manager/vocabulary/tags", + "GET /prompt_manager/vocabulary/tags/{tag_id}/impact", + "GET /prompt_manager/web", + "GET /prompt_manager/workspace", + "PATCH /prompt_manager/library/{positive_id}", + "PATCH /prompt_manager/pairs/{pair_id}", + "PATCH /prompt_manager/saved-views/{view_id}", + "PATCH /prompt_manager/vocabulary/tags/{tag_id}", + "POST /prompt_manager/autotag/apply", + "POST /prompt_manager/autotag/batch/preview", + "POST /prompt_manager/autotag/download/{model_type}", + "POST /prompt_manager/autotag/single", + "POST /prompt_manager/autotag/start", + "POST /prompt_manager/autotag/unload", + "POST /prompt_manager/bulk/category", + "POST /prompt_manager/bulk/delete", + "POST /prompt_manager/bulk/tags", + "POST /prompt_manager/cleanup", + "POST /prompt_manager/delete_duplicate_images", + "POST /prompt_manager/diagnostics/test-link", + "POST /prompt_manager/gallery/rescan", + "POST /prompt_manager/image-annotations/observe", + "POST /prompt_manager/image-annotations/suggest", + "POST /prompt_manager/images/clear-thumbnails", + "POST /prompt_manager/images/generate-thumbnails", + "POST /prompt_manager/images/generate-thumbnails/progress", + "POST /prompt_manager/images/link", + "POST /prompt_manager/images/reveal", + "POST /prompt_manager/jobs/autotag", + "POST /prompt_manager/jobs/duplicates", + "POST /prompt_manager/jobs/image-tags", + "POST /prompt_manager/jobs/model-download", + "POST /prompt_manager/jobs/scan", + "POST /prompt_manager/jobs/thumbnails", + "POST /prompt_manager/jobs/{job_id}/cancel", + "POST /prompt_manager/library", + "POST /prompt_manager/library/{positive_id}/variants", + "POST /prompt_manager/logs/config", + "POST /prompt_manager/logs/truncate", + "POST /prompt_manager/lora/enable", + "POST /prompt_manager/lora/import/apply", + "POST /prompt_manager/lora/import/preview", + "POST /prompt_manager/lora/refresh-cache", + "POST /prompt_manager/lora/scan", + "POST /prompt_manager/maintenance", + "POST /prompt_manager/maintenance/apply", + "POST /prompt_manager/maintenance/preview", + "POST /prompt_manager/pairs", + "POST /prompt_manager/prompts/tags", + "POST /prompt_manager/prompts/{prompt_id}/tags", + "POST /prompt_manager/restore", + "POST /prompt_manager/restore/apply", + "POST /prompt_manager/restore/preview", + "POST /prompt_manager/review/items/{item_id}/decision", + "POST /prompt_manager/save", + "POST /prompt_manager/saved-views", + "POST /prompt_manager/scan", + "POST /prompt_manager/scan/cancel", + "POST /prompt_manager/scan_duplicates/stream", + "POST /prompt_manager/settings", + "POST /prompt_manager/settings/validate-source", + "POST /prompt_manager/system/health", + "POST /prompt_manager/system/library/keep", + "POST /prompt_manager/system/library/move", + "POST /prompt_manager/tags/merge", + "POST /prompt_manager/vocabulary/categories/apply", + "POST /prompt_manager/vocabulary/categories/preview", + "POST /prompt_manager/vocabulary/tags", + "POST /prompt_manager/vocabulary/tags/{tag_id}/delete", + "PUT /prompt_manager/image-annotations/tags", + "PUT /prompt_manager/images/favorite", + "PUT /prompt_manager/prompts/{prompt_id}", + "PUT /prompt_manager/prompts/{prompt_id}/rating", + "PUT /prompt_manager/tags/{tag_name}" + ], + "settings_keys": [ + "monitor", + "monitor.observer_alive", + "monitor.present", + "monitor.scheduled", + "revision", + "settings", + "settings.autotag_character_threshold", + "settings.autotag_existing", + "settings.autotag_general_threshold", + "settings.autotag_model_type", + "settings.autotag_suggestion_limit", + "settings.autotag_template", + "settings.cpu_count", + "settings.gallery_root_path", + "settings.gallery_root_paths", + "settings.infinite_scroll", + "settings.monitored_directories", + "settings.monitoring_enabled", + "settings.result_timeout", + "settings.safety_backup", + "settings.thumbnail_edge", + "settings.thumbnail_format", + "settings.thumbnail_quality", + "settings.webui_display_mode", + "settings.worker_threads", + "sources", + "sources.auto_detected", + "sources.configured", + "sources.effective", + "success", + "thumbnail_encoders", + "thumbnail_encoders.jpeg", + "thumbnail_encoders.jxl", + "thumbnail_encoders.webp" + ] +} diff --git a/tests/test_legacy_contract_snapshot.py b/tests/test_legacy_contract_snapshot.py new file mode 100644 index 00000000..1ed7b7a0 --- /dev/null +++ b/tests/test_legacy_contract_snapshot.py @@ -0,0 +1,136 @@ +"""The legacy pages' backend contract does not move under System S13 (#330). + +S13 points the legacy admin and gallery buttons at native System and removes +their duplicate dialogs, changing no backend file. This snapshot pins what the +legacy pages talk to: every registered route (method and path) and the key +paths of ``GET /prompt_manager/settings``, the one route whose legacy request +body S13 shrinks. Record again only with a reason in the PR body: + + PM_RECORD_LEGACY_SNAPSHOT=1 python3 -m unittest tests.test_legacy_contract_snapshot + +Cutover (#334) removes this test and its fixture. +""" + +import json +import os +import unittest +from pathlib import Path + +try: + from tests.test_api_admin import AdminAPITestCase +except ImportError: # discovered with tests/ as the top-level directory + from test_api_admin import AdminAPITestCase + +from py.config import GalleryConfig # noqa: E402 + +SNAPSHOT = ( + Path(__file__).resolve().parent / "fixtures" / "legacy_contract_snapshot.json" +) +RECORD_ENV = "PM_RECORD_LEGACY_SNAPSHOT" +SETTINGS = "/prompt_manager/settings" +DEPTH = 2 +CHANGED = ( + "the legacy contract changed; if the change is intended (a new route, a new" + " settings key), record it with PM_RECORD_LEGACY_SNAPSHOT=1 and say why in" + " the PR body" +) + + +def key_paths(value, prefix="", depth=1): + """The key paths of a JSON object to ``DEPTH`` levels. + + ``{"a": {"b": 1}}`` gives ``a`` and ``a.b``; the items of a list of + objects give the union of their keys as ``a[].b``. + """ + paths = set() + for key, child in value.items(): + path = f"{prefix}{key}" + paths.add(path) + if depth >= DEPTH: + continue + if isinstance(child, dict): + paths |= key_paths(child, f"{path}.", depth + 1) + elif isinstance(child, list): + for item in child: + if isinstance(item, dict): + paths |= key_paths(item, f"{path}[].", depth + 1) + return paths + + +class TestLegacyContractSnapshot(AdminAPITestCase): + """Routes and settings keys against tests/fixtures/legacy_contract_snapshot.json. + + The base case isolates config.json (``PROMPT_MANAGER_CONFIG_PATH`` in the + temp tree), stubs ComfyUI's ``folder_paths`` and resets the gallery roots; + nothing here reads the maintainer's install. + """ + + async def asyncSetUp(self): + await super().asyncSetUp() + self.api._get_all_output_dirs = lambda: [str(self.output_dir)] + GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)] + + def _routes(self): + return sorted( + { + f"{route.method} {route.resource.canonical}" + for route in self.app.router.routes() + if route.method != "HEAD" + } + ) + + async def _settings_keys(self): + resp = await self.client.request("GET", SETTINGS) + body = await resp.json() + self.assertEqual(resp.status, 200, body) + return sorted(key_paths(body)) + + def _snapshot(self): + with open(SNAPSHOT, encoding="utf-8") as f: + return json.load(f) + + def _record(self, key, value): + recorded = self._snapshot() if SNAPSHOT.exists() else {} + recorded[key] = value + with open(SNAPSHOT, "w", encoding="utf-8", newline="\n") as f: + json.dump(recorded, f, indent=2) + f.write("\n") + + async def test_routes_match_the_snapshot(self): + routes = self._routes() + if os.environ.get(RECORD_ENV) == "1": + self._record("routes", routes) + self.assertGreater(len(routes), 50) + self.assertEqual(routes, self._snapshot()["routes"], CHANGED) + + async def test_settings_response_keeps_its_keys(self): + keys = await self._settings_keys() + if os.environ.get(RECORD_ENV) == "1": + self._record("settings_keys", keys) + self.assertIn("settings.infinite_scroll", keys) + self.assertEqual(keys, self._snapshot()["settings_keys"], CHANGED) + + +class TestKeyPaths(unittest.TestCase): + def test_depth_and_list_items(self): + value = { + "success": True, + "settings": {"a": 1, "deep": {"hidden": 1}}, + "rows": [{"path": "x"}, {"label": "y"}], + } + self.assertEqual( + sorted(key_paths(value)), + [ + "rows", + "rows[].label", + "rows[].path", + "settings", + "settings.a", + "settings.deep", + "success", + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/vendor/legacy-coverage.test.mjs b/tests/vendor/legacy-coverage.test.mjs index c1bf38d7..4ab74223 100644 --- a/tests/vendor/legacy-coverage.test.mjs +++ b/tests/vendor/legacy-coverage.test.mjs @@ -1,6 +1,7 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { COLUMNS, CONTROL_EXEMPTIONS, STATUSES, declaredTitles, parseControl, parseCoverageTable, readCoverageTable, +import { COLUMNS, CONTROL_EXEMPTIONS, PAGES, REMOVAL_COLUMNS, STATUSES, declaredTitles, duplicateRemovals, idUses, methodUses, + parseControl, parseCoverageTable, parseRemovalTable, readCoverageTable, readPage, readRemovalTable, removalProblems, rowProblems } from '../../tools/vendor/legacy-coverage.mjs'; // TESTING.md "Legacy System capability coverage" (#249 System S12, #329): every legacy System capability names an @@ -114,3 +115,99 @@ test('a covered row names its native control unless it is exempt', () => { assert.deepEqual([row.Status, row['Native control']], ['covered', '—']); } }); + +// TESTING.md "Legacy System UI removed by S13" (#330): a legacy element is removed only when its capability is +// `covered`, the removal copies that row's native test, and the element is really gone from its page. +const COVERAGE_HEADER = '| Capability | Legacy anchor | Owner | Status | Native control | Test |\n| --- | --- | --- | --- | --- | --- |'; +const REMOVAL_HEADER = '| Capability | Page | Removed ids | Removed methods | Native test |\n| --- | --- | --- | --- | --- |'; +const FIXTURE_COVERAGE = parseCoverageTable(['### Legacy System capability coverage', '', COVERAGE_HEADER, + '| View logs | `admin.html:1` | S12 Logs | covered | `diagnostics` · combobox · "Level" | level filter works (logs.spec.mjs) |', + '| Infinite scrolling default | `admin.html:2` | Legacy prompt list | kept in legacy | — | scrolls (list-state.test.js) |', + ''].join('\n')); +const removalRow = cells => parseRemovalTable(['### Legacy System UI removed by S13', '', REMOVAL_HEADER, `| ${cells} |`, ''] + .join('\n'))[0]; +const EMPTY_PAGE = () => ({ html: '<button id="kept">Kept</button>', script: 'class Admin {\n kept() {}\n}\n' }); + +test('every removed legacy element cites a covered capability and its native test', () => { + const coverage = readCoverageTable(); + const rows = readRemovalTable(); + for (const row of rows) { + const cited = coverage.find(item => item.Capability === row.Capability); + assert.ok(cited, `TESTING.md:${row.line} cites "${row.Capability}"`); + assert.equal(cited.Status, 'covered', `TESTING.md:${row.line} ${row.Capability}`); + assert.equal(row['Native test'], cited.Test, `TESTING.md:${row.line} ${row.Capability}`); + assert.ok(Object.keys(PAGES).includes(row.Page), `TESTING.md:${row.line} page ${row.Page}`); + } + assert.deepEqual(duplicateRemovals(rows), []); + assert.deepEqual(REMOVAL_COLUMNS, ['Capability', 'Page', 'Removed ids', 'Removed methods', 'Native test']); +}); + +test('a removed id or method is gone from its page', () => { + const coverage = readCoverageTable(); + for (const row of readRemovalTable()) { + assert.deepEqual(removalProblems(row, coverage, page => readPage(page)), [], `TESTING.md:${row.line} ${row.Capability}`); + } + // Each way a page can still hold an id or a method is found; a call through another object, or a longer name, is not + // a definition. + const page = { + html: '<div id="logsModal"></div><button onclick="window.admin.closeLogs()">x</button>', + script: [ + "document.getElementById('logLevel').value;", + "this.showModal('diagnosticsModal');", + '<span data-id="logsContainer"></span>', + 'document.querySelector("#logLimit option");', + "const backdrops = ['settingsModal', 'duplicatesModal'];", + 'class Admin {', + ' async refreshLogs(level) {', + ' }', + '} displayLogs(logs) {', + ' }', + 'this.loadLogStats();', + '// downloadLogs() {', + 'other.formatBytesLater(1);', + ].join('\n'), + }; + assert.deepEqual(idUses('logsModal', page), ['id="logsModal" in the markup']); + assert.deepEqual(idUses('logLevel', page), ["getElementById('logLevel')"]); + assert.deepEqual(idUses('logLimit', page), ["querySelector('#logLimit')"]); + assert.deepEqual(idUses('duplicatesModal', page), ["'duplicatesModal' in a modal list"]); + assert.deepEqual(idUses('logLevelX', page), []); + assert.deepEqual(idUses('diagnosticsModal', page), ["a …Modal('diagnosticsModal') call"]); + assert.deepEqual(idUses('logsContainer', page), []); + assert.deepEqual(methodUses('closeLogs', page), ['.closeLogs( called in the markup']); + assert.deepEqual(methodUses('refreshLogs', page), ['refreshLogs() defined']); + assert.deepEqual(methodUses('displayLogs', page), ['displayLogs() defined']); + assert.deepEqual(methodUses('loadLogStats', page), ['.loadLogStats( called']); + assert.deepEqual(methodUses('downloadLogs', page), []); + assert.deepEqual(methodUses('formatBytes', page), []); + // One id or method is listed on one row per page. + const twice = [removalRow('View logs | admin | `#logsModal` | — | level filter works (logs.spec.mjs)'), + { ...removalRow('View logs | admin | `#logsModal` | — | level filter works (logs.spec.mjs)'), line: 9 }]; + assert.match(duplicateRemovals(twice).join(), /admin #logsModal is listed on rows/); + assert.deepEqual(removalProblems(removalRow('View logs | admin | `logsModal`, `#ok` | `close` | level filter works (logs.spec.mjs)'), + FIXTURE_COVERAGE, EMPTY_PAGE), ['malformed entry "logsModal"', 'malformed entry "close"']); +}); + +test('a removal citing a kept-in-legacy capability fails', () => { + const row = removalRow('Infinite scrolling default | admin | `#infiniteScrollDefault` | — | scrolls (list-state.test.js)'); + assert.deepEqual(removalProblems(row, FIXTURE_COVERAGE, EMPTY_PAGE), ['cites a kept in legacy capability']); + const unknown = removalRow('Something else | admin | — | — | scrolls (list-state.test.js)'); + assert.deepEqual(removalProblems(unknown, FIXTURE_COVERAGE, EMPTY_PAGE), ['cites no single coverage row "Something else"']); +}); + +test('a removal whose native test differs fails', () => { + const row = removalRow('View logs | admin | `#logsModal` | `showLogsModal()` | another title (logs.spec.mjs)'); + assert.deepEqual(removalProblems(row, FIXTURE_COVERAGE, EMPTY_PAGE), + ['its native test differs from the coverage row\'s "level filter works (logs.spec.mjs)"']); + assert.deepEqual(removalProblems({ ...row, 'Native test': 'level filter works (logs.spec.mjs)' }, FIXTURE_COVERAGE, EMPTY_PAGE), []); +}); + +test('a removed id still on its page fails', () => { + const row = removalRow('View logs | admin | `#logsModal` | `showLogsModal()` | level filter works (logs.spec.mjs)'); + const page = () => ({ html: '<div id="logsModal" class="hidden"></div>', script: 'class Admin {\n showLogsModal() {\n }\n}\n' }); + assert.deepEqual(removalProblems(row, FIXTURE_COVERAGE, page), [ + '#logsModal is still on the admin page: id="logsModal" in the markup', + 'showLogsModal() is still on the admin page: showLogsModal() defined', + ]); + assert.deepEqual(removalProblems({ ...row, Page: 'metadata' }, FIXTURE_COVERAGE, page), ['unknown page "metadata"']); +}); diff --git a/tools/vendor/legacy-coverage.mjs b/tools/vendor/legacy-coverage.mjs index 46a807c7..4da4a74f 100644 --- a/tools/vendor/legacy-coverage.mjs +++ b/tools/vendor/legacy-coverage.mjs @@ -42,25 +42,30 @@ function cells(line) { return out; } -/** The table's rows as objects keyed by COLUMNS, from the section that starts at HEADING. */ -export function parseCoverageTable(markdown) { +/** The rows, keyed by `columns`, of the first Markdown table under `heading` (`label` names it in errors). */ +function parseTable(markdown, heading, columns, label) { const lines = markdown.split('\n'); - const start = lines.findIndex(line => line.trim() === HEADING); - if (start === -1) throw new Error(`TESTING.md has no "${HEADING}" section`); + const start = lines.findIndex(line => line.trim() === heading); + if (start === -1) throw new Error(`TESTING.md has no "${heading}" section`); const header = lines.findIndex((line, index) => index > start && line.trim().startsWith('|')); if (header === -1 || lines.slice(start + 1, header).some(line => /^#{1,3} /.test(line))) { - throw new Error('The coverage section has no table'); + throw new Error(`The ${label} section has no table`); } - const columns = cells(lines[header]); - if (columns.join('|') !== COLUMNS.join('|')) throw new Error(`The coverage table's columns are ${columns.join(', ')}`); + const found = cells(lines[header]); + if (found.join('|') !== columns.join('|')) throw new Error(`The ${label} table's columns are ${found.join(', ')}`); const rows = []; for (let index = header + 2; index < lines.length && lines[index].trim().startsWith('|'); index += 1) { const values = cells(lines[index]); - rows.push({ line: index + 1, ...Object.fromEntries(COLUMNS.map((column, at) => [column, values[at] ?? ''])) }); + rows.push({ line: index + 1, ...Object.fromEntries(columns.map((column, at) => [column, values[at] ?? ''])) }); } return rows; } +/** The table's rows as objects keyed by COLUMNS, from the section that starts at HEADING. */ +export function parseCoverageTable(markdown) { + return parseTable(markdown, HEADING, COLUMNS, 'coverage'); +} + /** * A Native control cell: `` `<section>` · <role> · "<name>" ``, optionally ` · in "<row title>"` when the name is shared * by several rows of the section → {section, role, name, row}; `—` → null; else it throws. @@ -213,3 +218,130 @@ export function rowProblems(row, { root = ROOT, locate = file => testFile(file, export function readCoverageTable(root = ROOT) { return parseCoverageTable(readFileSync(join(root, 'TESTING.md'), 'utf8')); } + +// ── Legacy System UI removed by S13 (#330) ────────────────────────────────────────────────────────────────────────── +// TESTING.md "Legacy System UI removed by S13": one row per covered capability whose legacy UI S13 removed, with the +// element ids and methods that went. A row must cite a `covered` coverage row and copy its Test cell, and none of its +// ids or methods may still be on its page. +export const REMOVAL_HEADING = '### Legacy System UI removed by S13'; +export const REMOVAL_COLUMNS = Object.freeze(['Capability', 'Page', 'Removed ids', 'Removed methods', 'Native test']); +/** Each legacy page's markup and script. */ +export const PAGES = Object.freeze({ + admin: Object.freeze({ html: 'web/admin.html', script: 'web/js/admin.js' }), + gallery: Object.freeze({ html: 'web/gallery.html', script: 'web/js/gallery.js' }), +}); + +/** The removal table's rows, from the section that starts at REMOVAL_HEADING. */ +export function parseRemovalTable(markdown) { + return parseTable(markdown, REMOVAL_HEADING, REMOVAL_COLUMNS, 'removal'); +} + +/** The removal table read from TESTING.md. */ +export function readRemovalTable(root = ROOT) { + return parseRemovalTable(readFileSync(join(root, 'TESTING.md'), 'utf8')); +} + +/** A Removed ids or Removed methods cell → its entries without backticks (`—` → none). */ +export function listCell(cell) { + if (cell.trim() === NONE) return []; + return cell.split(',').map(entry => entry.trim().replace(/^`|`$/g, '')).filter(Boolean); +} + +/** The removed ids of a row (without `#`) and its malformed entries. */ +function removedIds(row) { + const entries = listCell(row['Removed ids']); + return { ids: entries.filter(entry => /^#[\w-]+$/.test(entry)).map(entry => entry.slice(1)), + bad: entries.filter(entry => !/^#[\w-]+$/.test(entry)) }; +} + +/** The removed methods of a row (without `()`) and its malformed entries. */ +function removedMethods(row) { + const entries = listCell(row['Removed methods']); + return { names: entries.filter(entry => /^[A-Za-z_$][\w$]*\(\)$/.test(entry)).map(entry => entry.slice(0, -2)), + bad: entries.filter(entry => !/^[A-Za-z_$][\w$]*\(\)$/.test(entry)) }; +} + +const escapeRegExp = text => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + +/** + * Where `id` is still on a page: `id="<id>"` in its markup or script; in its script a lookup, a `…Modal("<id>")` call + * (`showModal`, `hideModal`) or a modal-list entry. + */ +export function idUses(id, { html, script }) { + const name = escapeRegExp(id); + const code = stripJsComments(script); + const uses = []; + // `id=`, not `data-id=`: the attribute starts after a space or a tag's `<name`. + const attribute = new RegExp(`(?:^|\\s)id=["']${name}["']`, 'm'); + if (attribute.test(html)) uses.push(`id="${id}" in the markup`); + if (attribute.test(code)) uses.push(`id="${id}" in the script`); + if (new RegExp(`getElementById\\(\\s*["'\`]${name}["'\`]\\s*\\)`).test(code)) uses.push(`getElementById('${id}')`); + if (new RegExp(`querySelector(?:All)?\\(\\s*["'\`]#${name}\\b`).test(code)) uses.push(`querySelector('#${id}')`); + if (new RegExp(`\\w*Modal\\(\\s*["'\`]${name}["'\`]`).test(code)) uses.push(`a …Modal('${id}') call`); + // A string list of modal ids (the gallery's backdrop list): the entry next to another `…Modal` entry. + for (const list of code.matchAll(/\[[^\[\]]*\]/g)) { + const entry = new RegExp(`["']${name}["']`); + if (entry.test(list[0]) && /["'][\w-]*Modal["']/.test(list[0].replace(entry, ''))) { + uses.push(`'${id}' in a modal list`); + break; + } + } + return uses; +} + +/** + * Where method `name` is still on a page: a definition in its script, or a `.<name>(` call (`this.`, `window.admin.`) + * in its script or markup. + */ +export function methodUses(name, { html, script }) { + const method = escapeRegExp(name); + const code = stripJsComments(script); + const uses = []; + if (new RegExp(`(?:^|[\\s;}])(?:async\\s+)?${method}\\s*\\([^)]*\\)\\s*\\{`, 'm').test(code)) uses.push(`${name}() defined`); + if (new RegExp(`\\.${method}\\s*\\(`).test(code)) uses.push(`.${name}( called`); + if (new RegExp(`\\.${method}\\s*\\(`).test(html)) uses.push(`.${name}( called in the markup`); + return uses; +} + +/** + * Every problem of one removal row (empty: sound). `coverage` is the coverage table's rows; `readPage(page)` returns + * that page's `{html, script}`. + */ +export function removalProblems(row, coverage, readPage) { + const problems = []; + const cited = coverage.filter(item => item.Capability === row.Capability); + if (cited.length !== 1) problems.push(`cites no single coverage row "${row.Capability}"`); + else if (cited[0].Status !== 'covered') problems.push(`cites a ${cited[0].Status} capability`); + else if (cited[0].Test !== row['Native test']) problems.push(`its native test differs from the coverage row's "${cited[0].Test}"`); + if (!Object.hasOwn(PAGES, row.Page)) { + problems.push(`unknown page "${row.Page}"`); + return problems; + } + const { ids, bad: badIds } = removedIds(row); + const { names, bad: badNames } = removedMethods(row); + for (const entry of [...badIds, ...badNames]) problems.push(`malformed entry "${entry}"`); + const page = readPage(row.Page); + for (const id of ids) for (const use of idUses(id, page)) problems.push(`#${id} is still on the ${row.Page} page: ${use}`); + for (const name of names) for (const use of methodUses(name, page)) problems.push(`${name}() is still on the ${row.Page} page: ${use}`); + return problems; +} + +/** Ids and methods listed on more than one row of the same page. */ +export function duplicateRemovals(rows) { + const seen = new Map(); + const duplicates = []; + for (const row of rows) { + for (const entry of [...listCell(row['Removed ids']), ...listCell(row['Removed methods'])]) { + const key = `${row.Page} ${entry}`; + if (seen.has(key)) duplicates.push(`${key} is listed on rows ${seen.get(key)} and ${row.line}`); + else seen.set(key, row.line); + } + } + return duplicates; +} + +/** A page's markup and script from the repository. */ +export function readPage(page, root = ROOT) { + const files = PAGES[page]; + return { html: readFileSync(join(root, files.html), 'utf8'), script: readFileSync(join(root, files.script), 'utf8') }; +} diff --git a/web/admin.html b/web/admin.html index f80b3565..2f064a3c 100644 --- a/web/admin.html +++ b/web/admin.html @@ -168,10 +168,11 @@ class="px-2.5 py-1 bg-pm-surface border border-pm hover:bg-pm-hover text-pm text-xs font-medium rounded-pm-sm transition-colors"> Settings </button> - <button id="diagnosticsBtn" - class="px-2.5 py-1 bg-pm-surface border border-pm hover:bg-pm-hover text-pm text-xs font-medium rounded-pm-sm transition-colors"> + <a id="diagnosticsBtn" href="workspace#/system/diagnostics" target="_blank" rel="noopener" + title="Opens System › Diagnostics & logs in a new tab" + class="px-2.5 py-1 bg-pm-surface border border-pm hover:bg-pm-hover text-pm text-xs font-medium rounded-pm-sm transition-colors"> Diagnostics - </button> + </a> <button id="maintenanceBtn" class="px-2.5 py-1 bg-pm-surface border border-pm hover:bg-pm-hover text-pm text-xs font-medium rounded-pm-sm transition-colors"> Maintenance @@ -192,10 +193,11 @@ class="px-2.5 py-1 bg-pm-surface border border-pm hover:bg-pm-hover text-pm text-xs font-medium rounded-pm-sm transition-colors"> Auto Tag </button> - <button id="logsBtn" - class="px-2.5 py-1 bg-pm-surface border border-pm hover:bg-pm-hover text-pm text-xs font-medium rounded-pm-sm transition-colors"> + <a id="logsBtn" href="workspace#/system/diagnostics" target="_blank" rel="noopener" + title="Opens System › Diagnostics & logs in a new tab" + class="px-2.5 py-1 bg-pm-surface border border-pm hover:bg-pm-hover text-pm text-xs font-medium rounded-pm-sm transition-colors"> View Logs - </button> + </a> <button id="metadataBtn" class="px-2.5 py-1 bg-pm-surface border border-pm hover:bg-pm-hover text-pm text-xs font-medium rounded-pm-sm transition-colors"> Metadata @@ -641,37 +643,6 @@ </div> </div> - <!-- Diagnostics Modal --> - <div id="diagnosticsModal" class="fixed inset-0 bg-black/60 backdrop-blur-[2px] hidden items-center justify-center z-50"> - <div class="bg-pm-surface rounded-pm-lg max-w-4xl w-full max-h-[90vh] overflow-hidden mx-4 border border-pm shadow-pm"> - <div class="p-4 border-b border-pm"> - <div class="flex items-center justify-between"> - <h3 class="text-sm font-semibold text-pm">Gallery System Diagnostics</h3> - <button onclick="window.admin.closeDiagnostics()" class="p-1 rounded-pm-sm hover:bg-pm-hover transition-colors"> - <svg class="w-4 h-4 text-pm-secondary" fill="none" stroke="currentColor" viewBox="0 0 24 24"> - <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M6 18L18 6M6 6l12 12"></path> - </svg> - </button> - </div> - </div> - <div class="p-4 overflow-y-auto max-h-[calc(90vh-120px)]"> - <div class="flex items-center justify-between mb-3"> - <button id="runDiagnosticsBtn" class="px-2.5 py-1.5 bg-pm-accent hover:bg-pm-accent-hover text-pm-accent-fg text-xs font-medium rounded-pm-sm transition-colors"> - Run Diagnostics - </button> - <button id="testImageLinkBtn" class="px-2.5 py-1.5 bg-pm-surface border border-pm hover:bg-pm-hover text-pm text-xs font-medium rounded-pm-sm transition-colors"> - Test Image Link - </button> - </div> - <div id="diagnosticsContent" class="space-y-3"> - <div class="text-center py-6 text-pm-secondary text-xs"> - Click "Run Diagnostics" to check the gallery system status - </div> - </div> - </div> - </div> - </div> - <!-- Maintenance Modal --> <div id="maintenanceModal" class="fixed inset-0 bg-black/60 backdrop-blur-[2px] hidden items-center justify-center z-50"> <div class="bg-pm-surface rounded-pm-lg max-w-4xl w-full max-h-[90vh] overflow-hidden mx-4 border border-pm shadow-pm"> @@ -864,130 +835,6 @@ <span class="inline-block w-1.5 h-1.5 rounded-full bg-pm-accent mr-1.5 align-middle animate-pulse"></span><span id="scanPillText" class="align-middle">Scanning...</span> </button> - <!-- Logs Modal --> - <div id="logsModal" class="fixed inset-0 bg-black/60 backdrop-blur-[2px] hidden items-center justify-center z-50"> - <div class="bg-pm-surface rounded-pm-lg max-w-6xl w-full max-h-[90vh] overflow-hidden mx-4 border border-pm shadow-pm"> - <div class="p-4 border-b border-pm"> - <div class="flex items-center justify-between"> - <h3 class="text-sm font-semibold text-pm">System Logs</h3> - <button onclick="window.admin.closeLogs()" class="p-1 rounded-pm-sm hover:bg-pm-hover transition-colors"> - <svg class="w-4 h-4 text-pm-secondary" fill="none" stroke="currentColor" viewBox="0 0 24 24"> - <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M6 18L18 6M6 6l12 12"></path> - </svg> - </button> - </div> - </div> - - <!-- Log Controls --> - <div class="p-4 border-b border-pm"> - <div class="grid grid-cols-1 md:grid-cols-5 gap-3 mb-3"> - <div> - <label class="block text-xs font-medium text-pm-secondary mb-1">Log Level</label> - <select id="logLevel" class="w-full px-2.5 py-1.5 bg-pm-input border border-pm rounded-pm-sm text-pm text-[13px] focus:outline-none"> - <option value="">All Levels</option> - <option value="DEBUG">Debug</option> - <option value="INFO">Info</option> - <option value="WARNING">Warning</option> - <option value="ERROR">Error</option> - <option value="CRITICAL">Critical</option> - </select> - </div> - <div> - <label class="block text-xs font-medium text-pm-secondary mb-1">Limit</label> - <select id="logLimit" class="w-full px-2.5 py-1.5 bg-pm-input border border-pm rounded-pm-sm text-pm text-[13px] focus:outline-none"> - <option value="50">Last 50</option> - <option value="100" selected>Last 100</option> - <option value="200">Last 200</option> - <option value="500">Last 500</option> - </select> - </div> - <div class="flex items-end"> - <button id="refreshLogsBtn" class="w-full px-2.5 py-1.5 bg-pm-accent hover:bg-pm-accent-hover text-pm-accent-fg text-xs font-medium rounded-pm-sm transition-colors"> - Refresh - </button> - </div> - <div class="flex items-end"> - <button id="downloadLogsBtn" class="w-full px-2.5 py-1.5 bg-pm-surface border border-pm hover:bg-pm-hover text-pm text-xs font-medium rounded-pm-sm transition-colors"> - Download - </button> - </div> - <div class="flex items-end"> - <button id="clearLogsBtn" class="w-full px-2.5 py-1.5 text-pm-error bg-transparent hover:bg-pm-error-tint border border-pm text-xs font-medium rounded-pm-sm transition-colors"> - Clear Logs - </button> - </div> - </div> - - <!-- Log Stats --> - <div class="grid grid-cols-2 md:grid-cols-5 gap-3 mb-3"> - <div class="bg-pm-input rounded-pm-md p-2"> - <div class="text-[11px] text-pm-secondary">Total Logs</div> - <div class="text-sm font-semibold text-pm-accent" id="logStatsTotal">-</div> - </div> - <div class="bg-pm-input rounded-pm-md p-2"> - <div class="text-[11px] text-pm-secondary">Errors</div> - <div class="text-sm font-semibold text-pm-error" id="logStatsErrors">-</div> - </div> - <div class="bg-pm-input rounded-pm-md p-2"> - <div class="text-[11px] text-pm-secondary">Warnings</div> - <div class="text-sm font-semibold text-pm-warning" id="logStatsWarnings">-</div> - </div> - <div class="bg-pm-input rounded-pm-md p-2"> - <div class="text-[11px] text-pm-secondary">Current Level</div> - <div class="text-sm font-semibold text-pm-success" id="logStatsLevel">-</div> - </div> - <div class="bg-pm-input rounded-pm-md p-2"> - <div class="text-[11px] text-pm-secondary">Log Size</div> - <div class="text-sm font-semibold text-pm" id="logStatsSize">-</div> - </div> - </div> - - <!-- Log Settings --> - <div class="grid grid-cols-1 md:grid-cols-3 gap-3"> - <div> - <label class="block text-xs font-medium text-pm-secondary mb-1">Set Log Level</label> - <select id="setLogLevel" class="w-full px-2.5 py-1.5 bg-pm-input border border-pm rounded-pm-sm text-pm text-[13px] focus:outline-none"> - <option value="DEBUG">Debug</option> - <option value="INFO" selected>Info</option> - <option value="WARNING">Warning</option> - <option value="ERROR">Error</option> - <option value="CRITICAL">Critical</option> - </select> - </div> - <div class="flex items-center"> - <label class="flex items-center gap-1.5 text-pm"> - <input type="checkbox" id="consoleLogging" checked class="w-3.5 h-3.5 accent-[var(--pm-accent)]"> - <span class="text-xs">Console Logging</span> - </label> - </div> - <div class="flex items-end"> - <button id="updateLogConfigBtn" class="w-full px-2.5 py-1.5 bg-pm-accent hover:bg-pm-accent-hover text-pm-accent-fg text-xs font-medium rounded-pm-sm transition-colors"> - Update Config - </button> - </div> - </div> - </div> - - <!-- Log Viewer --> - <div class="flex-1 overflow-hidden"> - <div class="h-96 overflow-y-auto bg-pm-primary font-mono text-xs"> - <div id="logsContainer" class="p-3 space-y-0.5"> - <div class="text-center py-6 text-pm-muted text-xs"> - Click "Refresh" to load logs - </div> - </div> - </div> - </div> - - <!-- Log Files --> - <div class="p-4 border-t border-pm"> - <h4 class="text-xs font-semibold text-pm-secondary uppercase tracking-wide mb-3">Log Files</h4> - <div id="logFilesList" class="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-2"> - </div> - </div> - </div> - </div> - <!-- Add Prompt Modal --> <div id="addPromptModal" class="fixed inset-0 bg-black/60 backdrop-blur-[2px] hidden items-center justify-center z-50"> <div class="bg-pm-surface rounded-pm-lg p-4 max-w-2xl w-full mx-4 border border-pm max-h-[90vh] overflow-y-auto shadow-pm"> diff --git a/web/js/admin.js b/web/js/admin.js index e48c0985..405251ed 100644 --- a/web/js/admin.js +++ b/web/js/admin.js @@ -47,7 +47,6 @@ edit: ({ promptId }) => this.editPrompt(promptId), delete: ({ promptId }) => this.deletePrompt(promptId), "open-film": ({ promptId, index }) => this.openFilmStripViewer(promptId, index), - "download-log": ({ filename }) => this.downloadLogFile(filename), "remove-review-tag": ({ index }) => this.removeReviewTag(index), "pick-tag-suggestion": ({ tag }) => this.pickTagSuggestion(tag), }; @@ -128,14 +127,12 @@ document.getElementById("exportBtn").addEventListener("click", () => this.exportPrompts()); document.getElementById("settingsBtn").addEventListener("click", () => this.showSettingsModal()); document.getElementById("statsBtn").addEventListener("click", () => this.showStats()); - document.getElementById("diagnosticsBtn").addEventListener("click", () => this.showDiagnosticsModal()); document.getElementById("maintenanceBtn").addEventListener("click", () => this.showMaintenanceModal()); document.getElementById("backupBtn").addEventListener("click", () => this.backupDatabase()); document.getElementById("restoreBtn").addEventListener("click", () => this.showRestoreModal()); document.getElementById("scanBtn").addEventListener("click", () => this.showScanModal()); document.getElementById("autoTagBtn").addEventListener("click", () => this.showAutoTagModal()); document.getElementById("addPromptBtn").addEventListener("click", () => this.showAddPromptModal()); - document.getElementById("logsBtn").addEventListener("click", () => this.showLogsModal()); document.getElementById("metadataBtn").addEventListener("click", () => this.openMetadataViewer()); document.getElementById("galleryBtn").addEventListener("click", () => this.openGallery()); @@ -215,10 +212,6 @@ document.getElementById("confirmRestore").addEventListener("click", () => this.confirmRestore()); document.getElementById("cancelRestore").addEventListener("click", () => this.hideModal("restoreModal")); - // Diagnostics modal - document.getElementById("runDiagnosticsBtn").addEventListener("click", () => this.runDiagnostics()); - document.getElementById("testImageLinkBtn").addEventListener("click", () => this.testImageLink()); - // Maintenance modal document.getElementById("runMaintenanceBtn").addEventListener("click", () => this.runMaintenance()); document.getElementById("selectAllMaintenanceBtn").addEventListener("click", () => this.selectAllMaintenance()); @@ -230,12 +223,6 @@ document.getElementById("scanPill").addEventListener("click", () => this.showScanModal()); document.getElementById("quickBackupBtn").addEventListener("click", () => this.quickBackup()); - // Logs modal - document.getElementById("refreshLogsBtn").addEventListener("click", () => this.refreshLogs()); - document.getElementById("downloadLogsBtn").addEventListener("click", () => this.downloadLogs()); - document.getElementById("clearLogsBtn").addEventListener("click", () => this.clearLogs()); - document.getElementById("updateLogConfigBtn").addEventListener("click", () => this.updateLogConfig()); - // Auto Tag modals document.getElementById("startAutoTagBtn").addEventListener("click", () => this.startAutoTag()); document.getElementById("startReviewBtn").addEventListener("click", () => this.startReview()); @@ -1801,15 +1788,6 @@ return `${size.toFixed(1)} ${units[unitIndex]}`; } - // Diagnostics functionality - showDiagnosticsModal() { - this.showModal("diagnosticsModal"); - } - - closeDiagnostics() { - this.hideModal("diagnosticsModal"); - } - // Maintenance functionality showMaintenanceModal() { this.showModal("maintenanceModal"); @@ -1981,160 +1959,6 @@ resultsContainer.innerHTML = html; } - async runDiagnostics() { - const content = document.getElementById("diagnosticsContent"); - content.innerHTML = ` - <div class="text-center py-4"> - <div class="w-12 h-12 border-4 border-orange-500/30 border-t-orange-500 rounded-full animate-spin mx-auto mb-4"></div> - <p class="text-pm-secondary">Running diagnostics...</p> - </div> - `; - - try { - const response = await ApiClient.fetchUrl('/prompt_manager/diagnostics'); - if (response.ok) { - const data = await response.json(); - if (data.success) { - this.renderDiagnostics(data.diagnostics); - } else { - throw new Error(data.error || 'Diagnostics failed'); - } - } else { - throw new Error(`HTTP ${response.status}: ${response.statusText}`); - } - } catch (error) { - console.error('Diagnostics error:', error); - content.innerHTML = ` - <div class="bg-pm-error-tint border border-pm-error rounded-pm-sm p-4"> - <h4 class="text-pm-error font-medium mb-2">❌ Diagnostics Failed</h4> - <p class="text-pm-secondary">${escapeHtml(error.message)}</p> - </div> - `; - } - } - - renderDiagnostics(diagnostics) { - const content = document.getElementById("diagnosticsContent"); - - let html = '<div class="space-y-4">'; - - // Summary - html += '<div class="bg-pm-surface rounded-pm-sm p-4">'; - html += '<h4 class="text-pm font-medium mb-3">📋 Diagnostic Summary</h4>'; - html += '<div class="grid grid-cols-1 md:grid-cols-2 gap-3">'; - - for (const [category, result] of Object.entries(diagnostics)) { - const status = result.status === 'ok' ? '✅ PASS' : (result.status === 'warning' ? '⚠️ WARNING' : '❌ FAIL'); - const statusColor = result.status === 'ok' ? 'text-pm-success' : (result.status === 'warning' ? 'text-pm-warning' : 'text-pm-error'); - - html += ` - <div class="flex justify-between items-center p-2 bg-pm-input rounded"> - <span class="text-pm-secondary capitalize">${escapeHtml(category)}</span> - <span class="${statusColor} font-mono text-sm">${status}</span> - </div> - `; - } - - html += '</div></div>'; - - // Detailed results - for (const [category, result] of Object.entries(diagnostics)) { - const bgColor = result.status === 'ok' ? 'bg-pm-success-tint border-pm-success' : - (result.status === 'warning' ? 'bg-pm-warning/20 border-pm-warning' : 'bg-pm-error-tint border-pm-error'); - - html += `<div class="${bgColor} border rounded-pm-sm p-4">`; - html += `<h4 class="text-pm font-medium mb-2 capitalize">${escapeHtml(category)}</h4>`; - - if (result.message) { - html += `<p class="text-pm-secondary mb-2">${escapeHtml(result.message)}</p>`; - } - - // Show specific details based on category - if (category === 'database' && result.status === 'ok') { - html += `<div class="text-sm text-pm-secondary">`; - html += `<p>Prompts: ${escapeHtml(result.prompt_count || 0)}</p>`; - html += `<p>Images table: ${result.has_images_table ? 'Yes' : 'No'}</p>`; - html += `</div>`; - } - - if (category === 'images_table' && result.status === 'ok') { - html += `<div class="text-sm text-pm-secondary">`; - html += `<p>Images: ${escapeHtml(result.image_count || 0)}</p>`; - if (result.recent_images && result.recent_images.length > 0) { - html += `<p>Recent images:</p>`; - html += `<ul class="ml-4 list-disc">`; - result.recent_images.slice(0, 3).forEach(img => { - html += `<li>${escapeHtml(img.filename)} → Prompt ${escapeHtml(img.prompt_id)}</li>`; - }); - html += `</ul>`; - } - html += `</div>`; - } - - if (category === 'comfyui_output' && result.output_dirs) { - html += `<div class="text-sm text-pm-secondary">`; - html += `<p>Output directories found:</p>`; - html += `<ul class="ml-4 list-disc">`; - result.output_dirs.forEach(dir => { - html += `<li>${escapeHtml(dir)}</li>`; - }); - html += `</ul>`; - html += `</div>`; - } - - if (category === 'dependencies' && result.dependencies) { - html += `<div class="text-sm text-pm-secondary">`; - for (const [dep, available] of Object.entries(result.dependencies)) { - const status = available ? '✅' : '❌'; - html += `<p>${status} ${escapeHtml(dep)}</p>`; - } - html += `</div>`; - } - - html += '</div>'; - } - - html += '</div>'; - content.innerHTML = html; - } - - async testImageLink() { - if (this.prompts.length === 0) { - this.showNotification('No prompts available for testing', 'warning'); - return; - } - - // Use the first prompt for testing - const testPrompt = this.prompts[0]; - - try { - const response = await ApiClient.fetchUrl('/prompt_manager/diagnostics/test-link', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - prompt_id: testPrompt.id, - image_path: '/test/fake/image.png' - }) - }); - - if (response.ok) { - const data = await response.json(); - if (data.success) { - this.showNotification('✅ Test image link created successfully!', 'success'); - // Refresh the gallery to show the test image - setTimeout(() => this.refreshList(), 1000); - } else { - throw new Error(data.result?.message || 'Test failed'); - } - } else { - throw new Error(`HTTP ${response.status}: ${response.statusText}`); - } - } catch (error) { - console.error('Test link error:', error); - this.showNotification(`❌ Test failed: ${error.message}`, 'error'); - } - } - // Database backup and restore functionality async backupDatabase() { try { @@ -2497,273 +2321,6 @@ this.showNotification(`Scan failed: ${state.message}`, "error"); } - // Logs functionality - showLogsModal() { - this.showModal("logsModal"); - this.loadLogStats(); - this.loadLogFiles(); - this.loadLogConfig(); - } - - closeLogs() { - this.hideModal("logsModal"); - } - - async loadLogStats() { - try { - const response = await ApiClient.fetchUrl('/prompt_manager/logs/stats'); - const data = await response.json(); - - if (data.success) { - const stats = data.stats; - document.getElementById("logStatsTotal").textContent = stats.buffer_count || 0; - document.getElementById("logStatsErrors").textContent = stats.level_counts?.ERROR || 0; - document.getElementById("logStatsWarnings").textContent = stats.level_counts?.WARNING || 0; - document.getElementById("logStatsLevel").textContent = stats.current_level || "INFO"; - document.getElementById("logStatsSize").textContent = this.formatBytes(stats.total_log_size || 0); - } - } catch (error) { - console.error("Failed to load log stats:", error); - } - } - - async loadLogConfig() { - try { - const response = await ApiClient.fetchUrl('/prompt_manager/logs/config'); - const data = await response.json(); - - if (data.success) { - const config = data.config; - document.getElementById("setLogLevel").value = config.level || "INFO"; - document.getElementById("consoleLogging").checked = config.console_logging !== false; - } - } catch (error) { - console.error("Failed to load log config:", error); - } - } - - async loadLogFiles() { - try { - const response = await ApiClient.fetchUrl('/prompt_manager/logs/files'); - const data = await response.json(); - - if (data.success) { - const container = document.getElementById("logFilesList"); - - if (data.files.length === 0) { - container.innerHTML = '<div class="col-span-full text-center text-pm-muted">No log files found</div>'; - return; - } - - container.innerHTML = data.files.map(file => ` - <div class="bg-pm-surface rounded-pm-sm p-3"> - <div class="flex items-center justify-between mb-2"> - <span class="text-sm font-medium text-pm">${escapeHtml(file.filename)}</span> - ${file.is_main ? '<span class="bg-pm-accent text-xs px-2 py-1 rounded">Active</span>' : ''} - </div> - <div class="text-xs text-pm-secondary mb-2"> - Size: ${this.formatBytes(file.size)} | Modified: ${new Date(file.modified).toLocaleString()} - </div> - <button data-action="download-log" data-filename="${escapeHtml(file.filename)}" - class="w-full px-3 py-1 bg-pm-success hover:bg-pm-success text-pm text-xs rounded transition-colors"> - 📥 Download - </button> - </div> - `).join(''); - } - } catch (error) { - console.error("Failed to load log files:", error); - } - } - - async refreshLogs() { - const level = document.getElementById("logLevel").value; - const limit = parseInt(document.getElementById("logLimit").value); - - try { - const url = new URL(ApiClient.resolveUrl('/prompt_manager/logs'), window.location.origin); - if (level) url.searchParams.set('level', level); - url.searchParams.set('limit', limit.toString()); - - const response = await ApiClient.fetchUrl(url); - const data = await response.json(); - - if (data.success) { - this.displayLogs(data.logs); - this.showNotification(`Loaded ${data.logs.length} log entries`, "success"); - } else { - this.showNotification(`Failed to load logs: ${data.error}`, "error"); - } - } catch (error) { - console.error("Failed to refresh logs:", error); - this.showNotification("Failed to refresh logs", "error"); - } - } - - displayLogs(logs) { - const container = document.getElementById("logsContainer"); - - if (logs.length === 0) { - container.innerHTML = '<div class="text-center py-4 text-pm-muted">No logs found</div>'; - return; - } - - container.innerHTML = logs.map(log => { - const levelColors = { - DEBUG: 'text-pm-secondary', - INFO: 'text-pm-accent', - WARNING: 'text-pm-warning', - ERROR: 'text-pm-error', - CRITICAL: 'text-pm-error' - }; - - const levelColor = levelColors[log.level] || 'text-pm-secondary'; - const timestamp = new Date(log.timestamp).toLocaleString(); - - return ` - <div class="border-l-2 border-pm pl-3 py-1 hover:bg-pm-surface transition-colors"> - <div class="flex items-start space-x-2 text-sm"> - <span class="text-pm-muted text-xs font-mono w-24 flex-shrink-0">${escapeHtml(timestamp.split(' ')[1])}</span> - <span class="${levelColor} font-semibold w-16 flex-shrink-0">${escapeHtml(log.level)}</span> - <span class="text-pm-accent text-xs w-32 flex-shrink-0">${escapeHtml(log.logger)}</span> - <span class="text-pm-secondary flex-1">${escapeHtml(log.message)}</span> - </div> - <div class="text-xs text-pm-muted ml-44"> - ${escapeHtml(log.filename)}:${escapeHtml(log.lineno)} - </div> - </div> - `; - }).join(''); - - // Auto-scroll to bottom - container.scrollTop = container.scrollHeight; - } - - async downloadLogs() { - const level = document.getElementById("logLevel").value; - const limit = parseInt(document.getElementById("logLimit").value); - - try { - const url = new URL(ApiClient.resolveUrl('/prompt_manager/logs'), window.location.origin); - if (level) url.searchParams.set('level', level); - url.searchParams.set('limit', limit.toString()); - - const response = await ApiClient.fetchUrl(url); - const data = await response.json(); - - if (data.success) { - const logText = data.logs.map(log => - `${log.timestamp} [${log.level}] ${log.logger}: ${log.message} (${log.filename}:${log.lineno})` - ).join('\n'); - - const blob = new Blob([logText], { type: 'text/plain' }); - const url2 = URL.createObjectURL(blob); - const a = document.createElement('a'); - a.href = url2; - a.download = `prompt_manager_logs_${new Date().toISOString().slice(0, 19).replace(/:/g, '-')}.txt`; - document.body.appendChild(a); - a.click(); - document.body.removeChild(a); - URL.revokeObjectURL(url2); - - this.showNotification("Logs downloaded successfully", "success"); - } else { - this.showNotification(`Failed to download logs: ${data.error}`, "error"); - } - } catch (error) { - console.error("Failed to download logs:", error); - this.showNotification("Failed to download logs", "error"); - } - } - - async downloadLogFile(filename) { - try { - const response = await ApiClient.fetchUrl(`/prompt_manager/logs/download/${encodeURIComponent(filename)}`); - - if (response.ok) { - const blob = await response.blob(); - const url = URL.createObjectURL(blob); - const a = document.createElement('a'); - a.href = url; - a.download = filename; - document.body.appendChild(a); - a.click(); - document.body.removeChild(a); - URL.revokeObjectURL(url); - - this.showNotification(`Downloaded ${filename}`, "success"); - } else { - this.showNotification(`Failed to download ${filename}`, "error"); - } - } catch (error) { - console.error("Failed to download log file:", error); - this.showNotification("Failed to download log file", "error"); - } - } - - async clearLogs() { - if (!confirm("Are you sure you want to clear all log files? This cannot be undone.")) { - return; - } - - try { - const response = await ApiClient.fetchUrl('/prompt_manager/logs/truncate', { - method: 'POST' - }); - const data = await response.json(); - - if (data.success) { - this.showNotification(data.message, "success"); - this.refreshLogs(); - this.loadLogStats(); - this.loadLogFiles(); - } else { - this.showNotification(`Failed to clear logs: ${data.error}`, "error"); - } - } catch (error) { - console.error("Failed to clear logs:", error); - this.showNotification("Failed to clear logs", "error"); - } - } - - async updateLogConfig() { - const level = document.getElementById("setLogLevel").value; - const consoleLogging = document.getElementById("consoleLogging").checked; - - try { - const response = await ApiClient.fetchUrl('/prompt_manager/logs/config', { - method: 'POST', - headers: { - 'Content-Type': 'application/json' - }, - body: JSON.stringify({ - level: level, - console_logging: consoleLogging - }) - }); - - const data = await response.json(); - - if (data.success) { - this.showNotification("Log configuration updated", "success"); - this.loadLogStats(); - } else { - this.showNotification(`Failed to update config: ${data.error}`, "error"); - } - } catch (error) { - console.error("Failed to update log config:", error); - this.showNotification("Failed to update log configuration", "error"); - } - } - - formatBytes(bytes) { - if (bytes === 0) return '0 B'; - const k = 1024; - const sizes = ['B', 'KB', 'MB', 'GB']; - const i = Math.floor(Math.log(bytes) / Math.log(k)); - return parseFloat((bytes / Math.pow(k, i)).toFixed(1)) + ' ' + sizes[i]; - } - // Pagination methods updatePaginationControls() { const { currentPage, totalPages, total, limit } = this.pagination; From d0339128e4a1b9e01ed6e8b7755c9acf24890d5f Mon Sep 17 00:00:00 2001 From: Vito <vito.sansevero@gmail.com> Date: Wed, 28 Oct 2026 05:18:00 +0000 Subject: [PATCH 07/11] test: isolate API fixture databases from other suites; CI lint pins, failing bandit hook, npm cache (#346, #352, #409) (#411) * ci: pin black and flake8, let the bandit hook fail, cache npm in the JS job (#352) The lint job installs black==26.10.1 and flake8==7.4.1 from requirements-dev.txt (the versions CI resolved on main today), so a new release cannot turn CI red with no code change. The local bandit hook ran 'bash -c ... "$@"; true': it could never fail, and bash bound the first staged file to $0 so it was never scanned. It now runs 'bandit -lll -q' directly, failing on high severity exactly like code-quality.yml. The frontend modules job's setup-node caches npm keyed on tools/vendor/package-lock.json. * test(history): wait on the writer's state, not short wall-clock pauses (#352) Four 0.08 s pauses before asserting that the worker had not probed yet now wait until the idle worker has run a whole loop pass at the current manual clock value (three clock reads), so a slow runner cannot end the pause before the worker decides. A mutation that probes 20 ms early now fails two tests deterministically. The real-clock bounded close no longer asserts close(0.02) returned in under 0.2 s wall time; it asserts the in-flight write is still blocked and the worker alive, like the output writer test. The manual-clock bound stays exact. The last 0.1 s non-blocking submit check uses the 1 s NON_BLOCKING_CEILING from #351. * test(api): give each API fixture database its own temp folder (#346) Every PromptManagerAPI sweeps its database's folder at startup for the stages, uploads and safety-backup copies an interrupted restore left. The API test fixtures created their database directly in the system temp folder, so any other test process starting an API there (another worktree's suite, a pre-push hook, an agent's targeted run) deleted this process's in-flight restore files. The restore, epoch and retirement tests then failed with 'staged restore changed', 'unable to open database file' or 503 instead of 409. Busy machines fail because they are running other suites, not because they are slow: the same CPU load with a private TMPDIR passes. private_temp_db() puts the fixture database in a new folder removed at cleanup (with any safety backups a restore left beside it). A regression test sweeps a stand-in shared folder in the middle of a multipart restore, as another suite's startup would. * test(history): count worker parks, hold saturation blocks until released (#352) wait_for_worker_pass now counts the idle worker's parks on its wake event (two parks from now = one whole pass begun after the call) instead of assuming a fixed number of clock reads per pass, which a future clock read would have made vacuous. The real-clock bounded close keeps a loose upper bound (0.02 s + NON_BLOCKING_CEILING). The two saturation tests (history and output writer) blocked the worker with release.wait(3) while they queued 1023 more items. On a loaded runner the fill took longer than 3 s, the worker unblocked on its own and drained part of the queue (pending 868..1017 != 1024), and cleanup then reported a leaked worker. The block now holds until the test releases it; 60 s is only a hang guard, and cleanup always sets the release. Under one core with 8 busy loops: 8/8 runs failed before, 6/6 pass after. * test(maintenance): wait on duplicate jobs with a disk-bound hang guard (#409) On Windows py3.13 CI (PR #405) a one-file duplicate cleanup missed its 10 s WAIT. The test already waits on the job's own task; the 10 s bound acted as a deadline on disk-bound work. In the same 50 s window the following tests' setups took up to 9 s against 0.1-0.6 s around it while the test bodies stayed fast, which is consistent with a process-wide disk I/O stall on the runner rather than a lease, handle, executor or job stall (the log has no thread dump to show more). tests/disk_waits.py holds the one disk-bound hang guard (90 s per wait, under the 120 s per-test watchdog), shared with RESTORE_SECONDS (#382). Waits on find and cleanup job tasks in the duplicate-files tests use it; gates and server-state polls keep WAIT. The keeper-validation test turns the safety backup off, since it does not test backups. --- .github/workflows/test.yml | 7 +++- .pre-commit-config.yaml | 4 +- requirements-dev.txt | 4 +- tests/disk_waits.py | 16 ++++++++ tests/private_temp.py | 24 +++++++++++ tests/test_api.py | 5 +-- tests/test_api_autotag.py | 4 +- tests/test_api_duplicate_files.py | 40 ++++++++++-------- tests/test_api_duplicate_safety.py | 6 +-- tests/test_api_history.py | 44 ++++++++++++++++++++ tests/test_api_images.py | 4 +- tests/test_api_lora.py | 4 +- tests/test_api_prompts.py | 5 +-- tests/test_execution_history_writer.py | 56 ++++++++++++++++++++++---- tests/test_execution_output_writer.py | 15 +++++-- tests/vendor/ci-workflow.test.mjs | 29 +++++++++++++ 16 files changed, 220 insertions(+), 47 deletions(-) create mode 100644 tests/disk_waits.py create mode 100644 tests/private_temp.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bc7321fa..43bf3d99 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -221,7 +221,10 @@ jobs: - uses: actions/setup-python@v7 with: python-version: "3.12" - - run: pip install black flake8 + cache: pip + cache-dependency-path: requirements-dev.txt + # The exact black and flake8 pins from requirements-dev.txt, so a new release cannot turn CI red. + - run: pip install $(grep -E '^(black|flake8)==' requirements-dev.txt) - run: black --check . - run: flake8 . @@ -235,6 +238,8 @@ jobs: - uses: actions/setup-node@v7 with: node-version: "24" + cache: npm + cache-dependency-path: tools/vendor/package-lock.json - name: Check shipped extension and page modules parse run: | diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ca3295df..9106e4fc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,9 +13,11 @@ repos: language: system types: [python] + # Fails at the severity CI enforces (code-quality.yml: -lll, high only), never stricter. + # pre-commit appends the staged files, so no shell wrapper can drop the first one. - id: bandit name: bandit - entry: bash -c 'bandit "$@" -ll -q 2>&1; true' + entry: bandit -lll -q language: system types: [python] exclude: ^(tests/|reference/) diff --git a/requirements-dev.txt b/requirements-dev.txt index 43e2130a..6db215e2 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,8 +8,8 @@ coverage>=7.16.2 numpy>=2.2.6 # WD14 engine tests # Code quality -black>=26.10.1 -flake8>=7.4.1 +black==26.10.1 +flake8==7.4.1 # Security scanning bandit[toml]>=1.9.4 diff --git a/tests/disk_waits.py b/tests/disk_waits.py new file mode 100644 index 00000000..26a1f879 --- /dev/null +++ b/tests/disk_waits.py @@ -0,0 +1,16 @@ +"""Hang guards for waits on disk-bound work (tests only). + +A test that waits for a job or handoff whose work is file and SQLite I/O +(a safety backup with fsync, a restore, hashing and unlinking files) waits +on that state; the ceiling below only ends a real hang. Hosted Windows +runners stall disk I/O for tens of seconds at a time: a restore took over +10 s (merge queue, #382), and on PR #405 every test setup in a 50 s window +took up to 9 s against 0.1-0.6 s around it while the test bodies stayed +fast, so a one-file duplicate cleanup missed a 10 s bound (#409). + +The ceiling is per wait and stays under the suite's 120 s per-test hang +watchdog. A test with several such waits in a row can reach the watchdog +first; either way a real hang fails, the watchdog with every thread's stack. +""" + +DISK_BOUND_SECONDS = 90 diff --git a/tests/private_temp.py b/tests/private_temp.py new file mode 100644 index 00000000..dee79ce9 --- /dev/null +++ b/tests/private_temp.py @@ -0,0 +1,24 @@ +"""A temp SQLite database in a folder no other test process shares (#346). + +``PromptManagerAPI`` sweeps its database's folder at startup for the stages, +uploads and safety-backup copies an interrupted restore or backup left. A +fixture database directly in the system temp folder made every other test +process that started an API (another worktree's suite, a pre-push hook) delete +this process's in-flight restore files, and left safety backups behind there. +""" + +import tempfile + + +def private_temp_db(case): + """Return a closed ``NamedTemporaryFile`` handle for an empty ``.db`` file. + + The file lives in a new folder that ``case`` removes at cleanup, after its + own teardown has closed and unlinked the database (cleanups run after + ``tearDown``), together with anything a restore or backup left next to it. + """ + folder = tempfile.TemporaryDirectory(ignore_cleanup_errors=True) + case.addCleanup(folder.cleanup) + handle = tempfile.NamedTemporaryFile(delete=False, suffix=".db", dir=folder.name) + handle.close() + return handle diff --git a/tests/test_api.py b/tests/test_api.py index 051881a8..efcb433a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -8,7 +8,6 @@ without requiring a full ComfyUI server. import json import os import sys -import tempfile import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -18,6 +17,7 @@ from aiohttp.test_utils import AioHTTPTestCase from database.operations import PromptDatabase from py.api import PromptManagerAPI +from tests.private_temp import private_temp_db from utils.hashing import generate_prompt_hash @@ -25,8 +25,7 @@ class APITestCase(AioHTTPTestCase): """Base class that stands up a test aiohttp app with PromptManager routes.""" async def get_application(self): - self._temp_db = tempfile.NamedTemporaryFile(delete=False, suffix=".db") - self._temp_db.close() + self._temp_db = private_temp_db(self) app = web.Application() routes = web.RouteTableDef() diff --git a/tests/test_api_autotag.py b/tests/test_api_autotag.py index e2d2dffc..cfe3f47a 100644 --- a/tests/test_api_autotag.py +++ b/tests/test_api_autotag.py @@ -35,6 +35,7 @@ from aiohttp.test_utils import AioHTTPTestCase from database.operations import PromptDatabase from py.api import PromptManagerAPI +from tests.private_temp import private_temp_db from py.api.autotag_routes import MODEL_BUSY_ERROR from py.api.jobs import JobBusy from utils.hashing import generate_prompt_hash @@ -143,8 +144,7 @@ class AutotagAPITestCase(AioHTTPTestCase): fake engine.""" async def get_application(self): - self._temp_db = tempfile.NamedTemporaryFile(delete=False, suffix=".db") - self._temp_db.close() + self._temp_db = private_temp_db(self) self.tmp_root = Path(tempfile.mkdtemp()) self.output_dir = self.tmp_root / "output" self.output_dir.mkdir() diff --git a/tests/test_api_duplicate_files.py b/tests/test_api_duplicate_files.py index 91bd0ea3..9c78ba4a 100644 --- a/tests/test_api_duplicate_files.py +++ b/tests/test_api_duplicate_files.py @@ -18,15 +18,19 @@ from pathlib import Path from unittest.mock import patch try: + from tests.disk_waits import DISK_BOUND_SECONDS from tests.open_handles import open_handles_under from tests.test_api_maintenance import ApplyTestCase, _digest except ImportError: # discovered with tests/ as the top-level directory + from disk_waits import DISK_BOUND_SECONDS from open_handles import open_handles_under from test_api_maintenance import ApplyTestCase, _digest from py.api import duplicate_files # noqa: E402 from py.config import GalleryConfig, PromptManagerConfig # noqa: E402 +# Gates and server-state polls. Waits on a find or cleanup job's task use +# DISK_BOUND_SECONDS: those jobs hash, back up and unlink files (#409). WAIT = 10 FIND = "/prompt_manager/jobs/duplicates" GROUPS = "/prompt_manager/maintenance/duplicates/" @@ -83,7 +87,7 @@ class DuplicateFilesTestCase(ApplyTestCase): self.assertEqual(response.status, 202, text) self.assertNotIn(self.tmpdir, text) job = self.api._jobs.get(json.loads(text)["job"]["id"]) - await asyncio.wait_for(asyncio.shield(job.task), WAIT) + await asyncio.wait_for(asyncio.shield(job.task), DISK_BOUND_SECONDS) return job async def groups(self, job_id, status=200, **query): @@ -127,7 +131,7 @@ class DuplicateFilesTestCase(ApplyTestCase): plan = await self.review(job.id, [self.keep_of(group, path)]) body = await self.apply(plan["token"]) cleanup = self.api._jobs.get(body["job"]["id"]) - await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT) + await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS) return plan, cleanup async def until(self, predicate): @@ -178,7 +182,7 @@ class TestFindJob(DuplicateFilesTestCase): gate.set() body = await (await legacy).text() job = self.api._jobs.get(summary["id"]) - await asyncio.wait_for(asyncio.shield(job.task), WAIT) + await asyncio.wait_for(asyncio.shield(job.task), DISK_BOUND_SECONDS) events = _sse_payloads(body) done = events[-1] self.assertEqual(done["type"], "complete") @@ -241,7 +245,7 @@ class TestFindJob(DuplicateFilesTestCase): self.assertTrue((await cancel.json())["cancellation_requested"]) gate.set() job = self.api._jobs.get(job_id) - await asyncio.wait_for(asyncio.shield(job.task), WAIT) + await asyncio.wait_for(asyncio.shield(job.task), DISK_BOUND_SECONDS) self.assertEqual(job.state, "cancelled") last = _sse_payloads(await (await stream).text())[-1] self.assertEqual(last, {"type": "error", "message": "Duplicate scan cancelled"}) @@ -341,6 +345,8 @@ class TestReview(DuplicateFilesTestCase): self.assertEqual(plan["protected"], {"count": 0, "reasons": []}) async def test_cannot_delete_keeper_or_outside_group(self): + # About which files may go, not the backup: skip its fsync-heavy copy. + PromptManagerConfig.SAFETY_BACKUP = False self.trio() self.write("e.png", b"other pair") self.write("f.png", b"other pair") @@ -372,7 +378,7 @@ class TestReview(DuplicateFilesTestCase): self.assertEqual([row["label"] for row in plan["details"]], ["b.png"]) body = await self.apply(plan["token"]) cleanup = self.api._jobs.get(body["job"]["id"]) - await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT) + await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS) self.assertEqual(cleanup.counters["removed"], 1) self.assertTrue((self.output_dir / "a.png").is_file()) self.assertTrue((self.output_dir / "sub" / "c.png").is_file()) @@ -496,7 +502,7 @@ class TestCleanupJob(DuplicateFilesTestCase): plan = await self.review(job.id, keep) body = await self.apply(plan["token"]) cleanup = self.api._jobs.get(body["job"]["id"]) - await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT) + await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS) self.assertTrue(kept.is_file()) self.assertFalse(gone.exists()) self.assertFalse(video_thumb.exists()) @@ -508,7 +514,7 @@ class TestCleanupJob(DuplicateFilesTestCase): self.write("b.png", b"new bytes") # changed since the review body = await self.apply(plan["token"]) cleanup = self.api._jobs.get(body["job"]["id"]) - await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT) + await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS) self.assertEqual(cleanup.counters, {"removed": 1, "kept": 2, "errors": 0}) self.assertEqual((self.output_dir / "b.png").read_bytes(), b"new bytes") self.assertFalse((self.output_dir / "sub" / "c.png").exists()) @@ -522,7 +528,7 @@ class TestCleanupJob(DuplicateFilesTestCase): self.write("a.png", b"edited keeper") body = await self.apply(plan["token"]) cleanup = self.api._jobs.get(body["job"]["id"]) - await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT) + await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS) self.assertEqual(cleanup.counters["removed"], 0) self.assertTrue((self.output_dir / "b.png").is_file()) self.assertTrue((self.output_dir / "sub" / "c.png").is_file()) @@ -549,7 +555,7 @@ class TestCleanupJob(DuplicateFilesTestCase): with patch.object(duplicate_files, "_unlink", side_effect=unlink): body = await self.apply(plan["token"]) cleanup = self.api._jobs.get(body["job"]["id"]) - await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT) + await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS) self.assertEqual(cleanup.state, "failed") self.assertEqual(cleanup.counters, {"removed": 1, "kept": 1, "errors": 1}) self.assertEqual( @@ -584,7 +590,7 @@ class TestCleanupJob(DuplicateFilesTestCase): ): body = await self.apply(plan["token"]) cleanup = self.api._jobs.get(body["job"]["id"]) - await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT) + await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS) self.assertEqual(cleanup.counters, {"removed": 0, "kept": 3, "errors": 1}) self.assertEqual( cleanup.outcomes["errors"][0]["detail"], duplicate_files.DELETE_FAILED @@ -619,7 +625,7 @@ class TestCleanupJob(DuplicateFilesTestCase): with context: body = await self.apply(plan["token"]) cleanup = self.api._jobs.get(body["job"]["id"]) - await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT) + await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS) finally: if handle is not None: # pragma: no cover - Windows handle.close() @@ -642,7 +648,7 @@ class TestCleanupJob(DuplicateFilesTestCase): with patch.object(duplicate_files, "_unlink", side_effect=unlink): body = await self.apply(plan["token"]) cleanup = self.api._jobs.get(body["job"]["id"]) - await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT) + await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS) self.assertEqual(cleanup.counters["removed"], 2) self.assertEqual(open_at_unlink, []) @@ -703,7 +709,7 @@ class TestCleanupJob(DuplicateFilesTestCase): with patch.object(duplicate_files, "_unlink", side_effect=unlink): body = await self.apply(plan["token"]) cleanup = self.api._jobs.get(body["job"]["id"]) - await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT) + await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS) # Its records moved to the identical kept copy before the unlink failed. [image] = self.api.db.get_prompt_images(prompt) self.assertTrue(image["image_path"].endswith("a.png")) @@ -719,7 +725,7 @@ class TestCleanupJob(DuplicateFilesTestCase): self.assertEqual((busy["code"], busy["running"]["id"]), ("busy", find["id"])) gate.set() await asyncio.wait_for( - asyncio.shield(self.api._jobs.get(find["id"]).task), WAIT + asyncio.shield(self.api._jobs.get(find["id"]).task), DISK_BOUND_SECONDS ) # The token survived the refusal; the cleanup holds off a new find. release = asyncio.Event() @@ -737,7 +743,7 @@ class TestCleanupJob(DuplicateFilesTestCase): self.assertEqual(legacy.status, 409) release.set() cleanup = self.api._jobs.get(body["job"]["id"]) - await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT) + await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS) async def test_cleanup_refused_up_front_when_the_backup_does_not_fit(self): from py.api import maintenance @@ -965,7 +971,7 @@ class TestReviewHelpers(DuplicateFilesTestCase): GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.second_source())] body = await self.apply(plan["token"]) cleanup = self.api._jobs.get(body["job"]["id"]) - await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT) + await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS) self.assertEqual(cleanup.state, "failed") self.assertEqual(cleanup.last_event["code"], "sources-changed") self.assertEqual(self.backups(), []) @@ -985,7 +991,7 @@ class TestReviewHelpers(DuplicateFilesTestCase): ): body = await self.apply(plan["token"]) cleanup = self.api._jobs.get(body["job"]["id"]) - await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT) + await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS) self.assertEqual(cleanup.state, "failed") self.assertEqual( cleanup.outcomes["errors"][-1]["detail"], duplicate_files.RECORDS_FAILED diff --git a/tests/test_api_duplicate_safety.py b/tests/test_api_duplicate_safety.py index 02115143..8ae50edb 100644 --- a/tests/test_api_duplicate_safety.py +++ b/tests/test_api_duplicate_safety.py @@ -19,17 +19,17 @@ from pathlib import Path from unittest.mock import patch try: + from tests.disk_waits import DISK_BOUND_SECONDS from tests.open_handles import open_handles_under from tests import test_api_duplicate_files as dup_tests except ImportError: # discovered with tests/ as the top-level directory import test_api_duplicate_files as dup_tests + from disk_waits import DISK_BOUND_SECONDS from open_handles import open_handles_under from py.api import duplicate_files # noqa: E402 from py.config import GalleryConfig # noqa: E402 -WAIT = 10 - def _locator(path): return os.path.normcase(os.path.realpath(str(path))) @@ -49,7 +49,7 @@ class SafetyTestCase(dup_tests.DuplicateFilesTestCase): async def apply_plan(self, plan): body = await self.apply(plan["token"]) cleanup = self.api._jobs.get(body["job"]["id"]) - await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT) + await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS) return cleanup diff --git a/tests/test_api_history.py b/tests/test_api_history.py index 67476d7a..dc952332 100644 --- a/tests/test_api_history.py +++ b/tests/test_api_history.py @@ -1169,3 +1169,47 @@ class HistoryHostCleanupTests(unittest.IsolatedAsyncioTestCase): fence.validate(epoch, generation) with self.assertRaises(sqlite3.Error): api._history_fence() + + +class SharedTempFolderRestoreTests(ImageAPITestCase): + """#346: a restore survives another suite starting in the shared temp folder. + + Every ``PromptManagerAPI`` sweeps its database's folder for stages and + uploads an interrupted restore left. A fixture database directly in the + shared temp folder let any other test process (another worktree's suite, + a pre-push hook) delete this restore's files mid-request. The shared + folder here is a scratch stand-in for the system temp folder, so the test + never sweeps the real one. + """ + + async def get_application(self): + shared = tempfile.TemporaryDirectory(ignore_cleanup_errors=True) + # Cleanups run last-in first-out: this one after the fixture's own. + self.addCleanup(shared.cleanup) + self.shared_tmp = shared.name + with patch.object(tempfile, "tempdir", shared.name): + return await super().get_application() + + async def test_restore_survives_another_suite_sweeping_the_shared_folder(self): + from database import restore_stage + from database.models import PromptModel + + self.api.db.save_prompt("kept by the upload") + source = self.comfy_dir / "upload.db" + self.assertTrue(self.api.db.model.backup_database(str(source))) + stage = PromptModel.stage_restore + swept = [] + + def stage_then_other_suite_starts(model, src_path): + staged = stage(model, src_path) + # What another suite's PromptManagerAPI does at startup. + other = os.path.join(self.shared_tmp, "other-suite.db") + swept.extend(restore_stage.sweep_restore_temps(other)) + return staged + + form = FormData() + form.add_field("database_file", source.read_bytes(), filename="upload.db") + with patch.object(PromptModel, "stage_restore", stage_then_other_suite_starts): + response = await self.client.post("/prompt_manager/restore", data=form) + self.assertEqual(response.status, 200, await response.text()) + self.assertEqual(swept, [], "another suite's sweep reached this restore") diff --git a/tests/test_api_images.py b/tests/test_api_images.py index 08713ddb..cc74c6cf 100644 --- a/tests/test_api_images.py +++ b/tests/test_api_images.py @@ -42,6 +42,7 @@ from py import lora_utils # noqa: E402 from py import lora_manager_layout as layout_module # noqa: E402 from py.api.thumbnail_settings import ThumbnailSettings # noqa: E402 from py.api import PromptManagerAPI # noqa: E402 +from tests.private_temp import private_temp_db # noqa: E402 from py.api.prompts import MAX_PAGE_LIMIT # noqa: E402 from py.config import GalleryConfig # noqa: E402 from utils.hashing import generate_prompt_hash # noqa: E402 @@ -60,8 +61,7 @@ class ImageAPITestCase(AioHTTPTestCase): """App with PromptManager routes, a temp DB and a temp output directory.""" async def get_application(self): - self._temp_db = tempfile.NamedTemporaryFile(delete=False, suffix=".db") - self._temp_db.close() + self._temp_db = private_temp_db(self) self._output_tmp = tempfile.TemporaryDirectory(ignore_cleanup_errors=True) # A fake ComfyUI tree: <tmp>/output is the gallery root and <tmp> is the diff --git a/tests/test_api_lora.py b/tests/test_api_lora.py index 8fda0562..99446360 100644 --- a/tests/test_api_lora.py +++ b/tests/test_api_lora.py @@ -36,6 +36,7 @@ from aiohttp.test_utils import AioHTTPTestCase from database.operations import PromptDatabase from py.api import PromptManagerAPI +from tests.private_temp import private_temp_db from py import config_store, lora_import, lora_utils from py.config import IntegrationConfig, PromptManagerConfig from py.api import settings_routes as settings_module @@ -57,8 +58,7 @@ class LoraAPITestCase(AioHTTPTestCase): """Stands up the PromptManager routes with a temp DB and neutral LoRA config.""" async def get_application(self): - self._temp_db = tempfile.NamedTemporaryFile(delete=False, suffix=".db") - self._temp_db.close() + self._temp_db = private_temp_db(self) self.tmp_root = Path(tempfile.mkdtemp()) self.custom_nodes = self.tmp_root / "custom_nodes" self.custom_nodes.mkdir() diff --git a/tests/test_api_prompts.py b/tests/test_api_prompts.py index c4af9c68..c8625c25 100644 --- a/tests/test_api_prompts.py +++ b/tests/test_api_prompts.py @@ -5,7 +5,6 @@ Route tests for the prompt API (py/api/prompts.py). import json import os import sys -import tempfile import unittest from unittest.mock import MagicMock, patch @@ -29,6 +28,7 @@ except ImportError: # discovered with tests/ as the top-level directory from database.operations import PromptDatabase # noqa: E402 from py.api import PromptManagerAPI # noqa: E402 +from tests.private_temp import private_temp_db # noqa: E402 import py.api.prompts as prompts_module # noqa: E402 from py.api.prompts import ( # noqa: E402 BULK_CATEGORY_MAX_PROMPTS, @@ -43,8 +43,7 @@ class PromptAPITestCase(AioHTTPTestCase): """App with PromptManager routes and a temporary SQLite database.""" async def get_application(self): - self._temp_db = tempfile.NamedTemporaryFile(delete=False, suffix=".db") - self._temp_db.close() + self._temp_db = private_temp_db(self) app = web.Application() routes = web.RouteTableDef() diff --git a/tests/test_execution_history_writer.py b/tests/test_execution_history_writer.py index 1d93dcbd..c0b11633 100644 --- a/tests/test_execution_history_writer.py +++ b/tests/test_execution_history_writer.py @@ -31,6 +31,12 @@ from utils.execution_history_writer import HistoryWriter # 0.19 s on macOS py3.13 (PR #351). NON_BLOCKING_CEILING = 1.0 +# Saturation tests block the worker while they queue 1023 more items. The +# block must hold until the test releases it (#346/#352): a 3 s timer let a +# loaded runner unblock the worker mid-fill and drain it. This is only a hang +# guard; the test, or cleanup, always sets the release. +SATURATION_HOLD_SECONDS = 60 + class ManualClock: def __init__(self): @@ -84,6 +90,26 @@ class HistoryWriterTests(unittest.TestCase): threading.Event().wait(0.005) self.fail("finite deadline expired waiting for worker behavior") + def wait_for_worker_pass(self, writer): + """Wait until the idle worker has run one whole loop pass from now. + + A worker with no accepted work ends every pass by parking on its wake + event. The first park counted may end a pass already under way; the + second ends a pass that began after this call, so whatever the worker + decides at the current clock value has happened (state, not a + wall-clock pause that a loaded runner can outlast). + """ + wake = writer._wake + park = wake.wait + parks = [] + + def counted(timeout=None): + parks.append(timeout) + return park(timeout) + + with patch.object(wake, "wait", counted): + self.wait_for(lambda: len(parks) >= 2) + def writer(self, factory=None, ready=True): writer = HistoryWriter(factory or (lambda: self.db), clock=self.clock) self.writers.append(writer) @@ -301,7 +327,7 @@ class HistoryWriterTests(unittest.TestCase): def write(command): entered.set() - release.wait(3) + release.wait(SATURATION_HOLD_SECONDS) return original(command) with patch.object(self.db, "write_execution_start", write): @@ -348,9 +374,12 @@ class HistoryWriterTests(unittest.TestCase): first = self.sealed(writer) charge = first.lease.byte_count + left = threading.Event() + def write(*args, **kwargs): entered.set() release.wait(3) + left.set() return original(*args, **kwargs) with patch.object(self.db, "write_execution_contexts_for_finish", write): @@ -396,8 +425,21 @@ class HistoryWriterTests(unittest.TestCase): before = monotonic() self.assertFalse(writer.close(0.02)) elapsed = monotonic() - before + # close() waits until its own monotonic deadline, so the lower + # bound holds on any runner. self.assertGreaterEqual(elapsed, 0.02) - self.assertLess(elapsed, 0.2) + if early_join: + # Manual clock: only the patched join/sleep advance it, so this + # exact bound proves early joins do not spin forever. + self.assertLess(elapsed, 0.2) + else: + # Real clock: a loaded runner may run late, but close() must + # not wait anywhere near the 3 s the blocked write holds. + self.assertLess(elapsed, 0.02 + NON_BLOCKING_CEILING) + # Bounded on the real clock too: close() gave up while the + # in-flight write was still blocked, instead of waiting it out. + self.assertFalse(left.is_set()) + self.assertTrue(writer.status().alive) self.assertEqual(pool.charged_bytes, charge) replacement = HistoryWriter(lambda: self.db, context_budget=pool) self.writers.append(replacement) @@ -771,7 +813,7 @@ class HistoryWriterTests(unittest.TestCase): self.wait_for(lambda: not writer.status().accepting) self.assertFalse(writer.submit(fresh)) self.clock.advance(4.99) - threading.Event().wait(0.08) + self.wait_for_worker_pass(writer) self.assertEqual(probes, [0]) self.clock.advance(0.01) self.wait_for(lambda: writer.status().accepting) @@ -794,12 +836,12 @@ class HistoryWriterTests(unittest.TestCase): writer = self.writer(factory, ready=False) self.wait_for(lambda: len(calls) == 1) self.clock.advance(4.99) - threading.Event().wait(0.08) + self.wait_for_worker_pass(writer) self.assertEqual(calls, [0]) self.clock.advance(0.01) self.wait_for(lambda: len(calls) == 2) self.clock.advance(4.99) - threading.Event().wait(0.08) + self.wait_for_worker_pass(writer) self.assertEqual(calls, [0, 5]) self.clock.advance(0.01) self.wait_for(lambda: writer.status().accepting) @@ -853,7 +895,7 @@ class HistoryWriterTests(unittest.TestCase): ) writer.invalidate() release.set() - threading.Event().wait(0.08) + self.wait_for_worker_pass(writer) self.assertFalse(writer.status().accepting) self.assertIsNone(writer.status().epoch) self.assertFalse(writer.submit(self.start_command(epoch=stale_epoch))) @@ -871,7 +913,7 @@ class HistoryWriterTests(unittest.TestCase): lock.execute("BEGIN IMMEDIATE") before = time.monotonic() self.assertTrue(writer.submit(command)) - self.assertLess(time.monotonic() - before, 0.1) + self.assertLess(time.monotonic() - before, NON_BLOCKING_CEILING) self.wait_for(lambda: not writer.status().accepting) lock.rollback() self.clock.advance(5) diff --git a/tests/test_execution_output_writer.py b/tests/test_execution_output_writer.py index 06475d24..b274e535 100644 --- a/tests/test_execution_output_writer.py +++ b/tests/test_execution_output_writer.py @@ -18,6 +18,7 @@ from utils.execution_output_capture import seal_outputs from utils.execution_outputs import snapshot_output_roots from utils.execution_history_writer import HistoryWriter from tests.history_waits import drain_writer +from tests.disk_waits import DISK_BOUND_SECONDS try: from tests import test_execution_history_writer as fixtures @@ -28,11 +29,17 @@ except ImportError: # Upper bound a blocked fake write waits for its release. Tests prove ordering # with events, so this only caps a failing run and never decides a pass. BLOCK_SECONDS = 10 + +# Saturation tests block the worker while they queue 1023 more items. The +# block must hold until the test releases it (#346/#352): a 3 s timer let a +# loaded runner unblock the worker mid-fill and drain it. This is only a hang +# guard; the test, or cleanup, always sets the release. +SATURATION_HOLD_SECONDS = 60 + # A restore is a full verify/migrate/backup/replace cycle and is bound by the # disk: a busy Windows runner took over 10 s (merge queue, #382). Waits that -# span a restore use this ceiling, which stays under the runner's 120 s hang -# watchdog, so a real hang still fails with the thread dump. -RESTORE_SECONDS = 90 +# span a restore use the shared disk-bound hang guard (tests/disk_waits.py). +RESTORE_SECONDS = DISK_BOUND_SECONDS class OutputWriterTests(fixtures.unittest.TestCase): @@ -591,7 +598,7 @@ class OutputWriterTests(fixtures.unittest.TestCase): def write(command): entered.set() - release.wait(3) + release.wait(SATURATION_HOLD_SECONDS) return original(command) with patch.object(self.db, "write_execution_start", write): diff --git a/tests/vendor/ci-workflow.test.mjs b/tests/vendor/ci-workflow.test.mjs index f489fd64..cfafcaae 100644 --- a/tests/vendor/ci-workflow.test.mjs +++ b/tests/vendor/ci-workflow.test.mjs @@ -247,3 +247,32 @@ test('the Python matrix and coverage run the suite through the watchdog runner', assert.doesNotMatch(found.get(id), /unittest discover/, `${id} no longer runs the bare unittest discover`); } }); + +// CI hygiene (#352): lint tools are pinned, the bandit hook can fail, the JS job caches npm. +const readRepoFile = path => readFileSync(new URL(`../../${path}`, import.meta.url), 'utf8'); + +test('black and flake8 are pinned exactly and the lint job installs those pins', () => { + const requirements = readRepoFile('requirements-dev.txt'); + assert.match(requirements, /^black==\d+\.\d+\.\d+$/m, 'black is pinned exactly'); + assert.match(requirements, /^flake8==\d+\.\d+\.\d+$/m, 'flake8 is pinned exactly'); + const lint = jobs(readWorkflow('test.yml')).get('lint'); + assert.match(lint, /pip install \$\(grep -E '\^\(black\|flake8\)==' requirements-dev\.txt\)/); + assert.doesNotMatch(lint, /pip install black flake8/, 'lint no longer installs the latest releases'); +}); + +test('the frontend modules job caches npm keyed on the vendor lockfile', () => { + const setupNode = steps(jobs(readWorkflow('test.yml')).get('js-test')) + .find(step => /uses: actions\/setup-node@v\d+/.test(step)); + assert.match(setupNode, /^ {10}cache: npm$/m); + assert.match(setupNode, /^ {10}cache-dependency-path: tools\/vendor\/package-lock\.json$/m); +}); + +test('the bandit pre-commit hook fails at the severity CI enforces and passes every file', () => { + const quality = readWorkflow('code-quality.yml'); + assert.match(quality, /^ {8}run: bandit -r \. -lll -q --exclude /m, 'CI fails on high severity'); + const hook = /- id: bandit\n([\s\S]*?)(?=\n {6}(?:- id:|#)|\n*$)/.exec(readRepoFile('.pre-commit-config.yaml'))?.[1]; + assert.ok(hook, 'the bandit hook exists'); + assert.match(hook, /^ {8}entry: bandit -lll -q$/m, 'the hook enforces the same -lll level as CI'); + assert.doesNotMatch(hook, /true|exit-zero|bash -c/, 'nothing masks the exit code or drops a file'); + assert.doesNotMatch(hook, /pass_filenames: false/, 'the hook scans the staged files'); +}); From e4dc781c26e8f541bc187ed35dbcaf8de9bb4c96 Mon Sep 17 00:00:00 2001 From: Vito <vito.sansevero@gmail.com> Date: Wed, 28 Oct 2026 06:16:57 +0000 Subject: [PATCH 08/11] fix(images): read a loading collection as loading, and de-flake the JXL fixture and restore specs (#391, #371, #361) (#405) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(images): read a loading collection as loading, never as 0 of 0 images (#391) A reset published {loading, loaded:0, total:0} and the caption painted it as "0 of 0 images" until the first page answered, so a fast reader (the inspector concept spec on WebKit) read an empty result the server never returned. The count line now says "Loading images…" while a first page is in flight; the spec waits for the answered count, and a held-listing spec pins the ordering. * test(thumbnails): make the no-JPEG-XL-encoder fixture authoritative (#371) {available:false} stood in only the cached detection, and every settings read probes a cached None again, finding the host's real encoder. The fixture now stands in for the detector itself and reset restores it; a unit test proves the route against a detector that finds an encoder. * test(system): wait for the restore job's own end before its toast (#361) The reviewed-restore specs waited for the "Database restore finished" toast with the 5 s expect, in effect a deadline on the disk-bound publish. They now wait for the job's progress stream to close, assert the job finished, and only then check the toast with the default timeout. * test(images): wait for the search's answered count in the concept comparison (#391) searchOf took the first changed caption as the search's count, the same read-before-the-response race as the inspector's tag chip. --- tests/browser/images-concept.spec.mjs | 3 +- tests/browser/inspector-concept.spec.mjs | 22 ++++++- .../browser/workspace-system-backups.spec.mjs | 16 ++++- tests/browser/workspace_system_fixture.py | 16 ++++- tests/js/workspace-images-state.test.js | 17 ++++++ tests/test_workspace_system_fixture.py | 60 ++++++++++++++++++- web/js/workspace/images-state.mjs | 9 ++- 7 files changed, 134 insertions(+), 9 deletions(-) diff --git a/tests/browser/images-concept.spec.mjs b/tests/browser/images-concept.spec.mjs index 08479b3d..fea7a5ea 100644 --- a/tests/browser/images-concept.spec.mjs +++ b/tests/browser/images-concept.spec.mjs @@ -286,7 +286,8 @@ const continuationText=page=>page.locator('.infinite-continuation > p').first(). async function searchOf(page,term) { const before=await countText(page); await page.locator('#search').fill(term); - await expect.poll(()=>countText(page)).not.toBe(before); + // Production's caption reads "Loading images…" until the search's listing answers (#391); the concept's is instant. + await expect.poll(async()=>{const text=await countText(page);return text!==before&&text!=='Loading images…';}).toBe(true); await expect.poll(()=>page.locator(CARDS).count()).toBeLessThan(FIXTURE_IMAGES); await settle(page); const result={count:await countText(page),continuation:await continuationText(page),cards:await page.locator(`${CARDS} .image-open`).evaluateAll(nodes=>nodes.map(node=>node.getAttribute('aria-label')))}; diff --git a/tests/browser/inspector-concept.spec.mjs b/tests/browser/inspector-concept.spec.mjs index 0a43d7f1..48b24739 100644 --- a/tests/browser/inspector-concept.spec.mjs +++ b/tests/browser/inspector-concept.spec.mjs @@ -228,7 +228,8 @@ async function productionSendOf(page,context) { async function tagOf(page) { const before=await page.locator('#result-count').textContent(); await panelOf(page).locator('.detail-inner > .tags').last().getByRole('button',{name:'cinematic',exact:true}).click(); - await expect.poll(()=>page.locator('#result-count').textContent()).not.toBe(before); + // Production's caption reads "Loading images…" until the tag's listing answers (#391); the concept's search is instant. + await expect.poll(async()=>{const text=await page.locator('#result-count').textContent();return text!==before&&text!=='Loading images…';}).toBe(true); await settle(page); return {count:await page.locator('#result-count').textContent(), cards:await page.locator(`${CARDS} .image-open`).evaluateAll(nodes=>nodes.map(node=>node.getAttribute('aria-label')))}; @@ -328,6 +329,25 @@ test('a recipe-linked image without recipe metadata shows its positive prompt me await expect(panelOf(page).locator('.detail-footer button')).toHaveText(['Send to ComfyUI','Prompt history']); }); +// #391: a tag chip replaces the collection; until its listing answers the caption says it is loading, never that the +// filter matched nothing ("0 of 0 images" was painted before the response and read by a fast reader). +test('a tag chip collection reads as loading, not empty, until its listing answers (#391)',async({page})=>{ + await page.setViewportSize({width:1600,height:1100}); + await openProductionPanel(page,'dark'); + let release,arrived; + const held=new Promise(resolve=>{release=resolve;}),reached=new Promise(resolve=>{arrived=resolve;}); + await page.route(url=>url.pathname==='/prompt_manager/images/output'&&url.searchParams.get('tags')==='cinematic', + async route=>{arrived();await held;await route.continue();},{times:1}); + const count=page.locator('#result-count'); + await panelOf(page).locator('.detail-inner > .tags').last().getByRole('button',{name:'cinematic',exact:true}).click(); + await reached; + await expect(count).toHaveText('Loading images…'); + await expect(page.locator(CARDS)).toHaveCount(0); + release(); + await expect(count).toHaveText(countCopy('8 of 8 images · 2 positive prompts')); + await expect(page.locator(CARDS)).toHaveCount(8); +}); + // Task C3 (D88, #249 PR C): the concept's chip row is the image's own tags (`i.tags`, app.js a402). With those names // saved as the file's current image tags, production draws the row from them (Tags filter buttons in the panel, plain // in the viewer, the linked positive's equal tags not repeated) and still equals the concept, in both sizes. diff --git a/tests/browser/workspace-system-backups.spec.mjs b/tests/browser/workspace-system-backups.spec.mjs index 6856490f..6251d0f1 100644 --- a/tests/browser/workspace-system-backups.spec.mjs +++ b/tests/browser/workspace-system-backups.spec.mjs @@ -47,6 +47,17 @@ async function serverJson(request, url) { return response.json(); } +/** + * The restore job's own end (#361): its progress stream closes when the job takes its final state, however long the + * disk-bound publish takes (bounded by the test, not by a 5 s expect), and the job finished. The toast the page shows + * for it is then the page's reaction alone, checked with the default timeout. + */ +async function restoreFinished(request, id) { + await expect(await request.get(`/prompt_manager/jobs/${id}/events`)).toBeOK(); + const { job } = await serverJson(request, `/prompt_manager/jobs/${id}`); + expect([job.kind, job.state]).toEqual(['restore', 'finished']); +} + async function openBackups(page) { // A fresh page each time: the drafts live for the page session (Q10), and a hash-only goto would not reload. if (page.url().endsWith('#/system/backups')) await page.reload(); @@ -216,6 +227,7 @@ test('Restore starts the job, opens Activity; views reload after it finishes', a const { job } = await (await applied).json(); expect([job.kind, job.title, job.cancellable]).toEqual(['restore', 'Database restore', false]); await expect(page).toHaveURL(`${workspace}#/activity/${job.id}`); + await restoreFinished(request, job.id); await expect(toast(page)).toHaveText(/^Database restore finished · Restored \d+ prompts?, /); // The library is the backup's again, and the page says so without a reload: the sidebar counts were read again. expect(await libraryTotal(request)).toBe(restoredTotal); @@ -244,7 +256,8 @@ test('a finished restore shows the open page again from the restored data', asyn const card = page.locator('#workspace-main article.prompt-card').filter({ hasText: 'S9 prompt the restore removes' }); await expect(card).toHaveCount(1); await expect(await request.post('/__test__/release')).toBeOK(); - await applied; + const { job } = await (await applied).json(); + await restoreFinished(request, job.id); await expect(toast(page)).toHaveText(/^Database restore finished · /); // The shelf was read again from the restored library, on the same route. await expect(card).toHaveCount(0); @@ -422,6 +435,7 @@ test('a refused restore reopens its review: Restore retries it, Discard releases await dialog(page).getByRole('button', { name: 'Restore', exact: true }).click(); const { job } = await (await applied).json(); await expect(page).toHaveURL(`${workspace}#/activity/${job.id}`); + await restoreFinished(request, job.id); await expect(toast(page)).toHaveText(/^Database restore finished · /); expect(await stagesOf(request)).toBe(0); }); diff --git a/tests/browser/workspace_system_fixture.py b/tests/browser/workspace_system_fixture.py index 9900abc8..c7d87af3 100644 --- a/tests/browser/workspace_system_fixture.py +++ b/tests/browser/workspace_system_fixture.py @@ -35,7 +35,8 @@ read. These per-server hooks only change what a System test needs on top: previews for Convert outdated; - ``POST /__test__/system/jxl-encoder`` ``{available}`` (#317) makes the server read as having (``cjxl``) or lacking a JPEG XL encoder without - probing; reset restores the thumbnail settings and the real detection. + probing, whatever the host has installed (#371: ``false`` stands in for the + detector itself); reset restores the thumbnail settings and the real detection. - ``POST /__test__/system/duplicate-prompts`` (S7) adds two prompts whose text differs only in case and spacing (a duplicate prompt text group); @@ -118,6 +119,11 @@ def register_system_commands(app, routes, api, root: Path): } from py.api import thumbnail_encode + real_detect = thumbnail_encode._detect + + def _no_encoder(): + return None + monitor = getattr(image_monitor, "_monitor_instance", None) failures: dict[str, int] = {} @@ -147,6 +153,7 @@ def register_system_commands(app, routes, api, root: Path): api._health_report = None for (owner, name), value in host_values.items(): setattr(owner, name, value) + thumbnail_encode._detect = real_detect thumbnail_encode.reset_encoders() image_monitor._monitor_instance = monitor api.__dict__.pop("_library_resolver", None) @@ -298,7 +305,12 @@ def register_system_commands(app, routes, api, root: Path): available = data.get("available") if not isinstance(available, bool): raise web.HTTPBadRequest() - # Test-only: stand in the cached detection (no probe). + # Test-only: the fixture, not the host, decides (#371). Without an encoder the + # detector itself stands in, so a re-probe (a "none found" is probed again on + # every settings read) cannot find the host's real encoder; with one the cached + # answer stands, and a failed write still probes the real detector as before. + # Reset restores the real detector. + thumbnail_encode._detect = real_detect if available else _no_encoder with thumbnail_encode._detect_lock: thumbnail_encode._detected = "cjxl" if available else None return web.json_response({"success": True}) diff --git a/tests/js/workspace-images-state.test.js b/tests/js/workspace-images-state.test.js index 49cc0801..db74a8b6 100644 --- a/tests/js/workspace-images-state.test.js +++ b/tests/js/workspace-images-state.test.js @@ -234,6 +234,23 @@ test('the count line reads N of M images and K positive prompts only when known' assert.equal(countLine({loaded:1200,total:3400,positiveCount:0}),'1,200 of 3,400 images · 0 positive prompts'); }); +test('the count line reads as loading, never as an empty result, until a first page answers (#391)', async () => { + const {countLine,createImageStream}=await source; + assert.equal(countLine({loaded:0,total:0,positiveCount:null,loading:true,error:null}),'Loading images…'); + // A failed first page and a settled empty one say what the server said; a later page keeps the loaded count. + assert.equal(countLine({loaded:0,total:0,positiveCount:null,loading:false,error:new Error('x')}),'0 of 0 images'); + assert.equal(countLine({loaded:0,total:0,positiveCount:0,loading:false,error:null}),'0 of 0 images · 0 positive prompts'); + assert.equal(countLine({loaded:40,total:102,positiveCount:2,loading:true,error:null}),'40 of 102 images · 2 positive prompts'); + // While a reset's first page is in flight the line reads as loading (the cleared state before it is replaced in + // the same task, so it is never painted). + let answer;const lines=[]; + const stream=createImageStream({api:{get:()=>new Promise(resolve=>{answer=resolve;})},onChange:state=>lines.push(countLine(state))}); + const loading=stream.reset({tags:['cinematic']}); + assert.equal(lines.at(-1),'Loading images…'); + answer({images:[{path:'a.png',root_index:0}],total:1,has_more:false,next_cursor:null,positive_count:1});await loading; + assert.equal(lines.at(-1),'1 of 1 image · 1 positive prompt'); +}); + test('the caption date names the newest result day under Newest first only', async () => { const {captionDate}=await source; const now=new Date(2026,9,17,15,0); diff --git a/tests/test_workspace_system_fixture.py b/tests/test_workspace_system_fixture.py index a43ec854..23189560 100644 --- a/tests/test_workspace_system_fixture.py +++ b/tests/test_workspace_system_fixture.py @@ -1,12 +1,26 @@ -"""The System browser hooks only ever reset their own server's config.json (#249 S3).""" +"""The System browser hooks only ever reset their own server's config.json (#249 S3) +and decide JPEG XL availability whatever the host has installed (#371).""" +import asyncio import os +import sys import tempfile +import types import unittest from pathlib import Path from unittest import mock -from tests.browser.workspace_system_fixture import register_system_commands +from aiohttp import web +from aiohttp.test_utils import TestClient, TestServer + +_mock_server = mock.MagicMock() +_mock_server.PromptServer.instance.routes = mock.MagicMock() +sys.modules.setdefault("server", _mock_server) + +from py.api import thumbnail_encode # noqa: E402 +from tests.browser.workspace_system_fixture import ( # noqa: E402 + register_system_commands, +) class TestWorkspaceSystemFixture(unittest.TestCase): @@ -33,6 +47,48 @@ class TestWorkspaceSystemFixture(unittest.TestCase): register_system_commands(None, None, None, self.root) self.assertEqual(elsewhere.read_bytes(), b'{"real": true}') + def test_no_encoder_holds_even_when_the_host_has_one(self): + """``{available: false}`` survives the settings read's re-probe (#371).""" + config = self.root / "config.json" + env = {**os.environ, "PROMPT_MANAGER_CONFIG_PATH": str(config)} + api = types.SimpleNamespace() + app, routes = web.Application(), web.RouteTableDef() + real_detect = thumbnail_encode._detect + with mock.patch.dict(os.environ, env, clear=True): + reset = register_system_commands(app, routes, api, self.root) + app.add_routes(routes) + self.addCleanup(thumbnail_encode.reset_encoders) + self.addCleanup(setattr, thumbnail_encode, "_detect", real_detect) + + async def scenario(): + async with TestClient(TestServer(app)) as client: + + async def post(available): + response = await client.post( + "/__test__/system/jxl-encoder", json={"available": available} + ) + self.assertEqual(response.status, 200) + + await post(False) + status = thumbnail_encode.encoder_status()["jxl"] + self.assertFalse(status["available"]) + self.assertIsNone( + thumbnail_encode.detect_jxl_encoder(recheck_missing=True) + ) + await post(True) + status = thumbnail_encode.encoder_status()["jxl"] + self.assertEqual(status["encoder"], "cjxl") + await post(False) + + # The host's real detector finds an encoder (Pillow can save JPEG XL). + with mock.patch.object(thumbnail_encode, "_pillow_can_save_jxl", lambda: True): + asyncio.run(scenario()) + # Reset gives detection back to the host. + reset() + self.assertIs(thumbnail_encode._detect, real_detect) + status = thumbnail_encode.encoder_status()["jxl"] + self.assertEqual(status["encoder"], "pillow") + if __name__ == "__main__": unittest.main() diff --git a/web/js/workspace/images-state.mjs b/web/js/workspace/images-state.mjs index 3b702f79..99364b92 100644 --- a/web/js/workspace/images-state.mjs +++ b/web/js/workspace/images-state.mjs @@ -275,8 +275,13 @@ export function createImageStream({ api, onChange = () => {}, batchSize = 40, re }; } -/** The concept's count line (app.js a446): "N of M images · K positive prompts", without the count while unknown. */ -export function countLine({ loaded, total, positiveCount }) { +/** + * The concept's count line (app.js a446): "N of M images · K positive prompts", without the count while unknown, and + * the page's first "Loading images…" while a first page loads. + */ +export function countLine({ loaded, total, positiveCount, loading = false, error = null }) { + // A first page in flight has no count yet: "0 of 0 images" would claim the filter matched nothing (#391). + if (loading && !loaded && !error) return 'Loading images…'; const images = `${loaded.toLocaleString('en-US')} of ${total.toLocaleString('en-US')} ${noun(total, 'image')}`; return isCount(positiveCount) ? `${images} · ${positiveCount.toLocaleString('en-US')} ${noun(positiveCount, 'positive prompt')}` : images; } From 534c95248aafd8f454e7e54ced27342a2a661a79 Mon Sep 17 00:00:00 2001 From: Vito <vito.sansevero@gmail.com> Date: Wed, 28 Oct 2026 07:22:48 +0000 Subject: [PATCH 09/11] fix(review): anchor files on their saver in the scan and Needs review (#390) (#414) * fix(review): anchor files on their saver in the scan and Needs review (#390) The scan and Apply now resolve with the file's name, as the workspace inspector does since #383, so the one saver whose filename_prefix names the file anchors its branch. A file whose branch that anchor resolves is never queued (from the inspection profile it offers no choice; a default-ambiguous file asks saver_anchored) and its pending, undecided items leave on the next scan. REVIEW_RULE_VERSION is folded into inspection item versions so old items are re-evaluated. No match, several matches or an unfinished walk anchor nothing. Apply refuses a choice the anchor contradicts. * fix(review): refuse a default choice the saver anchor contradicts (#390) Apply re-resolves a default item with the file's name: when its saver anchors the branch, no candidate is offered and the choice is refused. The scan's anchor check skips bytes carrying a decision and remembers unanchored file states per process, so a rescan does not inspect an unchanged file again. * fix(review): queue an anchored file with its saver's pair as the choice (#390) Maintainer ruling: a file never falls through both the link and the queue. When only the saver anchor resolves a file the default profile and the parser floor leave unlinked, the scan queues it with the anchor's pair as its one candidate (evidence anchor: saver), never linking it. Files already linked or resolved by the default path are unchanged. Apply accepts the anchored pair when the bytes now resolve to it, and an anchored item is a choice the untraced settle keeps. * fix(review): upgrade old default items of anchored files, tag anchored results (#390) A pending default item recorded before the saver anchor no longer shortcuts the rescan when only the anchor resolves the file: the anchored item supersedes it. Apply accepts a resolved re-resolution only when it is tagged as the saver anchor's, the anchored pair must carry a positive text to choose, and limited anchored evidence still counts as a choice. TESTING.md describes the ruling. --- TESTING.md | 18 +- database/review_schema.py | 6 +- py/api/admin.py | 17 +- py/api/review.py | 12 +- py/api/review_recording.py | 5 +- tests/test_api_review.py | 8 +- tests/test_review_inspection.py | 44 ++-- tests/test_review_saver_anchor.py | 333 ++++++++++++++++++++++++++++++ utils/review_capture.py | 207 ++++++++++++++++--- 9 files changed, 588 insertions(+), 62 deletions(-) create mode 100644 tests/test_review_saver_anchor.py diff --git a/TESTING.md b/TESTING.md index d66064e8..d0ab868c 100644 --- a/TESTING.md +++ b/TESTING.md @@ -4623,7 +4623,23 @@ files, oldest forgotten first, keyed by 16-byte digests: 8.6 MiB measured at the cap). A file whose inspection raises is logged, counted in `review_errors` and reported `unresolved`; the scan goes on. An older version's pending, undecided item of bytes now untraced leaves the queue -(`settle_stale_review_items`), so an item never stays stuck at 409. The 16-candidate limited +(`settle_stale_review_items`), so an item never stays stuck at 409. +Saver anchor (#390, `tests/test_review_saver_anchor.py`): the scan and Apply +resolve with the file's name as the workspace inspector does, so the one saver +whose `filename_prefix` names the file anchors its branch. A file is never both +unlinked and unqueued (maintainer ruling): when only that anchor resolves it +(the named inspection run resolves, the unnamed one does not) and neither the +default profile nor the parser floor links it, the scan queues it with the +anchor's pair as its one candidate (evidence `anchor: "saver"`; the page +preselects it), never linking it. Such an item counts as a choice for the +untraced settle. Files already linked or resolved by the default path are +unchanged. No match, several matches or an unfinished walk anchor nothing, and +the file is queued as before. `REVIEW_RULE_VERSION` is folded into inspection +item versions, so older inspection items are recorded again once. Apply accepts +the anchored pair and refuses a choice the anchor contradicts. Measured on the +maintainer's 40,890 outputs: the queue is unchanged (249 files); the 242 +`gpu_unlock_bench` files the inspector resolves are linked by the parser +floor. The 16-candidate limited rule and the 8 MiB bound apply unchanged. The queue heals itself: after each batch, the scan removes the pending items of every listed file (read or skipped as known) that a saved row now links, and the monitor does the same for each ambiguous output it links diff --git a/database/review_schema.py b/database/review_schema.py index 0aa8318b..c0c45d76 100644 --- a/database/review_schema.py +++ b/database/review_schema.py @@ -229,13 +229,15 @@ def evidence_offers_choice(evidence: Any) -> bool: """ if not isinstance(evidence, dict): return False + # The saver anchor's one pair (#390) is a suggestion to confirm. + needed = 1 if evidence.get("anchor") == "saver" else MIN_CHOOSABLE if evidence.get("limited"): total = evidence.get("candidate_total") - return type(total) is int and total >= MIN_CHOOSABLE + return type(total) is int and total >= needed candidates = evidence.get("candidates") if not isinstance(candidates, list): return False - return sum(1 for c in candidates if choosable(c)) >= MIN_CHOOSABLE + return sum(1 for c in candidates if choosable(c)) >= needed def evidence_digest(text: str) -> str: diff --git a/py/api/admin.py b/py/api/admin.py index ec3f6db7..047c6b8a 100644 --- a/py/api/admin.py +++ b/py/api/admin.py @@ -1630,7 +1630,14 @@ class AdminRoutesMixin: if self._review_file_linked(media_file, roots): outcome = "unchanged" # already linked: not a review case else: - self._record_scanned_review(media_file, roots, review) + # The records name the item versions that count; the + # saver anchor (#390) is checked on the captured bytes. + self._record_scanned_review( + media_file, + roots, + review, + scanned=(records.get("canonical"), None), + ) elif untraced and outcome == "unresolved": # Untraced by the default profile: the inspection profile may # still offer a choice (queued for review, never linked). @@ -1691,9 +1698,9 @@ class AdminRoutesMixin: ``"ambiguous"`` (nothing linked it) is a Needs review case. An ambiguous result with nothing to choose (``reviewable`` is False: fewer than two candidates carry a prompt) that nothing links returns - ``"unresolved"`` instead, and ``media_file`` goes to ``untraced``; - its parsed records go to ``records["canonical"]`` when ``records`` is - a dict. + ``"unresolved"`` instead, and ``media_file`` goes to ``untraced``. + The parsed records of either ambiguous file go to + ``records["canonical"]`` when ``records`` is a dict. """ if not metadata: # No prompt metadata (--disable-metadata, bare savers): only the @@ -1723,6 +1730,8 @@ class AdminRoutesMixin: records["canonical"] = canonical elif resolution["state"] == "ambiguous" and ambiguous is not None: ambiguous.append(media_file) + if records is not None: + records["canonical"] = canonical if resolution["state"] == "resolved": result = ingest_resolved_pair( self.db, str(media_file), canonical, resolution diff --git a/py/api/review.py b/py/api/review.py index 6f81e545..a07c02ce 100644 --- a/py/api/review.py +++ b/py/api/review.py @@ -677,7 +677,14 @@ def _verify_and_read(store, capture, item, root, need_metadata: bool): facts = store.verify(root, relative) captured = None if need_metadata: - captured = captured_review(store, capture, item_profile(item["evidence"])) + # The file's own name, so an inspection item resolves anchored on its + # saver as it was recorded (#390). + captured = captured_review( + store, + capture, + item_profile(item["evidence"]), + relative.rsplit("/", 1)[-1], + ) return relative, facts, captured @@ -723,6 +730,9 @@ def _candidate_still_offered(resolution: dict, choice: str) -> bool: that rewords the top-level reason or reorders candidates leaves a pending item applicable. """ + if isinstance(resolution, dict) and resolution.get("anchor") == "saver": + # Only the saver anchor resolves the bytes (#390): its pair alone. + resolution = {"state": "ambiguous", "candidates": [resolution]} if not isinstance(resolution, dict) or resolution.get("state") != "ambiguous": return False evidence, _count, limited = review_evidence(resolution) diff --git a/py/api/review_recording.py b/py/api/review_recording.py index 6de59641..141cf14a 100644 --- a/py/api/review_recording.py +++ b/py/api/review_recording.py @@ -88,7 +88,8 @@ class ReviewRecordingMixin: def _record_scanned_review(self, media_file, roots, review=None, scanned=None): """Record one ambiguous, unlinked scanned file (logged and counted). - ``scanned`` is the scan's ``(metadata, inspection_resolution)``. + ``scanned`` is the scan's ``(metadata, inspection_resolution)``; the + resolution is None when the scan did not run the inspection profile. """ try: outcome = self._record_review_sync( @@ -199,7 +200,7 @@ class ReviewRecordingMixin: key = _memory_key(locator) if state is not None and memory.get(key) == state: return None - resolution = review_capture.inspection_choice(canonical) + resolution = review_capture.inspection_choice(canonical, media_file.name) if resolution is not None: memory.pop(key, None) return canonical, resolution diff --git a/tests/test_api_review.py b/tests/test_api_review.py index 6eeaa89d..0e19c6ed 100644 --- a/tests/test_api_review.py +++ b/tests/test_api_review.py @@ -1040,8 +1040,8 @@ class TestApplyRefusals(ReviewRoutesTestCase): _path, item, detail, observed, cat = await self._opened() real = review_capture_module.resolve_prompt_pair - def reworded(metadata): - resolution = real(metadata) + def reworded(metadata, **kwargs): + resolution = real(metadata, **kwargs) return { **resolution, "reason": "a reworded reason from a newer resolver", @@ -1065,8 +1065,8 @@ class TestApplyRefusals(ReviewRoutesTestCase): _path, item, detail, observed, cat = await self._opened() real = review_capture_module.resolve_prompt_pair - def without_cat(metadata): - resolution = real(metadata) + def without_cat(metadata, **kwargs): + resolution = real(metadata, **kwargs) kept = [ c for c in resolution["candidates"] diff --git a/tests/test_review_inspection.py b/tests/test_review_inspection.py index bec27c2a..ea0774a1 100644 --- a/tests/test_review_inspection.py +++ b/tests/test_review_inspection.py @@ -298,19 +298,21 @@ class TestQwenEditBatchIsQueued(InspectionTestCase): class TestDefaultProfileStillDecides(InspectionTestCase): async def test_ambiguous_file_with_texts_keeps_the_default_profile(self): - """A default-ambiguous file with a choice never runs the inspection - profile, even when that profile would offer more candidates.""" + """A default-ambiguous file with a choice is recorded from the + default profile, even when the inspection profile would offer more + candidates. That profile only checks the saver anchor (#390): no + saver names this file, so it anchors nothing.""" self._roots() graph = { **_prefixed(qwen_edit_graph("make it night"), "q"), **unlinkable_graph("cat", "dog"), } - self._write(graph=graph) + self._write("unsaved_00001_.png", graph=graph) with patch.object( review_capture, - "resolve_inspection", - side_effect=AssertionError("inspection profile ran"), + "inspection_choice", + side_effect=AssertionError("inspection choices were asked"), ): await self._scan() @@ -321,17 +323,24 @@ class TestDefaultProfileStillDecides(InspectionTestCase): self.assertEqual(item["resolver_version"], review_capture.RESOLVER_VERSION) self.assertEqual(_texts(evidence), ["cat", "dog"]) - async def test_apply_of_a_default_item_never_runs_the_inspection_profile(self): + async def test_apply_of_a_default_item_inspects_only_for_the_anchor( + self, + ): self._roots() write_png(self.output_dir / "two.png", prompt=unlinkable_graph("cat", "dog")) await self._scan() - with patch.object( - review_capture, - "resolve_inspection", - side_effect=AssertionError("inspection profile ran"), + with ( + _counted(review_capture, "resolve_inspection") as calls, + patch.object( + review_capture, + "inspection_choice", + side_effect=AssertionError("inspection choices were asked"), + ), ): _item, (status, data) = await self._apply("cat") self.assertEqual(status, 200, data) + # The saver anchor check (#390): with the file's name, nothing matches. + self.assertEqual([c[1] for c in calls], ["two.png"]) self.assertEqual(self._linked_text(self.output_dir / "two.png"), "cat") @@ -369,15 +378,16 @@ class TestApplyInspectionCandidate(InspectionTestCase): calls = [] real = review_capture.resolve_inspection - def spy(metadata): - calls.append(metadata) - return real(metadata) + def spy(metadata, file_name=None): + calls.append((metadata, file_name)) + return real(metadata, file_name) with patch.object(review_capture, "resolve_inspection", side_effect=spy): _item, (status, data) = await self._apply("make it night") self.assertEqual(status, 200, data) self.assertEqual(len(calls), 1) + self.assertEqual(calls[0][1], NAME) # anchored on its saver (#390) async def test_table_change_keeps_the_item_applicable(self): """A table change for a class the graph uses moves the item's @@ -853,8 +863,8 @@ class TestInspectionResolverVersion(InspectionTestCase): version = _version() real = review_capture.resolve_inspection - def newer(metadata): - resolution = real(metadata) + def newer(metadata, file_name=None): + resolution = real(metadata, file_name) return { **resolution, "candidates": [ @@ -899,10 +909,10 @@ class TestScanContainment(InspectionTestCase): self._write("b.png", graph=qwen_edit_graph("snow", "rain")) real = review_capture.resolve_inspection - def fails_for_a(metadata): + def fails_for_a(metadata, file_name=None): if "make it night" in json.dumps(metadata): raise RuntimeError("inspection blew up") - return real(metadata) + return real(metadata, file_name) with ( patch.object(review_capture, "resolve_inspection", side_effect=fails_for_a), diff --git a/tests/test_review_saver_anchor.py b/tests/test_review_saver_anchor.py new file mode 100644 index 00000000..c61d8112 --- /dev/null +++ b/tests/test_review_saver_anchor.py @@ -0,0 +1,333 @@ +"""The scan and Needs review anchor a file on the saver that wrote it (#390). + +The workspace inspector (since #383) passes the file's name to the resolver, +which anchors the file on the one saver whose ``filename_prefix`` names it +(``_GraphResolver.saver_stages``). The scan and Needs review use the same +rule. Maintainer ruling: a file never falls through both the link and the +queue, so when only that anchor resolves a file the default profile and the +parser floor do not link, it is queued with the anchor's pair as its one +choice (never linked automatically). No match, several matches or an +unfinished walk anchor nothing, so such a file is queued as before. Apply +re-resolves with the file's name, so a choice the anchor contradicts is +refused as stale evidence. +""" + +import json +from unittest.mock import patch + +try: + from tests.test_review import unlinkable_graph, write_png + from tests.test_review_inspection import ( + PROMPTS, + _counted, + InspectionTestCase, + qwen_edit_graph, + ) +except ImportError: # discovered with tests/ as the top-level directory + from test_review import unlinkable_graph, write_png + from test_review_inspection import ( + PROMPTS, + _counted, + InspectionTestCase, + qwen_edit_graph, + ) + +from utils import review_capture + + +def branch_graph(*prompts, prefixes=None): + """A Qwen-edit batch whose every SaveImage has its own literal prefix.""" + graph = qwen_edit_graph(*prompts) + savers = [n for n in graph.values() if n["class_type"] == "SaveImage"] + names = prefixes or [f"branch_{index}" for index in range(len(savers))] + for saver, prefix in zip(savers, names): + saver["inputs"]["filename_prefix"] = prefix + return graph + + +def saved_unlinkable_graph(*positives): + """``unlinkable_graph`` whose branches save under their own prefix. + + The default profile finds it ambiguous with a choice and the scan's + browser-parser floor links nothing, so it is queued from the default + profile unless the saver that wrote the file anchors its branch. + """ + graph = unlinkable_graph(*positives) + samplers = [ + node_id + for node_id, node in graph.items() + if node["class_type"] == "KSampler" and node_id != "3" + ] + for index, sampler in enumerate(samplers): + graph[f"{sampler}1"] = { + "class_type": "VAEDecode", + "inputs": {"samples": [sampler, 0], "vae": ["1", 2]}, + } + graph[f"{sampler}2"] = { + "class_type": "SaveImage", + "inputs": { + "images": [f"{sampler}1", 0], + "filename_prefix": f"side_{index}", + }, + } + return graph + + +def _without_name(real): + """``resolve_inspection`` as it was before #390: the file name ignored.""" + + def unnamed(metadata, file_name=None): + return real(metadata) + + return unnamed + + +def _only_candidate(test, item): + """The positive text of an anchored item's one candidate.""" + evidence = json.loads(item["evidence"]) + test.assertEqual(evidence.get("anchor"), "saver") + test.assertEqual(evidence.get("profile"), "inspection") + (candidate,) = evidence["candidates"] + return candidate["positive"]["text"] + + +class TestInspectionItemsAnchor(InspectionTestCase): + async def test_file_its_saver_names_is_queued_with_the_anchor_choice(self): + self._roots() + path = self._write("branch_1_00001_.png", graph=branch_graph(*PROMPTS)) + + job = await self._scan() + + (item,) = self._items() + self.assertEqual(_only_candidate(self, item), "add a red hat") + self.assertEqual(item["candidate_count"], 1) + self.assertEqual( + item["resolver_version"], + review_capture.inspection_review_version( + {"prompt": branch_graph(*PROMPTS)} + ), + ) + (row,) = job["outcomes"]["review"] + self.assertEqual(row["path"], "branch_1_00001_.png") + # Queued, never linked: ingestion stays on the default profile. + self.assertIsNone(self._linked_text(path)) + + async def test_anchored_item_stays_on_rescan(self): + self._roots() + self._write("branch_1_00001_.png", graph=branch_graph(*PROMPTS)) + await self._scan() + (item,) = self._items() + + job = await self._scan("all") + + self.assertEqual([i["id"] for i in self._items()], [item["id"]]) + self.assertEqual(job["scope"]["review_untraced"], 0) + + async def test_file_no_saver_names_stays_queued(self): + self._roots() + self._write("elsewhere_00001_.png", graph=branch_graph(*PROMPTS)) + + await self._scan() + + (item,) = self._items() + self.assertEqual(item["candidate_count"], len(PROMPTS)) + self.assertNotIn("anchor", json.loads(item["evidence"])) + + async def test_file_several_savers_name_stays_queued(self): + self._roots() + graph = branch_graph(*PROMPTS, prefixes=["same", "same", "other"]) + self._write("same_00001_.png", graph=graph) + + await self._scan() + + (item,) = self._items() + self.assertEqual(item["candidate_count"], len(PROMPTS)) + + async def test_old_rule_item_is_replaced_by_the_anchor_choice(self): + self._roots() + self._write("branch_1_00001_.png", graph=branch_graph(*PROMPTS)) + real = review_capture.resolve_inspection + with ( + patch.object(review_capture, "REVIEW_RULE_VERSION", 0), + patch.object( + review_capture, "resolve_inspection", side_effect=_without_name(real) + ), + ): + await self._scan() + (old,) = self._items() # queued under the old rule with every branch + self.assertEqual(old["candidate_count"], len(PROMPTS)) + + await self._scan("all") + + (item,) = self._items() + self.assertNotEqual(item["id"], old["id"]) + self.assertEqual(_only_candidate(self, item), "add a red hat") + + async def test_apply_links_the_anchor_choice(self): + self._require_routes() + self._roots() + path = self._write("branch_1_00001_.png", graph=branch_graph(*PROMPTS)) + await self._scan() + + _item, (status, data) = await self._apply("add a red hat") + + self.assertEqual(status, 200, data) + self.assertEqual(self._linked_text(path), "add a red hat") + + async def test_apply_refuses_an_old_choice_the_anchor_contradicts(self): + self._require_routes() + self._roots() + path = self._write("branch_1_00001_.png", graph=branch_graph(*PROMPTS)) + real = review_capture.resolve_inspection + with ( + patch.object(review_capture, "REVIEW_RULE_VERSION", 0), + patch.object( + review_capture, "resolve_inspection", side_effect=_without_name(real) + ), + ): + await self._scan() + item, detail, observed = await self._open() + # branch_1 is the saver of "add a red hat"; choose another branch. + wrong = self._candidate(detail, "make it night") + + status, data = await self._decide( + item["id"], self._body(detail, observed, wrong["hash"]) + ) + + self.assertEqual(status, 409, data) + self.assertEqual(data["code"], "review-conflict", data) + self.assertIsNone(self._linked_text(path)) + + +class TestDefaultItemsAnchor(InspectionTestCase): + async def test_anchor_resolves_and_nothing_links_so_it_is_queued(self): + """Maintainer ruling: never unlinked and unqueued. The anchor resolves + the file, the default profile and the parser floor link nothing, so + it is queued with the anchor's pair as its choice.""" + self._roots() + path = self.output_dir / "side_0_00001_.png" + graph = saved_unlinkable_graph("cat", "dog") + write_png(path, prompt=graph) + self.assertEqual( + review_capture.resolve_prompt_pair({"prompt": graph})["state"], "ambiguous" + ) + + job = await self._scan() + + (item,) = self._items() + self.assertEqual(_only_candidate(self, item), "cat") + (row,) = job["outcomes"]["review"] + self.assertIn("more than one prompt could match", row["detail"]) + self.assertIsNone(self._linked_text(path)) + + async def test_default_ambiguous_file_no_saver_names_stays_queued(self): + self._roots() + write_png( + self.output_dir / "unsaved_00001_.png", + prompt=saved_unlinkable_graph("cat", "dog"), + ) + + await self._scan() + + (item,) = self._items() + evidence = json.loads(item["evidence"]) + self.assertNotIn("profile", evidence) + self.assertNotIn("anchor", evidence) + self.assertEqual(item["resolver_version"], review_capture.RESOLVER_VERSION) + + async def test_file_the_parser_floor_links_is_not_queued(self): + """Already linked by the default path: unchanged by the anchor.""" + self._roots() + graph = saved_unlinkable_graph("cat", "dog") + del graph["3"] # the parser floor now reads a real sampler + path = self.output_dir / "side_0_00001_.png" + write_png(path, prompt=graph) + + await self._scan() + + self.assertEqual(self._items(), []) + self.assertIsNotNone(self._linked_text(path)) + + async def test_apply_refuses_an_old_default_choice_the_anchor_contradicts(self): + self._require_routes() + self._roots() + path = self.output_dir / "side_0_00001_.png" + write_png(path, prompt=saved_unlinkable_graph("cat", "dog")) + with patch.object(review_capture, "_captured_inspection", return_value=None): + await self._scan() # queued from the default profile, as before #390 + item, detail, observed = await self._open() + # side_0 is the saver of "cat"; choose the other branch. + wrong = self._candidate(detail, "dog") + + status, data = await self._decide( + item["id"], self._body(detail, observed, wrong["hash"]) + ) + + self.assertEqual(status, 409, data) + self.assertEqual(data["code"], "review-conflict", data) + self.assertIsNone(self._linked_text(path)) + + async def test_old_default_item_is_replaced_by_the_anchor_choice(self): + self._roots() + write_png( + self.output_dir / "side_0_00001_.png", + prompt=saved_unlinkable_graph("cat", "dog"), + ) + with patch.object(review_capture, "_captured_inspection", return_value=None): + await self._scan() # queued from the default profile, as before #390 + (old,) = self._items() + self.assertEqual(old["resolver_version"], review_capture.RESOLVER_VERSION) + + await self._scan("all") + + (item,) = self._items() + self.assertNotEqual(item["id"], old["id"]) + self.assertEqual(_only_candidate(self, item), "cat") + + with _counted(review_capture, "resolve_inspection") as calls: + await self._scan("all") # now queued under its version: no re-read + self.assertEqual(calls, []) + self.assertEqual([i["id"] for i in self._items()], [item["id"]]) + + async def test_decided_anchored_item_is_kept(self): + self._require_routes() + self._roots() + write_png( + self.output_dir / "side_0_00001_.png", + prompt=saved_unlinkable_graph("cat", "dog"), + ) + await self._scan() + item, detail, observed = await self._open() + status, data = await self._decide( + item["id"], self._body(detail, observed, "unresolved") + ) + self.assertEqual(status, 200, data) + before = (self._items(), self._decisions()) + + await self._scan("all") + + self.assertEqual((self._items(), self._decisions()), before) + + async def test_apply_accepts_an_old_default_choice_the_anchor_names(self): + self._require_routes() + self._roots() + path = self.output_dir / "side_0_00001_.png" + write_png(path, prompt=saved_unlinkable_graph("cat", "dog")) + with patch.object(review_capture, "_captured_inspection", return_value=None): + await self._scan() + + _item, (status, data) = await self._apply("cat") + + self.assertEqual(status, 200, data) + self.assertEqual(self._linked_text(path), "cat") + + +class TestSaverAnchored(InspectionTestCase): + def test_only_the_anchor_counts(self): + anchored = {"prompt": saved_unlinkable_graph("cat", "dog")} + self.assertTrue(review_capture.saver_anchored(anchored, "side_1_00001_.png")) + self.assertFalse(review_capture.saver_anchored(anchored, "nope_00001_.png")) + self.assertFalse(review_capture.saver_anchored(anchored, None)) + # One sampler resolves without any name: the anchor decides nothing. + single = {"prompt": branch_graph("make it night")} + self.assertFalse(review_capture.saver_anchored(single, "branch_0_00001_.png")) diff --git a/utils/review_capture.py b/utils/review_capture.py index 1859dee7..8fadab72 100644 --- a/utils/review_capture.py +++ b/utils/review_capture.py @@ -40,6 +40,19 @@ while it is pending and undecided (never beside a decision, see profile the item recorded (``captured_review(..., item_profile(evidence))``). The inspection profile only ever produces review choices; ingestion's own links stay on the default profile, which never loads the table (D1). + +The saver anchor (#390): the inspection profile resolves a file with its +name, as the workspace inspector does since #383, so the one saver whose +``filename_prefix`` names the file anchors its branch. When that anchor alone +resolves a file the default profile left ambiguous and nothing links +(``saver_anchored``), the scan queues it with the anchor's pair as its one +candidate (evidence ``anchor: "saver"``), never linking it: a file never +falls through both the link and the queue (maintainer ruling). No match, +several matches or an unfinished walk anchor nothing and the file is queued +as before. Apply re-resolves with the file's name, so only the anchored +branch can be applied. ``REVIEW_RULE_VERSION`` is folded into inspection +item versions, so inspection items recorded before the anchor are recorded +again by the next scan. """ from __future__ import annotations @@ -104,10 +117,16 @@ MAX_CANDIDATE_DEPTH = 8 # The resolver profile an item's evidence was built with (``profile``). DEFAULT = "default" INSPECTION = "inspection" +# The ``anchor`` an item recorded from the saver anchor names (#390). +SAVER_ANCHOR = "saver" +ANCHOR_REASON = "The saver that wrote this file names its branch" # Inspection item versions: above every plausible ``RESOLVER_VERSION`` and, # with 40 bits of the folded digest, below 2**53 (exact as a JSON number). INSPECTION_VERSION_BASE = 1 << 40 _INSPECTION_VERSION_HEX = 10 +# How the scan and Apply use the inspection profile, folded into inspection +# item versions. 1: they resolve with the file's name (the saver anchor, #390). +REVIEW_RULE_VERSION = 1 # Above this many distinct node classes the whole table is the fingerprint. @@ -154,38 +173,92 @@ def inspection_review_version(metadata: Any = None) -> int | None: """The ``resolver_version`` of an item recorded from the inspection profile. Folds the profile with its budgets, ``RESOLVER_VERSION``, - ``INSPECTION_RESOLVER_VERSION`` and ``semantics_version(metadata)``, so - a change of either resolver, of the profile's bounds or of the table - about the graph's classes moves it. None when the table cannot be loaded. + ``INSPECTION_RESOLVER_VERSION``, ``REVIEW_RULE_VERSION`` and + ``semantics_version(metadata)``, so a change of either resolver, of how + review uses the profile, of the profile's bounds or of the table about + the graph's classes moves it. None when the table cannot be loaded. """ table = semantics_version(metadata) if table is None: return None profile = repr(astuple(INSPECTION_PROFILE)) seed = f"{profile}:{RESOLVER_VERSION}:{INSPECTION_RESOLVER_VERSION}:{table}" + if REVIEW_RULE_VERSION: # 0 is the seed items had before the rule existed + seed += f":review-rule-{REVIEW_RULE_VERSION}" folded = hashlib.sha256(seed.encode("utf-8")).hexdigest() return INSPECTION_VERSION_BASE + int(folded[:_INSPECTION_VERSION_HEX], 16) -def resolve_inspection(metadata: dict) -> dict: - """``metadata`` resolved with the inspection profile (bounded work).""" - return resolve_prompt_pair(metadata, profile=INSPECTION_PROFILE) +def resolve_inspection(metadata: dict, file_name: str | None = None) -> dict: + """``metadata`` resolved with the inspection profile (bounded work). + + ``file_name`` (the file's last path part) anchors the file on the one + saver that wrote it, as in the workspace inspector (#390). + """ + return resolve_prompt_pair( + metadata, profile=INSPECTION_PROFILE, file_name=file_name + ) -def inspection_choice(metadata: dict) -> dict | None: - """The inspection profile's ambiguous result when it offers a choice. +def inspection_choice(metadata: dict, file_name: str | None = None) -> dict | None: + """The inspection profile's result when the file is a review case. The scan asks only for a file the default profile found ambiguous - without a choice that nothing links. + without a choice that nothing links. The result is ambiguous with a + choice, or ``resolved`` when ``file_name`` anchors the file on its saver + and only that anchor resolves it (``saver_anchored``): the item then + offers the anchor's pair. None otherwise. """ if inspection_review_version(metadata) is None: return None - resolution = resolve_inspection(metadata) + resolution = resolve_inspection(metadata, file_name) + if resolution.get("state") == "resolved": + return _anchor_of(metadata, file_name, resolution) if resolution.get("state") != "ambiguous" or not reviewable(resolution): return None return resolution +def _anchor_of(metadata: Any, file_name: str | None, named: dict) -> dict | None: + """``named`` tagged ``anchor: "saver"`` when only the saver anchor + resolves the file, to a pair with a positive text to choose; else None. + + ``named`` is ``metadata`` resolved with ``file_name``; the profile runs + once more without the name only when it is resolved and choosable. + """ + if not file_name or named.get("state") != "resolved": + return None + try: + offered = choosable(_projection(named)) + except (KeyError, TypeError): + return None + if not offered or resolve_inspection(metadata).get("state") == "resolved": + return None + return {**named, "anchor": SAVER_ANCHOR} + + +def saver_anchored(metadata: Any, file_name: str | None) -> bool: + """True when the saver that wrote ``file_name`` resolves its branch (#390). + + The inspection profile resolves ``metadata`` with the file's name, and + without it the file does not resolve: the anchor alone decides it. A file + the default profile left ambiguous is then queued with the anchor's pair + as its one candidate. No name, an unusable table, no matching saver, + several, or an unfinished walk anchor nothing (False). The profile + without the name runs only when the named run resolved. + """ + return _anchored(metadata, file_name) is not None + + +def _anchored(metadata: Any, file_name: str | None) -> dict | None: + """The anchored result ``saver_anchored`` describes, else None.""" + if not file_name or not isinstance(metadata, dict): + return None + if inspection_review_version(metadata) is None: + return None + return _anchor_of(metadata, file_name, resolve_inspection(metadata, file_name)) + + def item_profile(evidence: Any) -> str: """The profile an item's stored evidence was built with.""" if isinstance(evidence, dict) and evidence.get("profile") == INSPECTION: @@ -246,7 +319,9 @@ def record_ambiguous_file( """ versions = item_versions(scanned[0] if scanned is not None else None) spelled_path, queued = _queued_spelling(db, root, relative, versions) - if queued is not None: + if queued is not None and not ( + inspection and _default_item_to_anchor(db, queued, scanned, relative) + ): return _already_queued(db, *queued, versions) epoch = db.get_execution_epoch() try: @@ -272,24 +347,24 @@ def record_ambiguous_file( if resolution is None or resolution.get("state") != "ambiguous": return "skipped" profile, version = DEFAULT, RESOLVER_VERSION - if not reviewable(resolution): - version = inspection_review_version(captured[0]) if inspection else None - if version is None: + named = _captured_inspection(captured[0], relative, scanned) if inspection else None + anchored = named is not None and named[0].get("anchor") == SAVER_ANCHOR + if anchored: + # Only the saver anchor resolves it (#390): offer that one pair. + profile, version = INSPECTION, named[1] + resolution = { + "state": "ambiguous", + "reason": ANCHOR_REASON, + "candidates": [named[0]], + } + elif not reviewable(resolution): + if named is None: return "untraced" - profile = INSPECTION - if ( - scanned is not None - and scanned[1] is not None - and _same_records(captured[0], scanned[0]) - ): - resolution = scanned[1] # the same records: no second run - else: - # Heavier, so only for a file ambiguous with nothing to choose. - resolution = resolve_inspection(captured[0]) + profile, (resolution, version) = INSPECTION, named if resolution.get("state") != "ambiguous" or not reviewable(resolution): return "untraced" evidence, count, limited = review_evidence( - resolution, profile, version, captured[0] + resolution, profile, version, captured[0], anchored=anchored ) try: inserted = db.record_review_item( @@ -313,6 +388,60 @@ def record_ambiguous_file( return "limited" if limited else "recorded" +def _default_item_to_anchor(db, queued: tuple, scanned, relative: str) -> bool: + """True when the bytes' only pending item is a default item recorded + before #390 and the saver anchor alone resolves the scanned records. + + The file is then captured again so the anchored item supersedes that + pending, undecided item. Decided bytes and bytes already queued under + the current inspection version are never re-read. + """ + if scanned is None or scanned[0] is None: + return False + version = inspection_review_version(scanned[0]) + if version is None or db.review_pending_at(*queued, version): + return False + if not db.review_pending_at(*queued, RESOLVER_VERSION): + return False + return _anchored(scanned[0], _file_name(relative)) is not None + + +def _captured_inspection(metadata: dict, relative: str, scanned) -> tuple | None: + """``(resolution, item_version)`` of the captured records under the + inspection profile, anchored on the file's name; None when the table is + unusable. + + A result tagged ``anchor: "saver"`` means the saver anchor alone + resolves the file (#390); any other resolved result is returned as + ``unresolved`` (no review case, as before). The scan's result is reused + when the capture holds the same records (``inspection_choice`` applies + the same rule). + """ + version = inspection_review_version(metadata) + if version is None: + return None + if ( + scanned is not None + and scanned[1] is not None + and _same_records(metadata, scanned[0]) + ): + return scanned[1], version # the same records: no second run + name = _file_name(relative) + resolution = resolve_inspection(metadata, name) + if resolution.get("state") == "resolved": + # Resolved without the anchor too: no review case, as before. + resolution = _anchor_of(metadata, name, resolution) or { + **resolution, + "state": "unresolved", + } + return resolution, version + + +def _file_name(relative: str) -> str: + """The last part of a ``/``-separated relative path (the saved name).""" + return relative.rsplit("/", 1)[-1] + + def _already_queued(db, locator: str, size: int, mtime_ns: int, versions) -> str: """``duplicate`` while an existing item is pending, else ``decided``.""" for version in versions: @@ -354,14 +483,19 @@ def _queued_spelling(db, root, relative: str, versions) -> tuple[str, tuple | No return spelled, None -def captured_review(store, capture, profile: str = DEFAULT) -> tuple[dict, dict] | None: +def captured_review( + store, capture, profile: str = DEFAULT, file_name: str | None = None +) -> tuple[dict, dict] | None: """``(metadata, resolution)`` of the capture's own bytes, else None. ``metadata`` is what the scan hands the ingester: prompt and workflow JSON-decoded and ``file_info`` (size, dimensions, format) from the capture. ``resolution`` comes from ``profile`` (``DEFAULT`` or - ``INSPECTION``). None when the capture is gone or carries no prompt - metadata. + ``INSPECTION``; the latter anchored on ``file_name``, the file's own + name, as the item was recorded). With ``file_name`` (Apply), a default + result its saver anchor alone resolves (``saver_anchored``) is that + anchored result, so only the anchored branch is still offered. None + when the capture is gone or carries no prompt metadata. """ if store.acquire(capture.token) is None: return None @@ -389,8 +523,15 @@ def captured_review(store, capture, profile: str = DEFAULT) -> tuple[dict, dict] info["format"] = read.format canonical["file_info"] = info if profile == INSPECTION: - return canonical, resolve_inspection(canonical) - return canonical, resolve_prompt_pair(canonical) + named = resolve_inspection(canonical, file_name) + return canonical, _anchor_of(canonical, file_name, named) or named + resolution = resolve_prompt_pair(canonical) + if file_name is not None and resolution.get("state") == "ambiguous": + anchored = _anchored(canonical, file_name) + if anchored is not None: + # Its saver names its branch (#390): only that one is offered. + return canonical, anchored + return canonical, resolution def reviewable(resolution: Any) -> bool: @@ -416,12 +557,14 @@ def review_evidence( profile: str = DEFAULT, version: int | None = None, metadata: Any = None, + anchored: bool = False, ) -> tuple[dict, int, bool]: """``(evidence, candidate_count, limited)`` for one ambiguous result. Inspection evidence also names its ``profile`` and ``semantics``, the table fingerprint of ``metadata``'s graph it was resolved with; - ``version`` is the item's version. + ``version`` is the item's version. ``anchored`` evidence (the saver + anchor's one pair, #390) also names ``anchor: "saver"``. """ base = { "resolver_version": RESOLVER_VERSION if version is None else version, @@ -430,6 +573,8 @@ def review_evidence( if profile == INSPECTION: base["profile"] = INSPECTION base["semantics"] = semantics_version(metadata) + if anchored: + base["anchor"] = SAVER_ANCHOR candidates: list[dict] = [] seen: set[str] = set() total = 0 From 95c4542c6f09bc9b398460fc667a59fb2310b8c5 Mon Sep 17 00:00:00 2001 From: Vito <vito.sansevero@gmail.com> Date: Wed, 28 Oct 2026 08:18:06 +0000 Subject: [PATCH 10/11] test(browser): API restores stay unseen by the open page's Activity poll (#413) (#416) * test(browser): API restores stay unseen by the open page's Activity poll (#413) Specs that restore through the API while a workspace page is open check how the page handles a restore it did not see. The page's Activity store reads the job list every 15 s; on a slow runner (CI run 40646112668, a 16 s restore) that read landed during the restore, the page followed the job and, as designed (S9), showed the open page again when it finished, closing the Create tag dialog the vocabulary spec was about to use. unseenRestore holds the page's job-list reads until the restore has answered, so they read it as already ended. Used by the vocabulary, library editor and history specs. * test(browser): arm the restore gate before the page loads and drain reads in flight (#413) A job-list read the page sent before the restore could still be answered while it ran. The gate now routes every job-list read from before the first navigation and, before a restore, waits for the reads in flight to be answered. The history and library editor conversions are pre-emptive: same race, never seen failing. --- tests/browser/unseen-restore.mjs | 54 +++++++++++++++++++ tests/browser/workspace-history.spec.mjs | 15 +++--- .../browser/workspace-library-editor.spec.mjs | 10 ++-- tests/browser/workspace-vocabulary.spec.mjs | 10 ++-- 4 files changed, 74 insertions(+), 15 deletions(-) create mode 100644 tests/browser/unseen-restore.mjs diff --git a/tests/browser/unseen-restore.mjs b/tests/browser/unseen-restore.mjs new file mode 100644 index 00000000..aabd8962 --- /dev/null +++ b/tests/browser/unseen-restore.mjs @@ -0,0 +1,54 @@ +// A restore the open workspace page has not noticed yet (#413). +// +// Specs that restore through the API while a page is open check what the page does with a restore it did not see: +// its next write is refused as "Prompt storage was restored since this dialog opened". But the page's Activity store +// reads the job list every 15 s (activity-store.mjs POLL_MS), and a read the server answers while the restore job runs +// makes the page follow it: when it finishes, the page is shown again from the restored data (S9, index.mjs +// onRestoreFinished), which closes any open dialog. On a slow runner a restore can take longer than the time left to +// the next read (CI run 40646112668: the restore took 16 s), so whether the page saw it depended on timing. +// +// `armUnseenRestores(page)`, before the page loads, passes every job-list read of the page through one route, so it +// knows which reads the server has not answered yet. `unseenRestore(page, restoring)` then closes the gate, waits +// for those reads' answers (sent before the restore began, they cannot list it), runs `restoring()`, and only after +// it has answered lets the held reads go on: they read the restore as already ended, as a tab that read its job list +// just before and just after the restore does. A restore that is never seen running never reloads the page. +const JOBS = '**/prompt_manager/jobs'; +const gates = new WeakMap(); + +/** Route `page`'s job-list reads through the restore gate. Call before the page's first navigation. */ +export async function armUnseenRestores(page) { + const gate = { closed: null, inFlight: 0, idle: [] }; + gates.set(page, gate); + await page.route(JOBS, async route => { + if (gate.closed) { + await gate.closed; + return route.fallback(); + } + gate.inFlight += 1; + try { + const response = await route.fetch(); + await route.fulfill({ response }).catch(() => {}); + } catch { + await route.abort().catch(() => {}); + } finally { + gate.inFlight -= 1; + if (gate.inFlight === 0) for (const resolve of gate.idle.splice(0)) resolve(); + } + }); +} + +/** Run `restoring()` (an API restore) unseen by `page`'s Activity store; resolves with what it resolves with. */ +export async function unseenRestore(page, restoring) { + const gate = gates.get(page); + if (!gate) throw new Error('armUnseenRestores(page) must run before the page loads'); + let open; + gate.closed = new Promise(resolve => { open = resolve; }); + try { + // Reads already on their way were sent before the restore: their answers cannot list it. + if (gate.inFlight > 0) await new Promise(resolve => gate.idle.push(resolve)); + return await restoring(); + } finally { + gate.closed = null; + open(); + } +} diff --git a/tests/browser/workspace-history.spec.mjs b/tests/browser/workspace-history.spec.mjs index 48f81074..f912a892 100644 --- a/tests/browser/workspace-history.spec.mjs +++ b/tests/browser/workspace-history.spec.mjs @@ -1,5 +1,6 @@ import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs'; import {test,expect} from './fixtures.mjs'; +import {armUnseenRestores,unseenRestore} from './unseen-restore.mjs'; import {readFile,writeFile} from 'node:fs/promises'; import {createHash} from 'node:crypto'; @@ -16,13 +17,13 @@ const marker=(page,id,anchor='10')=>recorded(page).locator(`button[data-run-id=" const exact=async locator=>locator.evaluate(node=>node.textContent); test.beforeEach(async({page,request,browserName})=>{ - await startCoverage(page,browserName);page.historyErrors=[];page.on('pageerror',error=>page.historyErrors.push(error.message)); + await startCoverage(page,browserName);await armUnseenRestores(page);page.historyErrors=[];page.on('pageerror',error=>page.historyErrors.push(error.message)); const reset=await request.post('/__test__/reset',{data:{chronological_history:true}});await expect(reset).toBeOK();page.fixture=await reset.json(); }); test.afterEach(async({page,browserName})=>{await saveCoverage(page,browserName);expect(page.historyErrors).toEqual([]);}); const command=async(request,data)=>{const response=await request.post('/__test__/history/commands',{data});await expect(response).toBeOK();return response.json();}; -const restoreSnapshot=async(request,bytes)=>{const response=await request.post('/prompt_manager/restore',{multipart:{database_file:{name:'history.db',mimeType:'application/octet-stream',buffer:bytes}}});await expect(response).toBeOK();expect((await response.json()).success).toBe(true);}; +const restoreSnapshot=async(page,request,bytes)=>{const response=await unseenRestore(page,()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'history.db',mimeType:'application/octet-stream',buffer:bytes}}}));await expect(response).toBeOK();expect((await response.json()).success).toBe(true);}; const historyRead=async(request,id)=>{const response=await request.get(`/prompt_manager/library/${id}/history`);await expect(response).toBeOK();return response.json();}; const frozenContexts=contexts=>contexts.map(({navigation,positive_prompt_id,negative_prompt_id,pair_id,...frozen})=>frozen); const immutableReceipts=items=>items.map(({current_candidate,...receipt})=>receipt); @@ -62,7 +63,7 @@ for(const boundary of ['window','detail','candidate','inspection','media']) test await command(request,{action:'arm',boundary});const pending=request.get(target); try { await expect.poll(async()=> (await(await request.get('/__test__/history/checkpoint')).json()).entered).toBe(true); - await restoreSnapshot(request,bytes); + await restoreSnapshot(page,request,bytes); } finally {await command(request,{action:'release'});} const retired=await pending;expect(retired.status()).toBe(409);expect((await retired.json()).code).toBe('history-changed'); for(const url of [`/prompt_manager/library/${f.detail_id}/history?limit=1&cursor=${encodeURIComponent(firstWindow.next_cursor)}`,`${base}?epoch=${epoch}`,`${base}/outputs?epoch=${epoch}`,`${base}/outputs?epoch=${epoch}&cursor=${encodeURIComponent(outputs.next_cursor)}`,media.url,media.thumbnail_url,media.inspection_url]) { @@ -130,7 +131,7 @@ test('archive delete merge and prune preserve frozen evidence without repairing test('actual restore with reused numeric owner rejects stale history and offers explicit saved recovery',async({page,request})=>{ const f=page.fixture;await open(page,f.detail_id);const original=await exact(saved(page).getByRole('region',{name:'Positive prompt',exact:true}).locator('p.prompt-text')); - const source=await request.get('/__test__/history/reused-snapshot');await expect(source).toBeOK();await restoreSnapshot(request,await source.body()); + const source=await request.get('/__test__/history/reused-snapshot');await expect(source).toBeOK();await restoreSnapshot(page,request,await source.body()); await marker(page,f.runs.shared).click();await expect(dialog(page)).toContainText('Recorded evidence changed');await page.keyboard.press('Escape'); await expect(page.locator('.library-parent-status')).toContainText('changed');expect(await exact(saved(page).getByRole('region',{name:'Positive prompt',exact:true}).locator('p.prompt-text'))).toBe(original); await expect(recorded(page).getByRole('button',{name:'Reload recorded history',exact:true})).toBeVisible(); @@ -142,12 +143,12 @@ for(const outcome of ['success','error'])test(`delayed actual old browser ${outc const f=page.fixture;await open(page,f.detail_id);const snapshot=await request.get('/__test__/history/snapshot');const bytes=await snapshot.body(); let entered,release;const started=new Promise(resolve=>{entered=resolve;}),held=new Promise(resolve=>{release=resolve;}); await page.route(new RegExp(`/history/runs/${f.runs.shared}\\?`),async route=>{ - if(outcome==='error')await restoreSnapshot(request,bytes); + if(outcome==='error')await restoreSnapshot(page,request,bytes); const response=await route.fetch();expect(response.status()).toBe(outcome==='success'?200:409);entered();await held; await route.fulfill({response}).catch(()=>{}); }); try { - await marker(page,f.runs.shared).click();await started;if(outcome==='success')await restoreSnapshot(request,bytes); + await marker(page,f.runs.shared).click();await started;if(outcome==='success')await restoreSnapshot(page,request,bytes); await page.evaluate(()=>{location.hash='#/overview';});await expect(page.locator('#workspace-main h1')).toHaveText('Overview');release(); await page.unroute(new RegExp(`/history/runs/${f.runs.shared}\\?`));await open(page,f.detail_id); await expect(recorded(page).locator('button[data-run-id]')).toHaveCount(0); @@ -258,7 +259,7 @@ test('inspection first discovers a real restore and immediately retires every su if(changed)window.recordedRecoveryStates.push({receipts:run.querySelectorAll('[data-receipt]').length,active:[...run.querySelectorAll('[data-send-role],[data-copy-role],[data-view-receipt]')].filter(node=>!node.disabled&&node.checkVisibility()).length}); });observer.observe(run,{subtree:true,childList:true,attributes:true,characterData:true}); }); - await restoreSnapshot(request,await snapshot.body()); + await restoreSnapshot(page,request,await snapshot.body()); } finally {await command(request,{action:'release'});} const response=await inspectionRead;expect(response.status()).toBe(409);expect((await response.json()).code).toBe('history-changed'); await expect(dialog(page).locator('.recorded-run-status')).toContainText('Recorded evidence changed'); diff --git a/tests/browser/workspace-library-editor.spec.mjs b/tests/browser/workspace-library-editor.spec.mjs index cac57771..2cc85b25 100644 --- a/tests/browser/workspace-library-editor.spec.mjs +++ b/tests/browser/workspace-library-editor.spec.mjs @@ -1,6 +1,7 @@ import {writeFile} from 'node:fs/promises'; import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs'; import {test,expect} from './fixtures.mjs'; +import {armUnseenRestores,unseenRestore} from './unseen-restore.mjs'; import {captureCanonicalState} from './history-canonical-comparison.mjs'; import {inventoryOf,mapConcept,diffInventory} from './library-editor-canonical-comparison.mjs'; @@ -13,7 +14,7 @@ const RATINGS=[['5','5 / 5'],['4','4 / 5'],['3','3 / 5'],['2','2 / 5'],['1','1 / const editorDialog=page=>page.locator('dialog.library-editor'); test.beforeEach(async({page,request,browserName})=>{ - await startCoverage(page,browserName); + await startCoverage(page,browserName);await armUnseenRestores(page); const reset=await request.post('/__test__/library-editor/reset',{data:{}});await expect(reset).toBeOK();page.fixture=await reset.json(); page.editorErrors=[];page.on('pageerror',error=>page.editorErrors.push(error.message)); page.writes=[];page.on('request',item=>{if(item.method()!=='GET')page.writes.push({method:item.method(),url:new URL(item.url()).pathname,body:item.postDataJSON()});}); @@ -260,7 +261,8 @@ test('recovery states render inline and never replay a write',async({page,reques page.writes.length=0; dialog=await openEditor(page,{mode:'new'});await ready(dialog); const snapshot=await request.get('/__test__/library-editor/snapshot');await expect(snapshot).toBeOK(); - const restore=await request.post('/prompt_manager/restore',{multipart:{database_file:{name:'editor.db',mimeType:'application/octet-stream',buffer:await snapshot.body()}}}); + const buffer=await snapshot.body(); + const restore=await unseenRestore(page,()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'editor.db',mimeType:'application/octet-stream',buffer}}})); await expect(restore).toBeOK();page.writes.length=0; await field(dialog,'Title').fill('After restore');await field(dialog,'Positive prompt').fill('Restored storage prompt');await button(dialog,'Save').click(); await expect(note(dialog)).toHaveText('Prompt storage was restored since this dialog opened. Nothing was saved.'); @@ -588,7 +590,7 @@ test('saved refresh failure retries only reads',async({page})=>{ test('restored storage after a confirmed save or before an archive offers an explicit Library reload',async({page,request})=>{ const f=page.fixture;const snapshot=await(await request.get('/__test__/library-editor/snapshot')).body(); - const restore=()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'editor.db',mimeType:'application/octet-stream',buffer:snapshot}}}); + const restore=()=>unseenRestore(page,()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'editor.db',mimeType:'application/octet-stream',buffer:snapshot}}})); await openHistory(page,f.positive_id); await page.route(`**/prompt_manager/library/${f.positive_id}`,async route=>{ if(route.request().method()!=='PATCH')return route.continue(); @@ -615,7 +617,7 @@ test('restored storage after a confirmed save or before an archive offers an exp const DISCARDS='Leaving this dialog discards your entries.'; test('restored storage before an Edit or Add negative save offers only the Library reload exit',async({page,request})=>{ const f=page.fixture;const snapshot=await(await request.get('/__test__/library-editor/snapshot')).body(); - const restore=()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'editor.db',mimeType:'application/octet-stream',buffer:snapshot}}}); + const restore=()=>unseenRestore(page,()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'editor.db',mimeType:'application/octet-stream',buffer:snapshot}}})); await openHistory(page,f.positive_id); await page.getByRole('button',{name:'Edit metadata',exact:true}).click();let dialog=editorDialog(page);await ready(dialog); await expect(await restore()).toBeOK(); diff --git a/tests/browser/workspace-vocabulary.spec.mjs b/tests/browser/workspace-vocabulary.spec.mjs index 44248694..b9b82da2 100644 --- a/tests/browser/workspace-vocabulary.spec.mjs +++ b/tests/browser/workspace-vocabulary.spec.mjs @@ -1,6 +1,7 @@ import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs'; import {writeFile} from 'node:fs/promises'; import {test,expect} from './fixtures.mjs'; +import {armUnseenRestores,unseenRestore} from './unseen-restore.mjs'; import {captureCanonicalState} from './history-canonical-comparison.mjs'; import {pageInventoryOf,inventoryOf,diffInventory,mapVocabularyConcept,mapVocabularyDialog} from './vocabulary-canonical-comparison.mjs'; @@ -40,7 +41,7 @@ async function expectLibrary(page,{tags='',category='',count}) { let manifest; test.beforeEach(async({page,request,browserName})=>{ - await startCoverage(page,browserName);manifest=await seed(request); + await startCoverage(page,browserName);await armUnseenRestores(page);manifest=await seed(request); page.vocabularyErrors=[];page.on('pageerror',error=>page.vocabularyErrors.push(error.message)); }); test.afterEach(async({page,browserName})=>{await saveCoverage(page,browserName);expect(page.vocabularyErrors).toEqual([]);}); @@ -264,7 +265,8 @@ const toast=page=>page.locator('#workspace-toast'); const actionNames=scope=>scope.locator('.dialog-actions button').evaluateAll(nodes=>nodes.map(node=>node.textContent.trim())); const snapshot=async request=>{const response=await request.get('/__test__/vocabulary/snapshot');await expect(response).toBeOK();return response.json();}; const backup=async request=>{const response=await request.get('/__test__/vocabulary/backup');await expect(response).toBeOK();return response.body();}; -const restore=async(request,buffer)=>expect(await request.post('/prompt_manager/restore',{multipart:{database_file:{name:'vocabulary.db',mimeType:'application/octet-stream',buffer}}})).toBeOK(); +// A restore the open page has not seen, so its dialog stays to report it (#413). +const restore=async(page,request,buffer)=>expect(await unseenRestore(page,()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'vocabulary.db',mimeType:'application/octet-stream',buffer}}}))).toBeOK(); const tagNames=rows=>rows.tags.map(([,name])=>name); function trackWrites(page) { const writes=[];page.on('request',item=>{if(item.method()!=='GET')writes.push({method:item.method(),url:new URL(item.url()).pathname,body:item.postDataJSON()});});return writes; @@ -482,7 +484,7 @@ test('conflict and restore states keep the dialog and write nothing',async({page const before=await snapshot(request); box=await openDialog(page,main(page).getByRole('button',{name:'+ Create tag',exact:true}),'Create tag'); - await restore(request,saved); + await restore(page,request,saved); await field(box,'Tag name').fill('sunrise');await button(box,'Apply').click(); await expect(note(box)).toHaveText('Prompt storage was restored since this dialog opened. Nothing was changed.'); await expect(button(box,'Apply')).toBeDisabled(); @@ -622,7 +624,7 @@ test('delete confirmation read failure, missing tag and restore keep Delete disa await expect(button(box,'Delete')).toBeDisabled();await expect(tagRow(page,'calm').locator('td:nth-child(2)')).toHaveText('2'); await button(box,'Cancel').click(); - box=await openRename(page,'mist');await restore(request,saved); + box=await openRename(page,'mist');await restore(page,request,saved); await button(box,'Delete tag…').click(); await expect(note(box)).toHaveText('Prompt storage was restored since this dialog opened. Nothing was changed.'); await expect(button(box,'Delete')).toBeDisabled();expect(await actionNames(box)).toEqual(['Reload','Cancel','Delete']); From 65b15bbe758fc1cfa8d00033c53a250f2cb30710 Mon Sep 17 00:00:00 2001 From: Vito <vito.sansevero@gmail.com> Date: Wed, 28 Oct 2026 09:06:15 +0000 Subject: [PATCH 11/11] fix(logs): Clear all logs deletes and empties only what its confirmation showed (#389) (#412) * fix(logs): Clear all logs deletes and empties only what its confirmation showed (#389) - /logs/files rows carry an identity string (size:mtime_ns:ino); the workspace sends back the active and rotated rows its dialog was built from. Under the files and handler locks every listed copy is re-checked with _matches_review and the listed active log's inode; any difference refuses with nothing changed (409 logs-changed). A copy that appeared after the confirmation is kept. - A second writer of the active log (our flushed stream is behind the file's size, or our file is now a rotated copy) refuses first (409 logs-foreign-writer); each unlink is preceded by a fresh identity check and the active log is emptied last. - An active log with a hard link (st_nlink > 1) is never truncated (409 logs-hard-link), also on the file-logging-off path. - The legacy admin page's POST without a body keeps clearing what is there, with the foreign-writer and hard-link refusals. - The browser fixture grows the active log through the handler's stream. * fix(logs): accept pre-1970 identities, refuse duplicate rows, say the foreign-writer check is best effort (#389) Review follow-ups: a negative mtime_ns no longer makes every workspace clear 400; a name listed twice is a malformed list; the confirmed-row cap is 10000; the hard-link refusal says how to recover; the hard-link and foreign-append tests assert the kept bytes before the refusal code, so on the old code they fail on lost data. --- TESTING.md | 25 +- py/api/logging_routes.py | 42 ++- .../workspace-system-diagnostics.spec.mjs | 23 +- .../browser/workspace_diagnostics_fixture.py | 34 +- tests/js/workspace-system-diagnostics.test.js | 22 ++ tests/test_api_log_cleanup.py | 69 +++++ tests/test_utils_logging_config.py | 214 ++++++++++++- utils/logging_config.py | 292 ++++++++++++++---- web/js/workspace/system-diagnostics.mjs | 32 +- 9 files changed, 684 insertions(+), 69 deletions(-) diff --git a/TESTING.md b/TESTING.md index d0ab868c..67df06a6 100644 --- a/TESTING.md +++ b/TESTING.md @@ -3983,7 +3983,30 @@ modals use the same routes and keep working. other: each file's identity is checked just before its unlink. - **Clear all logs** (`POST /logs/truncate`, also the admin page's Clear) empties the active log through its handler, deletes the rotated copies (Rule R) and clears the - buffer; other files and links are kept. Response keys are unchanged. + buffer; other files and links are kept. Response keys are unchanged. It deletes or + empties no more than its confirmation showed (#389): + - `/logs/files` rows carry `identity`, the string `"<size>:<mtime_ns>:<ino>"` (a + string: `mtime_ns` and many inodes are beyond a browser's exact integers). The + workspace sends `{"expected": [{name, identity}]}` with the active and rotated rows + its dialog was built from. Under the files lock and every handler lock each listed + rotated copy must still match (`_matches_review`) and the listed active log must be + the same inode, or nothing changes: 409 `logs-changed`. Only listed copies are + deleted; one that appeared after the confirmation is kept. A malformed list is 400. + - Without a body (the legacy admin page, whose confirm lists nothing; also the + workspace when the file list could not be read and the dialog said "deletes its + rotated copies") every rotated copy present at the locked rescan is deleted. + - Either way, before anything changes (best effort, not a cross-process lock: a + write before our own last write, or any while file logging is off here, is not + seen): a second writer of the active log (our + handler's flushed stream is behind the file's size, or its file is now a rotated + copy: another ComfyUI sharing the folder) is 409 `logs-foreign-writer`; an active + log with more than one name (`st_nlink > 1` by `lstat` or the handler's `fstat`, + NTFS included) is 409 `logs-hard-link`, never truncated (replacing it instead would + rename a file our handler holds open, which Windows refuses). Each copy is + `lstat`-checked again just before its unlink; one that changed stops the clear there + and keeps it, the copies after it and the active log, which is emptied last. + - The browser fixture grows the active log through the handler's own stream, since a + second handle appending to it is exactly what Clear refuses. - **Runtime settings** (`GET/POST /logs/config`): the level (DEBUG…CRITICAL) and console logging apply until the server restarts; the retention values are shown read-only in the workspace and the API keeps accepting them. diff --git a/py/api/logging_routes.py b/py/api/logging_routes.py index 8918986c..ed13096a 100644 --- a/py/api/logging_routes.py +++ b/py/api/logging_routes.py @@ -11,6 +11,11 @@ import stat from aiohttp import web +try: + from ...utils.logging_config import ClearRefused, parse_confirmed +except ImportError: # loaded as the top-level ``py`` package (tests) + from utils.logging_config import ClearRefused, parse_confirmed + # Keys PromptManagerLogger.update_config() understands. Anything else is # rejected so a client cannot inject arbitrary entries into the logging config. LOG_CONFIG_KEYS = frozenset( @@ -263,10 +268,43 @@ class LoggingRoutesMixin: ) async def truncate_logs(self, request): - """Truncate all log files.""" + """Clear all logs: what ``{"expected": [...]}`` lists, or all (#389). + + ``expected`` holds the ``{name, identity}`` rows of ``/logs/files`` the + confirmation showed. Without a body (the legacy admin page, whose + confirmation lists nothing) every rotated copy present now is deleted. + A malformed list is 400; a clear that would empty or delete what was + not confirmed changes nothing and is 409 with the reason's ``code``. + """ try: + args = () + if request.can_read_body: + data, error_response = await self._read_json_body(request) + if error_response is not None: + return error_response + if "expected" in data: + try: + parse_confirmed(data["expected"]) + except ValueError: + return web.json_response( + { + "success": False, + "error": "expected must list name and identity" + " rows from /logs/files", + }, + status=400, + ) + args = (data["expected"],) logger_manager = self._get_logger_manager() - results = await self._run_in_executor(logger_manager.truncate_logs) + try: + results = await self._run_in_executor( + logger_manager.truncate_logs, *args + ) + except ClearRefused as refused: + return web.json_response( + {"success": False, "error": str(refused), "code": refused.code}, + status=409, + ) return web.json_response( { diff --git a/tests/browser/workspace-system-diagnostics.spec.mjs b/tests/browser/workspace-system-diagnostics.spec.mjs index c0bcb963..9f811362 100644 --- a/tests/browser/workspace-system-diagnostics.spec.mjs +++ b/tests/browser/workspace-system-diagnostics.spec.mjs @@ -516,7 +516,7 @@ test('a rollover after the review deletes nothing and Activity offers Review aga test('Clear all logs states both counts, confirms first and reads everything again', async ({ page, request }) => { await seedLogs(request, LOG_FOLDER); const truncates = []; - page.on('request', item => { if (item.url().endsWith('/prompt_manager/logs/truncate')) truncates.push(item.url()); }); + page.on('request', item => { if (item.url().endsWith('/prompt_manager/logs/truncate')) truncates.push(item.postDataJSON()); }); await openDiagnostics(page); await expect(lines(page)).toContainText('Database backup verified'); const details = await openFiles(page); @@ -535,6 +535,10 @@ test('Clear all logs states both counts, confirms first and reads everything aga await dialog(page).getByRole('button', { name: 'Clear', exact: true }).click(); await expect(toast(page)).toHaveText('Logs cleared · 3 rotated copies deleted.'); expect(truncates).toHaveLength(1); + // #389: Clear sends back exactly the files the dialog was built from, each with its identity string. + expect(truncates[0].expected.map(row => row.name).sort()).toEqual(['prompt_manager.log', 'prompt_manager.log.1', + 'prompt_manager.log.2', 'prompt_manager.log.3']); + for (const row of truncates[0].expected) expect(row.identity).toMatch(/^\d+:\d+:\d+$/); // Read again: the viewer, the file line and the list. await expect(lines(page)).not.toContainText('Database backup verified'); await expect(details.locator(':scope > p.small').first()).toHaveText(/rotated copies: 0$/); @@ -564,6 +568,23 @@ test('Clear all logs reads the files again before it states what it deletes', as expect(await names(request)).toEqual(['prompt_manager.log', 'prompt_manager.log.1', 'prompt_manager.log.2', 'prompt_manager.log.3']); }); +test('a rollover while Clear all logs is open deletes nothing and says why (#389)', async ({ page, request }) => { + await seedLogs(request, { rotated: [1024, 1024], active: 100 }); + await openDiagnostics(page); + await openFiles(page); + await operation(page, 'Clear all logs').getByRole('button', { name: 'Clear all logs…', exact: true }).click(); + await expect(dialog(page).locator('#modal-body > p')).toHaveText(/^Empties prompt_manager\.log, deletes 2 rotated copies /); + // The log rolls over after the confirmation was built: the copies it listed are not the ones there now. + await expect(await request.post('/__test__/logs/rotate')).toBeOK(); + const rolled = await names(request); + const answer = page.waitForResponse(response => response.url().endsWith('/prompt_manager/logs/truncate')); + await dialog(page).getByRole('button', { name: 'Clear', exact: true }).click(); + expect((await answer).status()).toBe(409); + await expect(toast(page)).toHaveText('The log files changed since you confirmed; nothing was cleared. ' + + 'Read the list again and clear again.'); + expect(await names(request)).toEqual(rolled); +}); + test('each listed file downloads by its returned name and a link has no Download', async ({ page, request }) => { await seedLogs(request, LOG_FOLDER); await openDiagnostics(page); diff --git a/tests/browser/workspace_diagnostics_fixture.py b/tests/browser/workspace_diagnostics_fixture.py index b55cf343..bdbc1193 100644 --- a/tests/browser/workspace_diagnostics_fixture.py +++ b/tests/browser/workspace_diagnostics_fixture.py @@ -123,6 +123,34 @@ def _touch(path: Path, size: int, moment: datetime.datetime) -> None: os.utime(path, (stamp, stamp)) +def _append_active(manager, log_dir: Path, size: int) -> None: + """Grow the active log as this process's handler does (#389). + + A second handle appending to the file the handler writes is what Clear + all logs refuses as another program writing to it, so the bytes go + through the handler's own stream; with file logging off, to the file. + """ + + def same(path) -> bool: + return os.path.normcase(os.path.realpath(path)) == os.path.normcase( + os.path.realpath(log_dir / ACTIVE) + ) + + for handler in manager._file_handlers(): + stream = getattr(handler, "stream", None) + if stream is None or not same(handler.baseFilename): + continue + handler.acquire() + try: + stream.write("x" * size) + stream.flush() + finally: + handler.release() + return + with open(log_dir / ACTIVE, "ab") as handle: + handle.write(b"x" * size) + + def _bounded(value, name, top=MAX_SEED_BYTES): if isinstance(value, bool) or not isinstance(value, int) or not 0 <= value <= top: raise web.HTTPBadRequest(text=f"{name} must be an integer 0..{top}") @@ -169,8 +197,7 @@ def register_diagnostics_commands(app, routes, api, root: Path): for index, (time_text, level, message) in enumerate(CONCEPT_RECORDS): record = _record(time_text, level, message, index) manager.add_to_buffer(record, record.getMessage()) - with open(log_dir / ACTIVE, "ab") as handle: - handle.write(b"x" * CONCEPT_ACTIVE) + _append_active(manager, log_dir, CONCEPT_ACTIVE) for number, size in enumerate(CONCEPT_ROTATED, start=1): moment = ROTATED_TIME - datetime.timedelta(hours=number) _touch(log_dir / f"{ACTIVE}.{number}", size, moment) @@ -194,8 +221,7 @@ def register_diagnostics_commands(app, routes, api, root: Path): manager.add_to_buffer(record, record.getMessage()) active = _bounded(data.get("active", 0), "active") if active: - with open(log_dir / ACTIVE, "ab") as handle: - handle.write(b"x" * active) + _append_active(manager, log_dir, active) rotated = data.get("rotated", []) if not isinstance(rotated, list) or len(rotated) > 20: raise web.HTTPBadRequest(text="rotated must be a list of sizes") diff --git a/tests/js/workspace-system-diagnostics.test.js b/tests/js/workspace-system-diagnostics.test.js index 65b0bddb..161c1bfa 100644 --- a/tests/js/workspace-system-diagnostics.test.js +++ b/tests/js/workspace-system-diagnostics.test.js @@ -175,6 +175,28 @@ test('Clear all logs states both counts and its toast says what was done', async assert.equal(clearToast(undefined), 'Logs cleared · 0 rotated copies deleted.'); }); +test('Clear all logs sends back exactly the files its confirmation showed (#389)', async () => { + const { clearBody, clearFailureText, COPY } = await diagnostics; + const files = [ + { filename: 'prompt_manager.log', kind: 'active', identity: '10:1761000000123456789:9007199254740993' }, + { filename: 'prompt_manager.log.1', kind: 'rotated', identity: '20:1761000000123456789:42' }, + { filename: 'prompt_manager.log.4', kind: 'link', identity: '5:1:7' }, + { filename: 'prompt_manager.log.bak', kind: 'other', identity: '1:1:8' }, + { filename: 'prompt_manager.log.2', kind: 'rotated' }, + ]; + // Identities stay strings: mtime_ns and large inodes are not exact JS numbers. + assert.deepEqual(clearBody(files), { expected: [ + { name: 'prompt_manager.log', identity: '10:1761000000123456789:9007199254740993' }, + { name: 'prompt_manager.log.1', identity: '20:1761000000123456789:42' }, + ] }); + assert.deepEqual(clearBody([]), { expected: [] }); + assert.deepEqual(clearBody(null), {}); + const refused = Object.assign(new Error('The log files changed since you confirmed; nothing was cleared.'), { status: 409 }); + assert.equal(clearFailureText(refused), refused.message); + assert.equal(clearFailureText(Object.assign(new Error('boom'), { status: 500 })), COPY.clearFailed); + assert.equal(clearFailureText(undefined), COPY.clearFailed); +}); + test('the test-link dialog names the prompt and how Missing-file links removes the record', async () => { const { testLinkSentence, promptName } = await diagnostics; assert.equal(testLinkSentence({ id: 812, text: 'Misty harbor' }), 'Writes a placeholder image record ' diff --git a/tests/test_api_log_cleanup.py b/tests/test_api_log_cleanup.py index d9287088..9622e599 100644 --- a/tests/test_api_log_cleanup.py +++ b/tests/test_api_log_cleanup.py @@ -415,5 +415,74 @@ class TestApply(LogCleanupTestCase): self.assertEqual(job.status_line, "The log cleanup failed.") +TRUNCATE = "/prompt_manager/logs/truncate" +FILES = "/prompt_manager/logs/files" + + +class TestClearAllLogsConfirmed(LogCleanupTestCase): + """#389: POST /logs/truncate deletes no more than the confirmation listed.""" + + async def confirmed(self): + response = await self.client.request("GET", FILES) + data = await response.json() + return [ + {"name": row["filename"], "identity": row["identity"]} + for row in data["files"] + if row["kind"] in ("active", "rotated") + ] + + def handler(self): + return next( + h + for h in self.manager.logger.handlers + if isinstance(h, logging.FileHandler) + ) + + async def test_a_rollover_after_the_confirmation_is_refused(self): + self.write("prompt_manager.log.1", b"a" * 100) + self.write("prompt_manager.log.2", b"b" * 200) + expected = await self.confirmed() + self.app_logger.info("written after the confirmation") + self.handler().doRollover() + before = self.snapshot() + body = await self.post(TRUNCATE, {"expected": expected}, 409) + self.assertEqual(body["success"], False) + self.assertEqual(body["code"], "logs-changed") + self.assertIn("nothing was cleared", body["error"]) + self.assertEqual(self.snapshot(), before) + + async def test_the_confirmed_files_are_cleared(self): + self.write("prompt_manager.log.1", b"a" * 100) + expected = await self.confirmed() + body = await self.post(TRUNCATE, {"expected": expected}, 200) + self.assertEqual( + sorted(body["results"]["truncated"]), + ["prompt_manager.log", "prompt_manager.log.1"], + ) + self.assertEqual(self.names(), ["prompt_manager.log"]) + + async def test_the_legacy_call_without_a_body_still_clears(self): + self.write("prompt_manager.log.1", b"a" * 100) + response = await self.client.request("POST", TRUNCATE) + self.assertEqual(response.status, 200, await response.text()) + self.assertEqual(self.names(), ["prompt_manager.log"]) + + async def test_a_foreign_writer_is_refused_with_a_clear_message(self): + self.write("prompt_manager.log.1", b"a" * 100) + with open(self.path("prompt_manager.log"), "ab") as other: + other.write(b"line from another ComfyUI\n") + body = await self.post(TRUNCATE, {"expected": await self.confirmed()}, 409) + self.assertEqual(body["code"], "logs-foreign-writer") + self.assertIn("Another program is writing", body["error"]) + self.assertIn("prompt_manager.log.1", self.names()) + + async def test_a_malformed_confirmation_is_400(self): + self.write("prompt_manager.log.1", b"a" * 100) + for expected in ("all", [{"name": "prompt_manager.log.1", "identity": 5}]): + with self.subTest(expected=expected): + await self.post(TRUNCATE, {"expected": expected}, 400) + self.assertIn("prompt_manager.log.1", self.names()) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_utils_logging_config.py b/tests/test_utils_logging_config.py index 3b07911e..d64ee51e 100644 --- a/tests/test_utils_logging_config.py +++ b/tests/test_utils_logging_config.py @@ -709,7 +709,8 @@ class TestRotatedCopyRule(IsolatedLoggerTestCase): self.assertNotEqual(link["size"], os.path.getsize(target)) for row in rows.values(): self.assertEqual( - set(row), {"filename", "path", "size", "modified", "is_main", "kind"} + set(row), + {"filename", "path", "size", "modified", "is_main", "kind", "identity"}, ) def test_an_unreadable_timestamp_does_not_break_the_scan(self): @@ -763,6 +764,217 @@ class TestRotatedCopyRule(IsolatedLoggerTestCase): self.manager.log_file_entry(name) +class TestClearOnlyWhatWasConfirmed(IsolatedLoggerTestCase): + """#389: Clear all logs empties or deletes no more than its confirmation showed.""" + + def setUp(self): + super().setUp() + self.logger.info("active log line") # the active log exists + self.outside = os.path.join(self.tmp.name, "outside") + os.makedirs(self.outside) + + def _path(self, name): + return os.path.join(self.log_dir, name) + + def _write(self, name, data=b"rotated\n"): + with open(self._path(name), "wb") as handle: + handle.write(data) + + def _read(self, path): + with open(path, "rb") as handle: + return handle.read() + + def _snapshot(self): + return {name: self._read(self._path(name)) for name in os.listdir(self.log_dir)} + + def _assert_unchanged(self, before, growing="prompt_manager.log"): + """Nothing deleted or emptied (the refusal itself logs to ``growing``).""" + after = self._snapshot() + self.assertEqual(sorted(after), sorted(before)) + for name, data in before.items(): + if name == growing: + self.assertTrue(after[name].startswith(data), name) + else: + self.assertEqual(after[name], data, name) + + def _confirmed(self): + """What the workspace dialog showed: the listed active and rotated rows.""" + return [ + {"name": row["filename"], "identity": row["identity"]} + for row in self.manager.get_log_files() + if row["kind"] in ("active", "rotated") + ] + + def _file_handler(self): + return next( + h + for h in self.manager.logger.handlers + if isinstance(h, logging.FileHandler) + ) + + def test_listed_identity_is_a_string_that_survives_json_numbers(self): + # mtime_ns (~1.7e18) and many inodes exceed 2**53: as JSON numbers a + # browser would round them and every confirmed clear would be refused. + self._write("prompt_manager.log.1") + for row in self.manager.get_log_files(): + with self.subTest(name=row["filename"]): + self.assertIsInstance(row["identity"], str) + self.assertRegex(row["identity"], r"^\d+:-?\d+:\d+$") + + def test_a_copy_dated_before_1970_can_be_confirmed(self): + self._write("prompt_manager.log.1") + try: + os.utime(self._path("prompt_manager.log.1"), ns=(0, -(10**9))) + except (OSError, OverflowError, ValueError) as error: + self.skipTest(f"this filesystem cannot store a time before 1970: {error}") + confirmed = self._confirmed() + if not any(":-" in row["identity"] for row in confirmed): + self.skipTest("the filesystem did not keep the negative time") + result = self.manager.truncate_logs(confirmed) + self.assertIn("prompt_manager.log.1", result["truncated"]) + + def test_a_rollover_after_the_confirmation_deletes_nothing(self): + self._write("prompt_manager.log.1", b"one" * 10) + self._write("prompt_manager.log.2", b"two" * 10) + confirmed = self._confirmed() + self.logger.info("written after the confirmation") + self._file_handler().doRollover() # .2 -> .3, .1 -> .2, log -> .1 + before = self._snapshot() + with self.assertRaises(logging_config.ClearRefused) as caught: + self.manager.truncate_logs(confirmed) + self.assertEqual(caught.exception.code, "logs-changed") + self._assert_unchanged(before) + + def test_a_copy_that_appeared_after_the_confirmation_is_kept(self): + self._write("prompt_manager.log.1", b"one") + confirmed = self._confirmed() + self._write("prompt_manager.log.5", b"not shown") + result = self.manager.truncate_logs(confirmed) + self.assertEqual(result["errors"], []) + self.assertEqual( + sorted(result["truncated"]), ["prompt_manager.log", "prompt_manager.log.1"] + ) + self.assertEqual( + sorted(os.listdir(self.log_dir)), + ["prompt_manager.log", "prompt_manager.log.5"], + ) + + def _clear(self, confirmed=None): + """Run a clear and return what it raised (None), so the files are + asserted first: on the old code the test fails on lost bytes.""" + try: + if confirmed is None: + self.manager.truncate_logs() # the legacy page's call + else: + self.manager.truncate_logs(confirmed) + except Exception as error: # noqa: BLE001 - asserted below + return error + return None + + def test_a_foreign_append_to_the_active_log_is_refused(self): + # A second ComfyUI sharing the folder writes the same active log. + self._write("prompt_manager.log.1") + listings = (None, "listed") + for listing in listings: + with self.subTest(listed=listing is not None): + confirmed = self._confirmed() if listing else None + with open(self._path("prompt_manager.log"), "ab") as other: + other.write(b"line from another process\n") + before = self._snapshot() + error = self._clear(confirmed) + self._assert_unchanged(before) + self.assertEqual(getattr(error, "code", None), "logs-foreign-writer") + self.logger.info("our own line") # our stream catches up again + + @unittest.skipIf(os.name == "nt", "Windows cannot rename a log that is open") + def test_a_foreign_rotation_of_our_log_is_refused(self): + # Another process rolled the shared log over: our handler still writes + # the file now named .1, and a new active log is the other's. + os.rename(self._path("prompt_manager.log"), self._path("prompt_manager.log.1")) + self._write("prompt_manager.log", b"the other process's log\n") + before = self._snapshot() + with self.assertRaises(logging_config.ClearRefused) as caught: + self.manager.truncate_logs() + self.assertEqual(caught.exception.code, "logs-foreign-writer") + # Our handler still writes the renamed file, so the refusal lands there. + self._assert_unchanged(before, growing="prompt_manager.log.1") + + def test_a_copy_that_changes_mid_clear_stops_the_clear(self): + # The identity is checked again just before each unlink: another + # process rotating while we delete stops the clear there. + for n in (1, 2): + self._write(f"prompt_manager.log.{n}", b"r" * n) + confirmed = self._confirmed() + real_unlink = os.unlink + calls = [] + + def unlink_then_rotate(path, *args, **kwargs): + real_unlink(path, *args, **kwargs) + if not calls: + with open(self._path("prompt_manager.log.2"), "ab") as handle: + handle.write(b"rotated in by another process") + calls.append(os.path.basename(path)) + + active_before = self._read(self._path("prompt_manager.log")) + with mock.patch.object(logging_config.os, "unlink", unlink_then_rotate): + result = self.manager.truncate_logs(confirmed) + self.assertEqual(calls, ["prompt_manager.log.1"]) + self.assertEqual(result["truncated"], ["prompt_manager.log.1"]) + self.assertEqual(len(result["errors"]), 1) + self.assertIn("prompt_manager.log.2", result["errors"][0]) + self.assertTrue(os.path.exists(self._path("prompt_manager.log.2"))) + # The active log is emptied last, so it is kept too. + self.assertTrue( + self._read(self._path("prompt_manager.log")).startswith(active_before) + ) + + def _require_hard_link(self, source, link): + try: + os.link(source, link) + except (OSError, NotImplementedError, AttributeError) as error: + self.skipTest(f"this filesystem cannot make hard links: {error}") + + def test_a_hard_linked_active_log_is_never_emptied(self): + other_name = os.path.join(self.outside, "kept-by-the-user.log") + self._require_hard_link(self._path("prompt_manager.log"), other_name) + kept = self._read(other_name) + self.assertTrue(kept) + for listed in (False, True): + with self.subTest(listed=listed): + error = self._clear(self._confirmed() if listed else None) + self.assertTrue(self._read(other_name).startswith(kept)) + self.assertEqual(getattr(error, "code", None), "logs-hard-link") + + def test_a_hard_linked_active_log_with_file_logging_off_is_kept(self): + self.manager.update_config({"file_logging": False}) + other_name = os.path.join(self.outside, "kept-by-the-user.log") + self._require_hard_link(self._path("prompt_manager.log"), other_name) + kept = self._read(other_name) + error = self._clear() + self.assertEqual(self._read(other_name), kept) + self.assertEqual(getattr(error, "code", None), "logs-hard-link") + + def test_a_malformed_confirmation_is_refused_before_anything_changes(self): + self._write("prompt_manager.log.1") + before = self._snapshot() + for expected in ( + "prompt_manager.log.1", + [{"name": "prompt_manager.log.1"}], + [{"name": "prompt_manager.log.1", "identity": 7}], + [{"name": "prompt_manager.log.1", "identity": "1:2"}], + [{"name": 1, "identity": "1:2:3"}], + [ + {"name": "prompt_manager.log.1", "identity": "1:2:3"}, + {"name": "prompt_manager.log.1", "identity": "1:2:3"}, + ], + ["prompt_manager.log.1"], + ): + with self.subTest(expected=expected): + with self.assertRaises(ValueError): + self.manager.truncate_logs(expected) + self._assert_unchanged(before) + + class TestConfiguration(IsolatedLoggerTestCase): def test_in_flight_record_cannot_reopen_retired_file_handler(self) -> None: handler = next( diff --git a/utils/logging_config.py b/utils/logging_config.py index 97a37639..08bb0a26 100644 --- a/utils/logging_config.py +++ b/utils/logging_config.py @@ -39,6 +39,35 @@ class RotatedChanged(Exception): self.name = name +# Clear all logs (#389): why it changed nothing, as the API and the pages say it. +CLEAR_CHANGED = ( + "The log files changed since you confirmed; nothing was cleared." + " Read the list again and clear again." +) +CLEAR_FOREIGN_WRITER = ( + "Another program is writing to prompt_manager.log, such as a second ComfyUI" + " sharing this log folder; nothing was cleared. Stop it and clear again." +) +CLEAR_HARD_LINK = ( + "prompt_manager.log has another name (a hard link); emptying it would empty" + " that file too, so nothing was cleared. Remove the other name, then clear" + " again." +) +# A confirmed row's identity: "<size>:<mtime_ns>:<ino>", a string because +# mtime_ns and many inodes are larger than a browser's exact integers. +# mtime_ns is negative for a time before 1970 (a restored archive, say). +_IDENTITY = re.compile(r"([0-9]{1,20}):(-?[0-9]{1,20}):([0-9]{1,40})") +MAX_CONFIRMED_ROWS = 10000 + + +class ClearRefused(Exception): + """Clear all logs changed nothing; ``code`` names why, the message says it.""" + + def __init__(self, code: str, message: str): + super().__init__(message) + self.code = code + + def _name_key(name: str) -> str: """How a name compares: case-folded where the filesystem folds it (Windows).""" return os.path.normcase(name) @@ -87,35 +116,37 @@ def scan_log_folder(log_dir) -> List[Dict[str, Any]]: be read is left out. Never logs: the cleanup calls it while it holds the file handler's lock. """ - rows = [] try: names = [entry.name for entry in os.scandir(log_dir)] except OSError: - return rows - for name in names: - path = os.path.join(log_dir, name) - try: - st = os.lstat(path) - kind = _kind(name, path, st) - except OSError: - continue - try: - modified = datetime.fromtimestamp(st.st_mtime).isoformat() - except (OSError, OverflowError, ValueError): # a time before 1970 on Windows - modified = "" - rows.append( - { - "name": name, - "kind": kind, - "size": st.st_size, - "mtime_ns": st.st_mtime_ns, - "ino": st.st_ino, - "modified": modified, - "_dir": stat.S_ISDIR(st.st_mode) and kind != "link", - "_regular": stat.S_ISREG(st.st_mode) and kind != "link", - } - ) - return rows + return [] + rows = (scan_log_entry(log_dir, name) for name in names) + return [row for row in rows if row is not None] + + +def scan_log_entry(log_dir, name) -> Optional[Dict[str, Any]]: + """One entry's scan row (``lstat``), or None when it cannot be read.""" + path = os.path.join(log_dir, name) + try: + st = os.lstat(path) + kind = _kind(name, path, st) + except OSError: + return None + try: + modified = datetime.fromtimestamp(st.st_mtime).isoformat() + except (OSError, OverflowError, ValueError): # a time before 1970 on Windows + modified = "" + return { + "name": name, + "kind": kind, + "size": st.st_size, + "mtime_ns": st.st_mtime_ns, + "ino": st.st_ino, + "modified": modified, + "_dir": stat.S_ISDIR(st.st_mode) and kind != "link", + "_regular": stat.S_ISREG(st.st_mode) and kind != "link", + "_nlink": st.st_nlink, + } def public_row(row: Dict[str, Any]) -> Dict[str, Any]: @@ -127,6 +158,37 @@ def rotated_sort_key(row: Dict[str, Any]) -> int: return _rotated_number(row["name"]) or 0 +def file_identity(row: Dict[str, Any]) -> str: + """A listed file's identity as the client sends it back: size, mtime_ns, ino.""" + return f"{row['size']}:{row['mtime_ns']}:{row['ino']}" + + +def parse_confirmed(expected) -> List[Dict[str, Any]]: + """The rows a Clear all logs confirmation showed, as review rows. + + ``expected`` is a list of ``{name, identity}`` objects from + ``/logs/files``; anything else raises ValueError. + """ + if not isinstance(expected, list) or len(expected) > MAX_CONFIRMED_ROWS: + raise ValueError("expected must be a list of listed log files") + rows, seen = [], set() + for item in expected: + if not isinstance(item, dict): + raise ValueError("each expected file must be an object") + name, identity = item.get("name"), item.get("identity") + if not isinstance(name, str) or not isinstance(identity, str): + raise ValueError("each expected file needs a name and an identity") + match = _IDENTITY.fullmatch(identity) + if match is None: + raise ValueError("an expected file's identity is not size:mtime_ns:ino") + size, mtime_ns, ino = (int(part) for part in match.groups()) + if _name_key(name) in seen: + raise ValueError("an expected file is listed twice") + seen.add(_name_key(name)) + rows.append({"name": name, "size": size, "mtime_ns": mtime_ns, "ino": ino}) + return rows + + def log_file_entry(log_dir, name) -> Path: """The path of a downloadable log file named ``name`` (blocking). @@ -157,6 +219,11 @@ def _stream_is(stream, ino: int) -> bool: return not (ino and open_ino and open_ino != ino) +def _stream_ino_is(opened: os.stat_result, ino: int) -> bool: + """Whether a stream's ``fstat`` is the file with inode ``ino`` (0: unknown).""" + return not (ino and opened.st_ino and opened.st_ino != ino) + + def _error_text(error: BaseException) -> str: """An OSError's reason and file name, without the folder it is in.""" if isinstance(error, OSError): @@ -432,6 +499,7 @@ class PromptManagerLogger: "modified": row["modified"], "is_main": row["kind"] == "active", "kind": row["kind"], + "identity": file_identity(row), } ) @@ -530,8 +598,14 @@ class PromptManagerLogger: finally: handler.release() if not handled: - flags = os.O_WRONLY | os.O_TRUNC | getattr(os, "O_NOFOLLOW", 0) - os.close(os.open(str(log_file), flags)) + flags = os.O_WRONLY | getattr(os, "O_NOFOLLOW", 0) + fd = os.open(str(log_file), flags) + try: + if os.fstat(fd).st_nlink > 1: # emptying it would empty the other + raise ClearRefused("logs-hard-link", CLEAR_HARD_LINK) + os.ftruncate(fd, 0) + finally: + os.close(fd) def delete_rotated(self, expected: List[Dict[str, Any]]) -> Dict[str, Any]: """Delete exactly the reviewed rotated copies, or none of them (blocking). @@ -582,47 +656,126 @@ class PromptManagerLogger: ) return {"removed": removed, "failed": failed} - def truncate_logs(self) -> Dict[str, Any]: - """Empty the active log and delete its rotated copies (Rule R). + def _active_streams(self, active: Optional[Dict[str, Any]]): + """``fstat`` and offset of every handler stream writing the active name.""" + target = os.path.normcase( + os.path.realpath(os.path.join(self.log_dir, ACTIVE_LOG_NAME)) + ) + for handler in self._file_handlers(): + stream = getattr(handler, "stream", None) + base = getattr(handler, "baseFilename", None) + if stream is None or base is None: + continue + if os.path.normcase(os.path.realpath(base)) != target: + continue + try: + stream.flush() + fd = stream.fileno() + yield os.fstat(fd), os.lseek(fd, 0, os.SEEK_CUR) + except (OSError, ValueError, AttributeError): + continue - Every other file in the log folder, and every link, is kept. Also - clears the memory buffer. Runs under ``_files_lock`` and the file - handlers' locks, so no rollover recreates a just-deleted copy, and - logs only once they are released. + def _check_clear_is_safe(self, rows, active) -> None: + """Refuse a clear that would empty or delete what nobody confirmed. + + Called under the files lock and every handler lock, so nothing of this + process writes or rolls over meanwhile; what changes is another + program's doing. Our handler's stream is flushed: a file larger than + the stream's own offset was appended to by someone else, and a stream + whose file is now a rotated copy was rolled over by someone else. + Either is a second writer (another ComfyUI sharing the folder). This + is best effort, not a lock: writes before our own last write, or any + write while this process has file logging off, go unseen. An + active log with more than one name (a hard link) is never emptied: the + other name would be emptied too. + """ + rotated = {row["ino"] for row in rows if row["kind"] == "rotated"} + rotated.discard(0) + nlink = active["_nlink"] if active is not None else 1 + for opened, offset in self._active_streams(active): + if (opened.st_ino and opened.st_ino in rotated) or opened.st_size > offset: + raise ClearRefused("logs-foreign-writer", CLEAR_FOREIGN_WRITER) + if active is not None and _stream_ino_is(opened, active["ino"]): + nlink = max(nlink, opened.st_nlink) + if nlink > 1: + raise ClearRefused("logs-hard-link", CLEAR_HARD_LINK) + + @staticmethod + def _confirmed_targets(confirmed, current, active) -> List[Dict[str, Any]]: + """The rotated copies a confirmation listed, each as it was listed. + + Refuses (``logs-changed``) when any listed copy is missing or differs + in kind, size, ``mtime_ns`` or inode, or when the listed active log is + now another file (it was rolled over). A copy nobody listed is kept. + """ + targets = [] + for reviewed in confirmed: + key = _name_key(reviewed["name"]) + row = current.get(key) + if key == _name_key(ACTIVE_LOG_NAME): + ino = reviewed["ino"] + if active is None or (ino and active["ino"] and active["ino"] != ino): + raise ClearRefused("logs-changed", CLEAR_CHANGED) + continue + if _rotated_number(reviewed["name"]) is None: + continue # never deleted, listed or not + if not _matches_review(row, reviewed): + raise ClearRefused("logs-changed", CLEAR_CHANGED) + targets.append(row) + return targets + + def truncate_logs(self, expected=None) -> Dict[str, Any]: + """Empty the active log and delete its rotated copies (Rule R, #389). + + ``expected`` is what the confirmation showed: ``{name, identity}`` + rows of ``/logs/files`` (``parse_confirmed``; ValueError when + malformed). Then only the listed rotated copies are deleted, and only + when each still has the listed size, ``mtime_ns`` and inode and the + active log is still the listed file; otherwise ClearRefused + (``logs-changed``) with nothing changed. Without it (the legacy admin + page, whose confirmation lists nothing) every rotated copy present now + is deleted. + + Either way, a second writer of the active log (``logs-foreign-writer``) + or an active log with a hard link (``logs-hard-link``) refuses the + whole clear first. Each copy's identity is checked again just before + its unlink; one that changed (another process rolling over meanwhile) + stops the clear there, keeping it, the rest and the active log, which + is emptied last. Every other file in the log folder, and every link, + is kept. Also clears the memory buffer. Runs under ``_files_lock`` and + the file handlers' locks and logs only once they are released. Returns: Dictionary with: - truncated: List of successfully truncated files - errors: List of error messages for failed operations """ + confirmed = None if expected is None else parse_confirmed(expected) results = {"truncated": [], "errors": []} + refused = None with self._files_lock: held = self._acquire_file_handlers() try: rows = scan_log_folder(self.log_dir) - active = [row for row in rows if row["kind"] == "active"] - rotated = sorted( - (row for row in rows if row["kind"] == "rotated"), - key=rotated_sort_key, - ) - for row in active + rotated: - path = os.path.join(self.log_dir, row["name"]) - try: - if row["kind"] == "active": - self._truncate_main_log(Path(path), row["ino"]) - else: - os.unlink(path) - results["truncated"].append(row["name"]) - except OSError as error: - results["errors"].append( - f"Failed to truncate {row['name']}: {_error_text(error)}" - ) - with self._buffer_lock: - self._log_buffer.clear() + current = {_name_key(row["name"]): row for row in rows} + active = next((row for row in rows if row["kind"] == "active"), None) + try: + if confirmed is None: + targets = [row for row in rows if row["kind"] == "rotated"] + else: + targets = self._confirmed_targets(confirmed, current, active) + self._check_clear_is_safe(rows, active) + except ClearRefused as error: + refused = error + else: + self._clear(sorted(targets, key=rotated_sort_key), active, results) finally: self._release_handlers(held) + if refused is not None: + self.logger.warning("Clear all logs refused (%s)", refused.code) + raise refused if results["truncated"]: self.logger.info("Truncated log files: %s", ", ".join(results["truncated"])) for error_msg in results["errors"]: @@ -630,6 +783,39 @@ class PromptManagerLogger: return results + def _clear(self, targets, active, results) -> None: + """Delete ``targets`` (each re-checked) then empty ``active`` (locks held).""" + for reviewed in targets: + name = reviewed["name"] + if not _matches_review(scan_log_entry(self.log_dir, name), reviewed): + results["errors"].append( + f"{name} changed while clearing (another program may be writing" + " to this log folder); it, the copies after it and the active" + " log were kept" + ) + return + try: + os.unlink(os.path.join(self.log_dir, name)) + results["truncated"].append(name) + except OSError as error: + results["errors"].append( + f"Failed to truncate {name}: {_error_text(error)}" + ) + if active is not None: + try: + self._truncate_main_log( + Path(self.log_dir, active["name"]), active["ino"] + ) + results["truncated"].append(active["name"]) + except ClearRefused as error: + results["errors"].append(str(error)) + except OSError as error: + results["errors"].append( + f"Failed to truncate {active['name']}: {_error_text(error)}" + ) + with self._buffer_lock: + self._log_buffer.clear() + def update_config(self, new_config: Dict[str, Any]): """Update logging configuration. diff --git a/web/js/workspace/system-diagnostics.mjs b/web/js/workspace/system-diagnostics.mjs index 40382ae0..776be604 100644 --- a/web/js/workspace/system-diagnostics.mjs +++ b/web/js/workspace/system-diagnostics.mjs @@ -225,6 +225,23 @@ export function clearSentence(files) { return `Empties ${ACTIVE_LOG}, deletes ${number(rotated.length)} rotated ${noun(rotated.length, 'copy', 'copies')} (${sizeText(size)}) ${tail}`; } +/** + * #389: the `/logs/truncate` body for the files a confirmation showed. Their active and rotated rows go back with the + * server's `identity`, so the server deletes only those and refuses (409) when any changed meanwhile. A list that could + * not be read (`null`, the general sentence) sends no list: the server then clears what is there, as the sentence says. + */ +export function clearBody(files) { + if (!Array.isArray(files)) return {}; + const expected = files.filter(row => (row?.kind === 'active' || row?.kind === 'rotated') && typeof row.filename === 'string' + && typeof row.identity === 'string').map(row => ({ name: row.filename, identity: row.identity })); + return { expected }; +} + +/** A refused clear (409) says why in the server's words; any other failure the general line. */ +export function clearFailureText(error) { + return error?.status === 409 && typeof error.message === 'string' && error.message ? error.message : COPY.clearFailed; +} + /** CUJ 7: the toast for `/logs/truncate`'s `results` ({truncated: [names], errors: [messages]}). */ export function clearToast(results) { const truncated = Array.isArray(results?.truncated) ? results.truncated : []; @@ -469,13 +486,14 @@ function logsSurface({ api, activity, scope, drafts, navigate, notify, latestOnl const list = el('div', undefined, 'operation-list'); let listed = null; // The confirmation names what Clear deletes now: the files are read again before it opens (a read that fails - // gives the general sentence, never an old count). + // gives the general sentence, never an old count). Clear sends exactly the files the dialog was built from (#389). const clear = operationRow({ scope, title: 'Clear all logs', description: COPY.clearDescription, label: 'Clear all logs…', danger: true, onClick: button => pressing(button, async () => { await readFiles(); if (scope.signal.aborted) return; - confirmDialog({ scope, trigger: button, title: 'Clear all logs', sentence: clearSentence(listed), label: 'Clear', - danger: true, onConfirm: clearLogs }); + const shown = listed; + confirmDialog({ scope, trigger: button, title: 'Clear all logs', sentence: clearSentence(shown), label: 'Clear', + danger: true, onConfirm: () => clearLogs(shown) }); }) }); files.append(el('summary', 'Log files & retention'), line, review, retention, list, clear); const runtime = runtimeRows({ scope, drafts }); @@ -562,12 +580,12 @@ function logsSurface({ api, activity, scope, drafts, navigate, notify, latestOnl readConfig(); } - async function clearLogs() { + async function clearLogs(shown) { let answer; try { - answer = await api.post(TRUNCATE_URL, {}); - } catch { - if (!scope.signal.aborted) notify(COPY.clearFailed); + answer = await api.post(TRUNCATE_URL, clearBody(shown)); + } catch (error) { + if (!scope.signal.aborted) notify(clearFailureText(error)); } if (scope.signal.aborted) return; if (answer) notify(clearToast(answer.results));