nodes/ComfyAssets-ComfyUI_PromptManager
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into fix/252-no-sampler-scan

Browse Source
This commit is contained in:
Vito Sansevero
2026-10-27 16:57:01 -07:00
parent 0412589b11 d11b354ed6
commit fb89ead02b
37 changed files with 2993 additions and 1115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
py/api/__init__.py
+3
View File
@@ -363,6 +363,9 @@ class PromptManagerAPI(
# Private media captures of the annotation routes (py/api/annotations.py),
# created on first use with their restore subscription.
self._captures = None
# Capture tokens Generate suggestions ran on: a save of one marks the
# image auto-tagged (#379).
self._suggested_tokens = set()
self._captures_unsubscribe = None
# Set by close_annotation_captures (shutdown): no store is recreated.
self._captures_closed = False
py/api/annotations.py
+20
View File
@@ -296,6 +296,7 @@ class AnnotationRoutesMixin:
self._captures_closed = True
unsubscribe, self._captures_unsubscribe = self._captures_unsubscribe, None
store, self._captures = self._captures, None
self._suggested_tokens = set()
if unsubscribe is not None:
unsubscribe()
if store is not None:
@@ -515,6 +516,7 @@ class AnnotationRoutesMixin:
store = self._captures
if store is not None and _TOKEN.fullmatch(token) is not None:
store.release(token)
self._suggested_tokens.discard(token)
return web.json_response(
{"success": True}, headers={"Cache-Control": "no-store"}
)
@@ -564,6 +566,9 @@ class AnnotationRoutesMixin:
return _coded("capture-expired", _OTHER_FILE)
if facts.sha256 != capture.sha256:
return _coded("file-changed")
# A save after Generate suggestions ran on this capture marks the
# image auto-tagged (#379), so a Skip batch leaves the review alone.
suggested = token in self._suggested_tokens
try:
receipt = await self._run_in_executor(
self.db.replace_image_tags,
@@ -574,6 +579,7 @@ class AnnotationRoutesMixin:
expected_epoch=expected_epoch,
expected_tag_revision=revision,
tags=tags,
mark_autotagged=suggested,
)
except AnnotationCommandError as exc:
current = exc.current or {}
@@ -590,10 +596,23 @@ class AnnotationRoutesMixin:
except Exception:
self.logger.exception("Image tag save failed")
return _error(500, "save-failed", "The tags could not be saved.")
# Dropped only after a saved or replayed receipt: a retry after a
# tag-conflict still marks.
self._suggested_tokens.discard(token)
return web.json_response(
{"success": True, **receipt}, headers={"Cache-Control": "no-store"}
)
def _remember_suggested(self, store, token: str) -> None:
"""Note that Generate ran on *token*; forget tokens whose capture is
gone. ``expires_in`` does not reset a capture's idle clock."""
self._suggested_tokens = {
kept
for kept in self._suggested_tokens
if store.expires_in(kept) is not None
}
self._suggested_tokens.add(token)
# -- suggest ---------------------------------------------------------------
async def suggest_image_tags(self, request: web.Request) -> web.Response:
@@ -675,6 +694,7 @@ class AnnotationRoutesMixin:
finally:
if not started:
store.release_use(token)
self._remember_suggested(store, token)
return web.json_response(
{
"success": True,
py/api/autotag_routes.py
+520 -409
View File
File diff suppressed because it is too large Load Diff
py/api/image_tag_jobs.py
+75 -10
View File
@@ -2,7 +2,9 @@
``POST /prompt_manager/jobs/image-tags`` starts one ``image-tags`` job over
1..500 listed images, named by the listing identity ``{root_index,
root_scope, path}``. The job is record-changing (it writes image tag sets and
root_scope, path}``, or over every image a listing ``filter`` admits (at most
10,000, resolved at start in the listing's order; "Select all N matching",
#392). The job is record-changing (it writes image tag sets and
new vocabulary names), so it never runs beside the metadata scan.
- **Add tags** (``mode: "add"``) hashes each current file and unions a fixed
@@ -54,7 +56,14 @@ from .annotations import (
_target,
)
from .autotag_routes import BATCH_CANCELLED, MODEL_BUSY_ERROR, model_busy_response
from .images import _current_fingerprint, _listing_posix, _public_path
from .images import (
MAX_LISTING_SELECTION,
ListingSourceChanged,
_current_fingerprint,
_listing_posix,
_public_path,
parse_listing_filter,
)
from .job_routes import busy_response
from .jobs import (
AUTOTAG_BATCH_JOB_KIND,
@@ -90,6 +99,11 @@ except ImportError:
from utils.video_metadata import is_video_path
MAX_JOB_TARGETS = 500
# A filter start resolves on the server, so the request body does not bound it.
MAX_FILTER_TARGETS = MAX_LISTING_SELECTION
NO_IMAGES = "No images match this view."
FILTER_LISTING_FAILED = "The matching images could not be listed. Try again."
TOO_MANY_IMAGES = "Narrow the view to at most {cap:,} images."
# 500 maximal targets plus the options: a path of MAX_TARGET_PATH code points
# takes at most 6 bytes each on the wire (a \uXXXX escape; raw UTF-8 is at most
# 4), and the root index and scope fit in 256. The annotation routes' 64 KiB
@@ -103,9 +117,10 @@ ADD_TITLE = "Add tags to images"
AUTO_TITLE = "Auto-tag images"
GATE_OWNER_KIND = "image-tag-job"
# Each mode's fields besides the images, named by ``targets`` or ``filter``.
_FIELDS = {
"add": frozenset({"mode", "targets", "tags"}),
"auto": frozenset({"mode", "targets", "model_type", "options"}),
"add": frozenset({"mode", "tags"}),
"auto": frozenset({"mode", "model_type", "options"}),
}
_VERBS = {"add": "Add tags", "auto": "Auto-tag"}
_LOG_PREFIXES = {"tagged": "+", "unchanged": "=", "skipped": "-", "errors": "!"}
@@ -164,6 +179,8 @@ class _Spec:
tags: tuple = ()
model_type: str | None = None
options: dict | None = None
# A listing filter the start resolves into ``targets`` (#392).
filter: dict | None = None
@property
def title(self) -> str:
@@ -222,21 +239,35 @@ def parse_image_tag_request(data: dict):
"""A :class:`_Spec` for a start body, or ``(None, error response)``.
``{mode:"add", targets, tags}`` or ``{mode:"auto", targets, model_type,
options}`` exactly; options follow the Tag dialog's Generate rules.
options}`` exactly; options follow the Tag dialog's Generate rules. A
``filter`` (a listing query object, parse_listing_filter) may stand in for
``targets``; the start resolves it.
"""
fields = _FIELDS.get(data.get("mode"))
if fields is None:
return None, bad_request('mode must be "add" or "auto"')
fields = fields | {"filter" if "filter" in data else "targets"}
if set(data) != fields:
return None, bad_request(
"Request must contain exactly " + ", ".join(sorted(fields))
)
targets, error = _parse_targets(data["targets"])
if error is not None:
return None, error
listing_filter, targets = None, ()
if "filter" in data:
try:
listing_filter = parse_listing_filter(data["filter"])
except ValueError as exc:
return None, bad_request(str(exc))
else:
targets, error = _parse_targets(data["targets"])
if error is not None:
return None, error
if data["mode"] == "add":
return _parse_add(targets, data)
return _parse_auto(targets, data)
spec, error = _parse_add(targets, data)
else:
spec, error = _parse_auto(targets, data)
if spec is not None and listing_filter is not None:
spec = dataclasses.replace(spec, filter=listing_filter)
return spec, error
def _capture_outcome(exc: CaptureError, root, relative: str):
@@ -398,6 +429,10 @@ class ImageTagJobsMixin:
spec, error = parse_image_tag_request(data)
if error is not None:
return error
if spec.filter is not None:
spec, error = await self._resolve_job_filter(spec)
if error is not None:
return error
roots = await self._run_in_executor(self._job_roots, spec.targets)
if any(root is None for root, _label in roots.values()):
return _coded("source-changed")
@@ -639,6 +674,36 @@ class ImageTagJobsMixin:
job.log_line(f"Skipped {_plural(skipped, 'image')} before tagging")
return remaining
async def _resolve_job_filter(self, spec: _Spec):
"""``spec`` with its filter resolved into targets, or ``(None, error)``.
The listing's own identities, in its order, kept only where a target
could have named them (a media path the job can tag).
"""
cap = MAX_FILTER_TARGETS
try:
selection = await self.resolve_listing_filter(spec.filter, cap)
except ListingSourceChanged:
return None, _coded("source-changed")
except ValueError as exc:
return None, bad_request(str(exc))
except Exception:
# Never echo it: it can carry SQL or database text.
self.logger.error("A tagging filter could not be resolved", exc_info=True)
return None, _error(500, "listing-failed", FILTER_LISTING_FAILED)
if selection.truncated:
return None, _error(400, "too-many-images", TOO_MANY_IMAGES.format(cap=cap))
targets = []
for index, scope, relative in selection.targets:
target, error = _target(
{"root_index": index, "root_scope": scope, "path": relative}
)
if error is None:
targets.append(target)
if not targets:
return None, _error(400, "no-images", NO_IMAGES)
return dataclasses.replace(spec, targets=tuple(targets)), None
# -- runner ----------------------------------------------------------------
def _job_roots(self, targets) -> dict:
py/api/images.py
+194 -45
View File
@@ -1099,37 +1099,94 @@ def _workspace_filters(query: Mapping) -> dict:
}
# The listing parameters a bulk request may name as its filter (#392): the
# workspace query without mode, limit, cursor and the Activity-only ``path``,
# plus the chosen root's ``root_scope`` (a renumbered source is refused).
LISTING_FILTER_KEYS = frozenset(
{
"q",
"subfolder",
"category",
"tags",
"root",
"root_scope",
"media",
"sort",
"favorites",
}
)
MAX_LISTING_FILTER_VALUE = 2048
# The most images "Select all N matching" acts on in one request (#392).
MAX_LISTING_SELECTION = 10_000
def parse_listing_filter(raw: object) -> dict:
"""A bulk request's listing filter (#392): the listing's own query as an object.
Keys are a subset of LISTING_FILTER_KEYS, each value a string of at most
MAX_LISTING_FILTER_VALUE characters, validated as the listing validates its
query. The root's bounds need the current roots, so resolve_listing_filter
checks those. Raises ValueError.
"""
if not isinstance(raw, dict) or not set(raw) <= LISTING_FILTER_KEYS:
raise ValueError("filter must be an object of listing parameters")
if any(
not isinstance(value, str) or len(value) > MAX_LISTING_FILTER_VALUE
for value in raw.values()
):
raise ValueError("filter values must be short strings")
if "root_scope" in raw and "root" not in raw:
raise ValueError("root_scope needs root")
_workspace_filters(raw)
return dict(raw)
class ListingSourceChanged(Exception):
"""A listing filter's ``root_scope`` no longer names its root (#392)."""
class ListingSelection(NamedTuple):
"""Every image a listing filter admits, in the listing's order (#392).
``targets`` are listing identities ``(root_index, root_scope, path)``;
``positive_ids`` the distinct linked positives (first seen), ``unlinked``
the admitted images without one, and ``truncated`` that more than the cap
matched (the lists then stop at the cap).
"""
targets: list
positive_ids: list
unlinked: int
truncated: bool
# A plain listing's cursor page leaves positive_count out (see
# _select_workspace_page); None instead means the count could not be read.
_COUNT_OMITTED = object()
def _filtered_page(
def _admitted_entries(
entries: list,
start: int,
size: int,
filters: dict,
links: dict | None,
favorites: frozenset | None = None,
image_tags: dict | None = None,
) -> tuple[int, list, int | None]:
"""Count the *entries* the filters admit and take *size* from *start* on.
):
"""Yield ``(position, entry, link match)`` for each entry the filters admit.
*links* maps normalized paths to (hit, rating, positive id) for every
unambiguously linked image (see PromptDatabase.get_image_link_matches); a
category or tag filter admits only those, and ``q`` admits a filename match
or a linked-positive hit. Returns the total, the page and the number of
distinct linked positives among all admitted entries (None without links).
*favorites*, when given, admits only those listed paths (``favorites=1``).
*image_tags* maps normalized paths to their _ImageTagMatch: a tag filter
also admits a file whose current image tags (with its linked positive's
tags) carry every tag, and ``q`` also admits an image tag name hit.
or a linked-positive hit. *favorites*, when given, admits only those listed
paths (``favorites=1``). *image_tags* maps normalized paths to their
_ImageTagMatch: a tag filter also admits a file whose current image tags
(with its linked positive's tags) carry every tag, and ``q`` also admits an
image tag name hit.
"""
query, media, folder = filters["query"], filters["media"], filters["folder"]
members_only = filters["category"] is not None or bool(filters["tags"])
native_folder = folder.replace("/", os.sep)
nested = native_folder + os.sep
total, page, positives = 0, [], set()
exact = filters.get("path", "")
for position, entry in enumerate(entries):
if exact and entry.folded != exact:
@@ -1153,6 +1210,28 @@ def _filtered_page(
and not (image and image.query)
):
continue
yield position, entry, match
def _filtered_page(
entries: list,
start: int,
size: int,
filters: dict,
links: dict | None,
favorites: frozenset | None = None,
image_tags: dict | None = None,
) -> tuple[int, list, int | None]:
"""Count the *entries* the filters admit and take *size* from *start* on.
Admission is _admitted_entries'. Returns the total, the page and the number
of distinct linked positives among all admitted entries (None without
links).
"""
total, page, positives = 0, [], set()
for position, entry, match in _admitted_entries(
entries, filters, links, favorites, image_tags
):
total += 1
if match is not None:
positives.add(match[2])
@@ -2131,35 +2210,23 @@ class ImageRoutesMixin:
)
return matches
def _select_workspace_page(
self,
listings: list[tuple],
filters: dict,
after: tuple | None,
limit: int,
scope: str,
def _workspace_admission(
self, listings: list[tuple], filters: dict, *, read_links: bool, strict=False
) -> tuple:
"""One keyset page of the filtered listing (blocking; run in executor).
"""The ordered entries and what admits them (blocking; run in executor).
One SELECT (get_image_link_matches) reads every linked positive of the
listing: the filters, the rating sort and the positive count share it.
A plain listing (no ``q``, category, tags or rating sort) reads it only
for its first page; its cursor pages read no links and the returned
count is _COUNT_OMITTED, so the response leaves ``positive_count`` out
and the frontend keeps the first page's value. On that plain first
page a failed read leaves the listing usable and the count unknown
(None); so does a failed read under ``q`` alone, whose search then
matches filenames only (as before the linked search existed).
Returns ``(entries, key, links, image_tags, favorites)``: the listing in
the filters' order, its keyset key, and the link, image-tag and favorite
maps _admitted_entries reads. One SELECT (get_image_link_matches) reads
every linked positive when *read_links*. A failed read raises when the
filters need it (category, tags, the rating sort) or *strict*; else the
listing stays usable without links (a search then matches filenames
only). A failed image-tag read raises under a Tags filter; a search then
matches without image tags.
"""
order, category, tags = filters["order"], filters["category"], filters["tags"]
needed = (
bool(filters["query"])
or category is not None
or bool(tags)
or order == "rating"
)
links = None
if needed or after is None:
if read_links:
try:
links = self.db.get_image_link_matches(
filters["text"],
@@ -2169,10 +2236,7 @@ class ImageRoutesMixin:
wanted=self._workspace_wanted(listings),
)
except Exception:
# Category, tags and the rating sort cannot be served without
# the links. A plain listing and a filename search can: the
# search then matches filenames only, and the count is None.
if category is not None or tags or order == "rating":
if strict or category is not None or tags or order == "rating":
raise
self.logger.warning(
"Workspace prompt links could not be read; %s",
@@ -2188,8 +2252,6 @@ class ImageRoutesMixin:
try:
image_tags = self._image_tag_matches(listings, filters)
except Exception:
# A Tags filter cannot be served without them (as without the
# links); a search then matches without image tags.
if tags:
raise
self.logger.warning(
@@ -2228,6 +2290,41 @@ class ImageRoutesMixin:
def key(entry):
return _workspace_key(entry, order)
favorites = (
self._workspace_favorite_paths(listings) if filters["favorites"] else None
)
return entries, key, links, image_tags, favorites
def _select_workspace_page(
self,
listings: list[tuple],
filters: dict,
after: tuple | None,
limit: int,
scope: str,
) -> tuple:
"""One keyset page of the filtered listing (blocking; run in executor).
The filters, the rating sort and the positive count share one link
read (_workspace_admission). A plain listing (no ``q``, category, tags
or rating sort) reads it only for its first page; its cursor pages read
no links and the returned count is _COUNT_OMITTED, so the response
leaves ``positive_count`` out and the frontend keeps the first page's
value. On that plain first page a failed read leaves the listing usable
and the count unknown (None); so does a failed read under ``q`` alone,
whose search then matches filenames only (as before the linked search
existed).
"""
order = filters["order"]
needed = (
bool(filters["query"])
or filters["category"] is not None
or bool(filters["tags"])
or order == "rating"
)
entries, key, links, image_tags, favorites = self._workspace_admission(
listings, filters, read_links=needed or after is None
)
start = 0
if after is not None:
if order == "name-desc":
@@ -2235,9 +2332,6 @@ class ImageRoutesMixin:
start = len(entries) - bisect_left(ascending, after, key=key)
else:
start = bisect_right(entries, after, key=key)
favorites = (
self._workspace_favorite_paths(listings) if filters["favorites"] else None
)
total, page, positive_count = _filtered_page(
entries, start, limit + 1, filters, links, favorites, image_tags
)
@@ -2254,6 +2348,61 @@ class ImageRoutesMixin:
positive_count,
)
def _select_workspace_matches(
self, listings: list[tuple], filters: dict, cap: int
) -> ListingSelection:
"""Every admitted entry, up to *cap*, as a ListingSelection (blocking).
The links are read strictly: the positives must be the listing's own.
"""
entries, _key, links, image_tags, favorites = self._workspace_admission(
listings, filters, read_links=True, strict=True
)
targets, positives, unlinked, scopes = [], {}, 0, {}
for _position, entry, match in _admitted_entries(
entries, filters, links, favorites, image_tags
):
if len(targets) == cap:
return ListingSelection(targets, list(positives), unlinked, True)
scope = scopes.get(entry.index)
if scope is None:
scope = scopes[entry.index] = self._gallery_root_scope(
entry.root, entry.index
)
relative = entry.path.relative_to(entry.root).as_posix()
targets.append((entry.index, scope, relative))
if match is None or match[2] is None:
unlinked += 1
else:
positives.setdefault(match[2])
return ListingSelection(targets, list(positives), unlinked, False)
async def resolve_listing_filter(
self, params: Mapping, cap: int = MAX_LISTING_SELECTION
) -> ListingSelection:
"""The images a parse_listing_filter result admits right now (#392).
Raises ValueError for a root outside the current roots, and
ListingSourceChanged when the filter's ``root_scope`` no longer names
its root (the sources were renumbered since the listing).
"""
output_dirs = list(self._get_all_output_dirs())
root_index = _workspace_root(params, output_dirs)
error = self._gallery_scope_error(params, output_dirs)
if error is not None:
if error.status == 409:
raise ListingSourceChanged()
raise ValueError("Invalid gallery root scope")
filters = _workspace_filters(params)
listings = [
(index, root, await self._get_gallery_files(root))
for index, root in enumerate(output_dirs)
if root_index is None or index == root_index
]
return await self._run_in_executor(
self._select_workspace_matches, listings, filters, cap
)
async def serve_image(self, request):
"""Serve the actual image file using streamed FileResponse."""
try:
py/api/prompts.py
+62
View File
@@ -1178,6 +1178,8 @@ class PromptRoutesMixin:
return web.json_response(
{"success": False, "error": "Expected a JSON object"}, status=400
)
if "images_filter" in data:
return await self._bulk_category_by_filter(data)
if not data.get("prompt_ids"):
return web.json_response(
{"success": False, "error": "No prompt IDs provided"}, status=400
@@ -1214,6 +1216,66 @@ class PromptRoutesMixin:
status=500,
)
async def _bulk_category_by_filter(self, data: dict) -> web.Response:
"""Set the category on every linked positive a listing filter admits.
#392 "Select all N matching": ``{images_filter, category}``, the filter
being the Images listing's own query (parse_listing_filter), resolved
now. The answer adds ``prompt_count`` (the distinct linked positives)
and ``skipped`` (matching images without one); none linked changes
nothing.
"""
from .images import (
MAX_LISTING_SELECTION,
ListingSourceChanged,
parse_listing_filter,
)
def refused(message, **extra):
return web.json_response(
{"success": False, "error": message, **extra}, status=400
)
if set(data) != {"images_filter", "category"}:
return refused("Request must contain exactly category, images_filter")
category = data["category"]
try:
params = parse_listing_filter(data["images_filter"])
validate_category(category)
selection = await self.resolve_listing_filter(params)
except ListingSourceChanged:
return web.json_response(
{
"success": False,
"code": "source-changed",
"error": "The source changed. Refresh the image collection.",
},
status=409,
)
except ValueError as exc:
return refused(str(exc))
if selection.truncated:
return refused(
f"Narrow the view to at most {MAX_LISTING_SELECTION:,} images.",
code="too-many-images",
)
category = (category or "").strip()
ids = selection.positive_ids
updated = (
await self._run_in_executor(self.db.bulk_set_category, ids, category)
if ids
else 0
)
return web.json_response(
{
"success": True,
"message": f"Set category for {updated} prompts",
"updated_count": updated,
"prompt_count": len(ids),
"skipped": selection.unlinked,
}
)
def _all_prompts_for_export(self):
"""Every prompt, fetched in EXPORT_PAGE_SIZE pages (blocking)."""
prompts, offset = [], 0