Compare commits
12
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
71b55bc630 | ||
|
|
4c9cb7f5ea | ||
|
|
56d1072e58 | ||
|
|
e71d0ca6b6 | ||
|
|
6f442bf423 | ||
|
|
bf2fa3f50e | ||
|
|
2e9322d5c4 | ||
|
|
a1377cec36 | ||
|
|
fa98d0644d | ||
|
|
043403d09e | ||
|
|
7e11d9cae2 | ||
|
|
3295ff8b20 |
@@ -0,0 +1,21 @@
|
||||
# Left out of the Comfy Registry package (comfy node publish). Git clones still
|
||||
# have everything. tests/test_package_contents.py holds these rules to what
|
||||
# ComfyUI needs at run time; keep each one anchored to the root with a leading /.
|
||||
|
||||
# Development folders
|
||||
/tests/
|
||||
/tools/
|
||||
/docs/
|
||||
/.github/
|
||||
|
||||
# Screenshots used only by README-legacy.md on GitHub
|
||||
/images/
|
||||
|
||||
# Development files
|
||||
/TESTING.md
|
||||
/Makefile
|
||||
/requirements-dev.txt
|
||||
/.coveragerc
|
||||
/.flake8
|
||||
/.pre-commit-config.yaml
|
||||
/.gitignore
|
||||
@@ -17,6 +17,11 @@ updates:
|
||||
prefix: "deps(pip)"
|
||||
include: "scope"
|
||||
rebase-strategy: "auto"
|
||||
# These floors follow ComfyUI's requirements.txt, never the newest release.
|
||||
ignore:
|
||||
- dependency-name: "Pillow"
|
||||
- dependency-name: "aiohttp"
|
||||
- dependency-name: "numpy"
|
||||
# Optional: group non-breaking updates to reduce PR noise
|
||||
groups:
|
||||
python-minor-and-patch:
|
||||
|
||||
@@ -80,7 +80,7 @@ jobs:
|
||||
# pulled them from Ubuntu's apt mirror on every run, and three runs (#349, #378, #382) crawled at
|
||||
# ~26 kB/s for 11-12 minutes until the job hit its limit after every test had passed.
|
||||
container:
|
||||
image: mcr.microsoft.com/playwright:v1.64.1-noble
|
||||
image: mcr.microsoft.com/playwright:v1.64.1-noble@sha256:47b510dd9230bf0b9bfc9b20102241e577ea7a99334ab4c3ca19f19770e9b8ea
|
||||
options: --user 1001
|
||||
timeout-minutes: 24
|
||||
steps:
|
||||
|
||||
@@ -53,7 +53,7 @@ jobs:
|
||||
# same pinned Playwright image as local runs (tag = @playwright/test in tools/vendor/package-lock.json;
|
||||
# tests/vendor/browser-image.test.mjs checks it). Browsers and fonts come from the image.
|
||||
container:
|
||||
image: mcr.microsoft.com/playwright:v1.64.1-noble
|
||||
image: mcr.microsoft.com/playwright:v1.64.1-noble@sha256:47b510dd9230bf0b9bfc9b20102241e577ea7a99334ab4c3ca19f19770e9b8ea
|
||||
options: --user 1001
|
||||
timeout-minutes: 20
|
||||
# Public-repo ubuntu-latest runners have 4 vCPUs: three Playwright workers, each with its own test
|
||||
@@ -121,7 +121,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
# Same image as the shards: the raw files name sources by the container's workspace path.
|
||||
container:
|
||||
image: mcr.microsoft.com/playwright:v1.64.1-noble
|
||||
image: mcr.microsoft.com/playwright:v1.64.1-noble@sha256:47b510dd9230bf0b9bfc9b20102241e577ea7a99334ab4c3ca19f19770e9b8ea
|
||||
options: --user 1001
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
@@ -192,7 +192,7 @@ jobs:
|
||||
# same pinned Playwright image as local runs (tag = @playwright/test in tools/vendor/package-lock.json;
|
||||
# tests/vendor/browser-image.test.mjs checks it). Browsers and fonts come from the image.
|
||||
container:
|
||||
image: mcr.microsoft.com/playwright:v1.64.1-noble
|
||||
image: mcr.microsoft.com/playwright:v1.64.1-noble@sha256:47b510dd9230bf0b9bfc9b20102241e577ea7a99334ab4c3ca19f19770e9b8ea
|
||||
options: --user 1001
|
||||
timeout-minutes: 20
|
||||
# Public-repo ubuntu-latest runners have 4 vCPUs: three Playwright workers, each with its own test
|
||||
|
||||
@@ -12,10 +12,10 @@ Prompt Manager saves your prompts as you generate, links each image back to the
|
||||
|
||||
## What's new: rebuilt from the ground up
|
||||
|
||||
- **A new UI.** One app, the workspace, replaces the separate dashboard, gallery and metadata pages. The old pages still work if you want them.
|
||||
- **A new UI.** One app, the workspace, replaces the separate dashboard, gallery and metadata pages.
|
||||
- **Classic and Nodes 2.0.** The nodes work in both ComfyUI renderers. CI starts a real ComfyUI and checks both.
|
||||
- **Tested end to end.** Unit tests, browser tests in Chromium, Firefox, WebKit and touch emulation, and tests against a real ComfyUI.
|
||||
- **Mac, Linux and Windows.** The Python suite runs on all three, on Python 3.12, 3.13 and 3.14.
|
||||
- **Linux and Windows.** The Python suite runs on both, on Python 3.12, 3.13 and 3.14. macOS should work but is not tested.
|
||||
|
||||
## See it
|
||||
|
||||
@@ -212,9 +212,9 @@ Set `PROMPT_MANAGER_DATA_DIR` to an absolute folder to keep the library somewher
|
||||
- **Older images:** a one-time notice offers a metadata scan that links them to prompts and negatives. Scans don't count as new runs.
|
||||
- **Going back isn't supported.** Workflows are now stored once, and an older version shows new records without their workflow or prompt graph. Back up first in System > Backup & export. See [the old README's changelog](README-legacy.md#changelog) for the manual way back.
|
||||
|
||||
## The old pages still work
|
||||
## The old pages will be removed
|
||||
|
||||
They stay at their URLs, and System's Overview section links to them:
|
||||
They still work at their URLs for now and will be removed in a future release. Everything they do is moving to the workspace.
|
||||
|
||||
| Page | URL |
|
||||
| --- | --- |
|
||||
|
||||
+86
-8
@@ -77,7 +77,7 @@ PM_TEST_WORKERS=8 npm run --prefix tools/vendor test:coverage # whole
|
||||
npm run --prefix tools/vendor test:coverage:shard -- 1/4 # node units, viewer suite, workflows 1/4
|
||||
npm run --prefix tools/vendor test:coverage:shard -- 2/4 # workflows 2/4 (likewise 3/4, 4/4)
|
||||
npm run --prefix tools/vendor coverage:merge # report and baseline check of every raw file
|
||||
npm run --prefix tools/vendor test:workflows -- --project webkit --shard 2/2
|
||||
npm run --prefix tools/vendor test:workflows -- --project webkit --shard 2/3
|
||||
```
|
||||
|
||||
`coverage:merge` refuses an empty raw directory; it does not count shards, so run
|
||||
@@ -111,8 +111,11 @@ distributions, browser system dependencies may need manual installation. For a
|
||||
quick browser run, use `test:workflows -- --project chromium`.
|
||||
|
||||
CI runs the browser jobs inside the pinned image
|
||||
`mcr.microsoft.com/playwright:v<@playwright/test version>-noble`
|
||||
`mcr.microsoft.com/playwright:v<@playwright/test version>-noble@sha256:<digest>`
|
||||
(`tests/vendor/browser-image.test.mjs` keeps the tag in step with the lockfile).
|
||||
The digest is part of the pin, because Microsoft re-pushes tags: `v1.64.1-noble`
|
||||
moved from Node 24 to Node 26 on 2026-11-01 (#609). To bump it, change the tag and
|
||||
digest together and check the new image's Node against the Node 24 guard.
|
||||
Concept comparisons measure geometry, and geometry follows the fonts a browser
|
||||
falls back to: the concept names Inter, which neither the image nor a stock
|
||||
runner ships. On a plain `ubuntu-latest` runner the fallback is DejaVu Sans,
|
||||
@@ -190,8 +193,8 @@ waiting for the `frontend coverage` jobs.
|
||||
|
||||
The `viewerjs` check runs on every PR, including backend-only changes. It is an
|
||||
aggregate job. It requires the four `frontend coverage (shard N/4)` jobs and
|
||||
`frontend coverage (merge)` to succeed on every event. The four
|
||||
`page workflows (firefox|webkit, shard N/2)` jobs run only in the merge queue,
|
||||
`frontend coverage (merge)` to succeed on every event. The six
|
||||
`page workflows (firefox|webkit, shard N/3)` jobs run only in the merge queue,
|
||||
on `main` and on `workflow_dispatch` (see "Pull requests and the merge queue").
|
||||
`viewerjs` requires them to be skipped on a pull request and to succeed
|
||||
everywhere else. Any other failed, cancelled or skipped job fails it. Every browser job runs with `PM_TEST_WORKERS: 3` (the public
|
||||
@@ -665,6 +668,10 @@ Browser tests: `workspace-stale-source.spec.mjs` (`/__test__/sources/reverse` an
|
||||
`/__test__/scope-key/rotate`, both undone by every reset); the manual-Refresh
|
||||
recovery specs in `workspace-images.spec.mjs` answer every rescope 409.
|
||||
|
||||
## Registry package
|
||||
|
||||
`comfy node publish` zips the tracked files minus `.comfyignore`, so a new development file or folder at the root ships until it is listed there. `tests/test_package_contents.py` keeps each rule anchored to the root, checks that development paths are left out and that nothing ComfyUI needs at run time is. Git clones are unaffected.
|
||||
|
||||
## Windows portability preflight
|
||||
|
||||
Before changing paths, SQLite, streams, restore or cleanup:
|
||||
@@ -2728,7 +2735,7 @@ the grid) and D43 (sorted chips, checked against the concept with its chips sort
|
||||
D44 needs no mapping: the comparison runs with no Refine filter active, so the button
|
||||
reads "Filters" as the concept's does. Nothing else follows the grid: the legacy
|
||||
"Current prompt library" link is removed from the shelf; the legacy Library stays at
|
||||
`/prompt_manager/admin` by direct URL, linked only from System (#525, D165)
|
||||
`/prompt_manager/admin` by direct URL (#525, #620)
|
||||
(`tests/browser/workspace.spec.mjs`). `library comparison detects
|
||||
mutations` injects `.card-footer { padding: 2px }` and removes `.card-meta .chip.red`.
|
||||
|
||||
@@ -3236,7 +3243,7 @@ npm run --prefix tools/vendor test:workflows -- history-concept.spec.mjs \
|
||||
|
||||
`#/inspect` ("Inspect a file") is a native workspace route; it no longer hands off to
|
||||
the legacy metadata page, which stays at its URL
|
||||
(`/prompt_manager/gallery.html`) linked only from System (#525, D165). The page is the concept's `#metadata`
|
||||
(`/prompt_manager/gallery.html`) at its URL only (#525, #620). The page is the concept's `#metadata`
|
||||
(app.js `renderMetadata` a740–748, selection a1715–1729; style.css 709–722, 1180–1194,
|
||||
1543–1545, 1559–1564) in the concept's own classes (`.surface`, `.file-drop`,
|
||||
`.spaced-20`, `.local-file-preview`, all from the synced `concept.css`), with no
|
||||
@@ -4658,6 +4665,9 @@ the per-section hand-off panel, was retired by S12 (#329): every section is nati
|
||||
concept row's), `skipped tags add, preview, remove, save and reload (D202)`, `skipped
|
||||
tags preview checks library tags only …` (workspace-system-tagging.spec) and
|
||||
`suggestions omit skipped tags and say how many were hidden` (workspace-tagging.spec).
|
||||
- **D205** (Prompt library, #620, beyond the concept): the sort menu appends Recently used, Most used and Name Z–A
|
||||
after Oldest first. These are the old Dashboard's sorts, kept before its removal because the community asked for
|
||||
them. A prompt never run sorts last in both use orders.
|
||||
- **D203** (Auto-tagging, #431, beyond the concept): "Saved instructions" replaces the
|
||||
concept's "LLM tag template" field (the mock's state 2): a `.setting-block`
|
||||
(`data-setting="tag-instructions"`, `data-key="autotag_instructions"`) with the label,
|
||||
@@ -5899,6 +5909,73 @@ are closed before `os.replace`; renames stay inside one folder; folder identity
|
||||
folder are refused. The browser test server closes its own log handlers before its temp
|
||||
root (which now holds the logs) is removed.
|
||||
|
||||
### Thumbnail location contract (#320)
|
||||
|
||||
Previews live in each source's `thumbnails/` (the default) or in one folder, `<folder>/<source key>/…`. Every reader,
|
||||
writer and remover asks `py/api/preview_store.py`; `tests/test_preview_store.py` fails on a `thumbnails` path join
|
||||
anywhere else. The setting is `thumbnails.location` in config.json, and only the Move thumbnails job writes it: POST
|
||||
`/settings` answers 409 `location-needs-move`.
|
||||
|
||||
- **Keys and marker.** A key is a slug plus a hash of the source's identity. The folder's
|
||||
`.promptmanager-thumbnails.json` stores each key with its identity and install. Only a single `<slug>-<hex>` name
|
||||
is ever joined onto the folder, and anything else in a marker is dropped. Every write and delete passes
|
||||
`preview_store.may_write`: below a key folder, with no link, junction or reparse point on the way
|
||||
(`tests/test_preview_store_folder.py`, including tampered markers).
|
||||
- **Unavailable.** A folder is unavailable when it is missing, is a link, holds no marker or holds another
|
||||
folder's marker. A probe checks it every 5 s, and a store call that fails or stalls marks it at once. While it is
|
||||
unavailable:
|
||||
- originals are served and nothing is written anywhere;
|
||||
- generate, convert, clear and prune answer 409 `location-unavailable`;
|
||||
- the folder is never made again except by Recreate folder.
|
||||
|
||||
The configured location is in force before any request. A bad value is `invalid`, never a fall-back to the
|
||||
sources.
|
||||
- **Move thumbnails.** The job is a `thumbnails` job with `scope.mode` `move` (`tests/test_thumbnail_move.py`). It
|
||||
runs in this order:
|
||||
1. The review answers `POST /images/thumbnails/plan`; not enough space answers 507.
|
||||
2. A token starts the job.
|
||||
3. Automatic previews are held.
|
||||
4. Copies run on the job pool in chunks of 2 or 4 files, each with its size checked and its mtime rounded up to
|
||||
2 s.
|
||||
5. One config commit makes the switch and records the old stores under `cleanup`. The job cannot be cancelled
|
||||
from here.
|
||||
6. The old copies are removed.
|
||||
|
||||
If it fails or is cancelled before the switch, nothing changes. A crash leaves
|
||||
`<data>/data/thumbnail-move.json`, which the next start undoes. Old copies still held open are removed by the
|
||||
next start's "Move thumbnails (finishing)" job. Leaving a folder also removes the previews of this install's
|
||||
sources that are no longer listed (Q320-8).
|
||||
- **Remove thumbnails of removed sources** (`scope.mode` `prune`). It is offered in Thumbnail maintenance only in
|
||||
folder mode, and it touches only this install's keys.
|
||||
- **Page tests.** `tests/browser/workspace-thumbnail-location.spec.mjs` covers the row, both moves, the in-place
|
||||
refusal, the confirmation, the notice with Recreate and with Use this folder. The legacy gallery runs once in
|
||||
folder mode (`workflows.spec.mjs`). The hooks are `tests/browser/thumbnail_location_fixture.py`.
|
||||
|
||||
Beyond the concept, against the approved static mock `docs/egm/post-3.4.1/320-mock`:
|
||||
|
||||
- D196: the Thumbnail location row in System › Sources & scans, after Thumbnail generation.
|
||||
- The status line is "Now: inside each source." or "Now: one folder, <path>.". It does not show the mock's
|
||||
"(3 sources, 8,512 thumbnails)" counts, because counting means walking every preview on each view. It says
|
||||
"Using the default folder next to your library." and names old copies that wait to be removed.
|
||||
- D197: the Move thumbnails review uses the shared review.
|
||||
- A grid value may be a lead sentence with a list (`[lead, items]`).
|
||||
- Sources no longer listed appear under "Removed after the move", where the mock has a "Not moved" row
|
||||
(maintainer answer to Q320-8: remove).
|
||||
- The Free space line is "<size> free", without naming the disk.
|
||||
- D198: Activity's view of `mode: move` and `mode: prune`.
|
||||
- Tiles for a move: Moved, Already there, Left in place, Errors. Tiles for a prune: Removed, Left in place,
|
||||
Errors.
|
||||
- The step line under the bar is `#job-phase`.
|
||||
- Cancel shows only until the switch.
|
||||
- D199: the unavailable notice on Overview and Sources & scans.
|
||||
- Recreate folder appears for a missing folder, after a confirmation.
|
||||
- Use this folder appears for another folder's marker.
|
||||
- Change location focuses the row; on Overview it opens Sources & scans.
|
||||
- D200: Thumbnail maintenance's "Remove thumbnails of removed sources" scope, with the plan line and a grid of
|
||||
key folders.
|
||||
- D201: Move library's review adds "Thumbnails: Stay in <path>. Move them in System › Sources & scans." in folder
|
||||
mode.
|
||||
|
||||
### Backup and restore contract (#249 System S8)
|
||||
|
||||
A restore is staged, then published (`database/restore_stage.py`). The upload is
|
||||
@@ -7418,7 +7495,7 @@ Deploying this backend needs a ComfyUI restart (Python and route changes).
|
||||
|
||||
`#/tags` ("Tags & categories", page title "Tags") is a native workspace route; it no
|
||||
longer hands off to the legacy tag manager. That page stays at
|
||||
`/prompt_manager/admin#/tags` linked only from System (#525, D165); tag search and sort, the untagged
|
||||
`/prompt_manager/admin#/tags` at its URL only (#525, #620); tag search and sort, the untagged
|
||||
entry (D149) and multi-tag OR (the Library's Tag match, D150) moved to the workspace
|
||||
(#444); bulk prompt delete stays on the legacy admin page (bulk tags: the Prompt
|
||||
library's selection bar, D151). A prompt's category is changed in the Prompt library
|
||||
@@ -7550,7 +7627,8 @@ The legacy pages (`/prompt_manager/admin`, `admin#/tags`, `gallery`, `gallery.ht
|
||||
their URLs, and admin's Metadata and Gallery buttons still open their legacy pages (maintainer
|
||||
ruling; pinned by `legacy-system-links.spec.mjs`).
|
||||
|
||||
- **D165** (beyond the concept, maintainer ruling 2026-10-30): System's Overview section ends
|
||||
- **D165 retired** (#620): the old pages show their removal notice since 3.5.1, so System no longer links them and the load failure below offers a reload and ComfyUI. The original ruling:
|
||||
**D165** (beyond the concept, maintainer ruling 2026-10-30): System's Overview section ends
|
||||
with one small line, "Legacy pages: Dashboard · Tags · Gallery · Metadata viewer"
|
||||
(`.legacy-pages`, new tabs), outside `.settings-layout`. The concept has no such line;
|
||||
`mapOverviewConcept` adds it to the page tree.
|
||||
|
||||
+21
-5
@@ -92,10 +92,21 @@ try:
|
||||
routes = config.routes
|
||||
api = PromptManagerAPI()
|
||||
api.add_routes(routes)
|
||||
# Temp files of preview writes a crash interrupted (#317), off the
|
||||
# import path: the walk may cover a large thumbnails folder.
|
||||
|
||||
# The thumbnail location's startup check (#320), then the temp files of
|
||||
# preview writes a crash interrupted (#317), off the import path: the
|
||||
# walk may cover a large thumbnails folder.
|
||||
# #320: the configured thumbnail location is in force before any request
|
||||
# (no I/O); the thread below checks it.
|
||||
api.use_preview_location()
|
||||
|
||||
def _preview_startup():
|
||||
api.start_preview_location()
|
||||
api.recover_thumbnail_move() # an interrupted move, before the sweep
|
||||
api.sweep_thumbnail_temps()
|
||||
|
||||
threading.Thread(
|
||||
target=api.sweep_thumbnail_temps,
|
||||
target=_preview_startup,
|
||||
name="PromptManagerPreviewTempSweep",
|
||||
daemon=True,
|
||||
).start()
|
||||
@@ -108,6 +119,8 @@ try:
|
||||
# Build each output root's Images listing once the server's loop runs,
|
||||
# so the first open does not wait for the walk (#348).
|
||||
host_app.on_startup.append(api.warm_gallery_listings)
|
||||
# #320: old thumbnails a move left to remove, once it is checked.
|
||||
host_app.on_startup.append(api.finish_thumbnail_move)
|
||||
if host_app is not None and hasattr(host_app, "on_shutdown"):
|
||||
# Open Needs review change streams end before the server waits on them.
|
||||
host_app.on_shutdown.append(api.close_review_events)
|
||||
@@ -448,8 +461,11 @@ except Exception as e:
|
||||
|
||||
__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"]
|
||||
|
||||
# One startup line. ASCII only: a Windows console using a legacy code page
|
||||
# cannot print emoji, and a UnicodeEncodeError here would abort loading the node.
|
||||
# One startup line, printed at import on purpose: it is the package's banner on
|
||||
# stdout, as ComfyUI's other custom nodes print theirs. Not a log record: the
|
||||
# console log handler hides INFO by default, so a logger would drop it. ASCII
|
||||
# only: a Windows console using a legacy code page cannot print emoji, and a
|
||||
# UnicodeEncodeError here would abort loading the node.
|
||||
print(
|
||||
f"\033[94m[ComfyUI-PromptManager] Version:\033[0m {get_version()}"
|
||||
f" - {len(NODE_CLASS_MAPPINGS)} tools loaded"
|
||||
|
||||
@@ -291,13 +291,6 @@ class AnnotationOperations:
|
||||
conn.execute(_BUMP_INDEX)
|
||||
return conn.execute(_REVISION, (byte_id,)).fetchone()[0]
|
||||
|
||||
def _current_set(self, conn, sha256, byte_id, revision) -> dict:
|
||||
return {
|
||||
"sha256": sha256,
|
||||
"tags": self._byte_tags(conn, byte_id),
|
||||
"tag_revision": revision,
|
||||
}
|
||||
|
||||
def read_image_tags(self, locator: str, sha256: str) -> dict:
|
||||
"""The stored set of these exact bytes, or an empty set at revision 0.
|
||||
|
||||
|
||||
+20
-25
@@ -16,18 +16,17 @@ check, and none of them carries a server path.
|
||||
|
||||
import datetime
|
||||
import logging
|
||||
import ntpath
|
||||
import os
|
||||
import sqlite3
|
||||
from pathlib import PurePosixPath, PureWindowsPath
|
||||
from urllib.parse import quote
|
||||
|
||||
try:
|
||||
from ..utils.hashing import register_prompt_functions
|
||||
from .models import normalize_image_path
|
||||
from .sqlite_uri import readonly_uri
|
||||
except ImportError:
|
||||
from utils.hashing import register_prompt_functions
|
||||
from database.models import normalize_image_path
|
||||
from database.sqlite_uri import readonly_uri
|
||||
|
||||
logger = logging.getLogger("prompt_manager.database.health")
|
||||
|
||||
@@ -89,25 +88,6 @@ def _counted(lines):
|
||||
return _result(count, detail)
|
||||
|
||||
|
||||
def readonly_uri(db_path, *, windows=None):
|
||||
"""The ``mode=ro`` SQLite URI of *db_path*, on this platform or the given one.
|
||||
|
||||
pathlib spells a UNC path ``\\\\server\\share\\x.db`` as
|
||||
``file://server/share/x.db``, a remote authority SQLite refuses to open.
|
||||
With an empty authority (``file:////server/share/x.db``) SQLite hands
|
||||
Windows the ``//server/share`` path, which it opens as the UNC path.
|
||||
"""
|
||||
if windows is None:
|
||||
windows = os.name == "nt"
|
||||
db_path = os.path.abspath(db_path)
|
||||
if not windows:
|
||||
return PurePosixPath(db_path).as_uri() + "?mode=ro"
|
||||
path = PureWindowsPath(db_path)
|
||||
if ntpath.splitdrive(str(path))[0].startswith("\\\\"):
|
||||
return "file:" + quote("//" + path.as_posix()) + "?mode=ro"
|
||||
return path.as_uri() + "?mode=ro"
|
||||
|
||||
|
||||
def _integrity(conn, _roots):
|
||||
rows = [
|
||||
row[0] for row in conn.execute(f"PRAGMA integrity_check({INTEGRITY_LIMIT})")
|
||||
@@ -199,6 +179,23 @@ def link_states(rows, roots, readable):
|
||||
check's ``missing_files`` and the maintenance missing-link plan both
|
||||
judge links this way, so their counts agree.
|
||||
"""
|
||||
for row, owner, state in _placed_links(rows, roots, readable):
|
||||
if state is None:
|
||||
state = "present" if os.path.exists(row[1]) else "missing"
|
||||
yield row, owner, state
|
||||
|
||||
|
||||
def held_links(rows, roots, readable):
|
||||
"""``(row, owner, state)`` for each link ``link_states`` holds back
|
||||
(``unavailable`` or ``outside``), without looking at any file (#586)."""
|
||||
for row, owner, state in _placed_links(rows, roots, readable):
|
||||
if state is not None:
|
||||
yield row, owner, state
|
||||
|
||||
|
||||
def _placed_links(rows, roots, readable):
|
||||
"""``link_states`` before any file is looked at: state None means the
|
||||
link lies under a readable source."""
|
||||
prefixes = [_root_prefix(root) for root in roots]
|
||||
for row in rows:
|
||||
owner = _image_owner(row[1], row[3], prefixes)
|
||||
@@ -206,10 +203,8 @@ def link_states(rows, roots, readable):
|
||||
yield row, None, "outside"
|
||||
elif not readable[owner]:
|
||||
yield row, owner, "unavailable"
|
||||
elif os.path.exists(row[1]):
|
||||
yield row, owner, "present"
|
||||
else:
|
||||
yield row, owner, "missing"
|
||||
yield row, owner, None
|
||||
|
||||
|
||||
def _unavailable_line(unavailable, *, partial):
|
||||
|
||||
@@ -97,6 +97,9 @@ _ORDERS = {
|
||||
"newest": ("COALESCE(julianday(p.created_at),0)", "DESC"),
|
||||
"oldest": ("COALESCE(julianday(p.created_at),0)", "ASC"),
|
||||
"name": (_NAME, "ASC"),
|
||||
"name-desc": (_NAME, "DESC"),
|
||||
"most-used": ("COALESCE(p.run_count,0)", "DESC"),
|
||||
"last-used": ("COALESCE(julianday(p.last_used_at),0)", "DESC"),
|
||||
"rating": ("COALESCE(p.rating,0)", "DESC"),
|
||||
}
|
||||
# Pair archive scopes for variant and image reads, as conditions on alias pp.
|
||||
|
||||
@@ -47,11 +47,11 @@ import sqlite3
|
||||
import time
|
||||
import types
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Callable, Dict, List, Optional, Tuple
|
||||
|
||||
from .models import BUSY_TIMEOUT_SECONDS, PromptModel
|
||||
from .sqlite_uri import readonly_uri
|
||||
|
||||
try:
|
||||
from ..utils import library_location as ll
|
||||
@@ -78,6 +78,10 @@ MESSAGES = {
|
||||
),
|
||||
"same-folder": "The new folder is the node folder itself.",
|
||||
"target-invalid": "The new folder cannot hold the library.",
|
||||
"data-dir-served": (
|
||||
"The new folder is inside Prompt Manager's thumbnail folder. Nothing was "
|
||||
"moved."
|
||||
),
|
||||
"target-has-library": (
|
||||
"The new folder already holds a Prompt Manager library. Nothing was moved."
|
||||
),
|
||||
@@ -318,7 +322,7 @@ def _verify_copy(temp: str, counts, documents) -> None:
|
||||
"verify-failed",
|
||||
f"{MESSAGES['verify-failed']} {detail}" if detail else None,
|
||||
)
|
||||
uri = Path(os.path.abspath(temp)).as_uri() + "?mode=ro"
|
||||
uri = readonly_uri(temp)
|
||||
try:
|
||||
with contextlib.closing(sqlite3.connect(uri, uri=True)) as conn:
|
||||
copied = _describe(conn)
|
||||
|
||||
+81
-23
@@ -43,7 +43,7 @@ from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from .execution_contexts import LIVE_HINT_CTES
|
||||
from .health import BATCH_ROWS, link_states, readable_roots, readonly_uri
|
||||
from .health import BATCH_ROWS, held_links, link_states, readable_roots, readonly_uri
|
||||
from .image_removal_schema import TABLE as MARKS_TABLE
|
||||
from .image_removal_schema import clear_marks, has_marks_table
|
||||
from .pairs import PairDependencyError
|
||||
@@ -130,6 +130,16 @@ _UNLINKED_PROTECTIONS_SQL = (
|
||||
+ "), 0) FROM prompts p WHERE NOT EXISTS "
|
||||
"(SELECT 1 FROM generated_images gi WHERE gi.prompt_id = p.id)"
|
||||
)
|
||||
# Each prompt no image links to, with which protections keep it (#586); the
|
||||
# ones with none are filtered out in Python, so each predicate runs once.
|
||||
_UNLINKED_KEPT_SQL = (
|
||||
"WITH "
|
||||
+ LIVE_HINT_CTES
|
||||
+ " SELECT p.id, "
|
||||
+ ", ".join(f"({predicate})" for predicate, _many, _one in PROTECTIONS)
|
||||
+ " FROM prompts p WHERE NOT EXISTS "
|
||||
"(SELECT 1 FROM generated_images gi WHERE gi.prompt_id = p.id) ORDER BY p.id"
|
||||
)
|
||||
|
||||
# Duplicate prompt groups, as the legacy cleanup finds them (case-insensitive
|
||||
# text; GROUP_CONCAT order is sorted by created_at afterwards).
|
||||
@@ -159,6 +169,14 @@ _PROMPT_STATE_SQL = (
|
||||
"ORDER BY id",
|
||||
)
|
||||
|
||||
# Why the missing-link cleanup holds a link back: (one, many) (#586).
|
||||
HELD_LINK_REASONS = {
|
||||
"unavailable": (
|
||||
"link under an unavailable source",
|
||||
"links under unavailable sources",
|
||||
),
|
||||
"outside": ("link outside the image sources", "links outside the image sources"),
|
||||
}
|
||||
NO_FILES_DELETED = "No image file is deleted."
|
||||
LINKS_ONLY = "Database links only; no image file is deleted."
|
||||
UNUSED_DETAIL = "No image, saved pair, Library state or recorded run uses it"
|
||||
@@ -172,6 +190,7 @@ CONFLICT_REASONS = {
|
||||
"groups have their own Library titles",
|
||||
),
|
||||
}
|
||||
PAIRED_DETAIL = "Kept: belongs to a saved pair"
|
||||
CONFLICT_DETAILS = {
|
||||
"collision": "Kept: two copies link the same image",
|
||||
"title": "Kept: a copy has its own Library title",
|
||||
@@ -413,6 +432,35 @@ def _source_name(root, index):
|
||||
return os.path.basename(str(root).rstrip("\\/")) or f"Source {index + 1}"
|
||||
|
||||
|
||||
def _kept_duplicates(paired, conflicted):
|
||||
"""``{prompt_id, reasons}`` for each prompt of a group kept whole (#586).
|
||||
|
||||
The reasons are the review's details without "Kept: ".
|
||||
"""
|
||||
held = [(ids, PAIRED_DETAIL) for ids in paired] + [
|
||||
(ids, CONFLICT_DETAILS[reason]) for ids, reason in conflicted
|
||||
]
|
||||
return [
|
||||
{"prompt_id": prompt_id, "reasons": [detail.removeprefix("Kept: ")]}
|
||||
for ids, detail in held
|
||||
for prompt_id in ids
|
||||
]
|
||||
|
||||
|
||||
def _kept_unused(conn):
|
||||
"""``{prompt_id, reasons}`` for each prompt the unused prune keeps (#586).
|
||||
|
||||
The reasons are the review's, one per protection that applies, in the
|
||||
order the plan names them.
|
||||
"""
|
||||
kept = []
|
||||
for row in conn.execute(_UNLINKED_KEPT_SQL):
|
||||
reasons = [one for flag, (_p, _m, one) in zip(row[1:], PROTECTIONS) if flag]
|
||||
if reasons:
|
||||
kept.append({"prompt_id": row[0], "reasons": reasons})
|
||||
return kept
|
||||
|
||||
|
||||
class MaintenanceOperations:
|
||||
"""Mixin for PromptDatabase: maintenance previews and their apply."""
|
||||
|
||||
@@ -592,7 +640,7 @@ class MaintenanceOperations:
|
||||
+ [
|
||||
{
|
||||
"label": _excerpt(texts.get(group[0])),
|
||||
"detail": "Kept: belongs to a saved pair",
|
||||
"detail": PAIRED_DETAIL,
|
||||
}
|
||||
for group in paired
|
||||
]
|
||||
@@ -670,6 +718,24 @@ class MaintenanceOperations:
|
||||
],
|
||||
)
|
||||
|
||||
def kept_links(self, roots):
|
||||
"""``{image_path, owner, reason}`` for each link the missing-link
|
||||
cleanup holds back under ``roots`` (blocking, reads only, no file
|
||||
opened); ``owner`` indexes ``roots`` (None outside every source).
|
||||
|
||||
``reason`` is the review's (``HELD_LINK_REASONS``), one link each.
|
||||
"""
|
||||
roots = [str(root) for root in roots]
|
||||
rows = _read_snapshot(self.model.db_path, self._link_rows)
|
||||
return [
|
||||
{
|
||||
"image_path": row[1],
|
||||
"owner": owner,
|
||||
"reason": HELD_LINK_REASONS[state][0],
|
||||
}
|
||||
for row, owner, state in held_links(rows, roots, readable_roots(roots))
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _links_fingerprint(candidates, stale=()):
|
||||
fingerprint = _Fingerprint("missing_links")
|
||||
@@ -689,23 +755,11 @@ class MaintenanceOperations:
|
||||
unavailable += 1
|
||||
elif state == "outside":
|
||||
outside += 1
|
||||
reasons = []
|
||||
if unavailable:
|
||||
reasons.append(
|
||||
_plural(
|
||||
unavailable,
|
||||
"link under an unavailable source",
|
||||
"links under unavailable sources",
|
||||
)
|
||||
)
|
||||
if outside:
|
||||
reasons.append(
|
||||
_plural(
|
||||
outside,
|
||||
"link outside the image sources",
|
||||
"links outside the image sources",
|
||||
)
|
||||
)
|
||||
reasons = [
|
||||
_plural(count, *HELD_LINK_REASONS[state])
|
||||
for state, count in (("unavailable", unavailable), ("outside", outside))
|
||||
if count
|
||||
]
|
||||
count = len(candidates)
|
||||
parts = []
|
||||
if count:
|
||||
@@ -809,7 +863,10 @@ class MaintenanceOperations:
|
||||
(``{prompt_id[, kept_id]}`` or ``{image_path}``). ``marks`` are a
|
||||
missing-links plan's stale removal marks; that apply also returns
|
||||
``marks``, how many it forgot. ``pair_metadata`` returns every filled
|
||||
prompt, uncapped (its job keeps the complete record, #550).
|
||||
prompt, uncapped (its job keeps the complete record, #550). The
|
||||
prompt operations also return ``kept``: every prompt the plan keeps,
|
||||
``{prompt_id, reasons}`` in the review's wording (#550, #586); the
|
||||
links a missing-links plan keeps are read by ``kept_links``.
|
||||
"""
|
||||
if operation not in MAINTENANCE_OPERATIONS or operation == "workflow_storage":
|
||||
raise ValueError(f"Unknown maintenance operation {operation!r}")
|
||||
@@ -835,10 +892,11 @@ class MaintenanceOperations:
|
||||
return appliers[operation](conn, fingerprint, targets, roots)
|
||||
|
||||
def _apply_duplicate_prompts(self, conn, fingerprint, targets, _roots):
|
||||
groups, _paired, _conflicted, current = self._duplicates_fingerprint(conn)
|
||||
groups, paired, conflicted, current = self._duplicates_fingerprint(conn)
|
||||
if current != fingerprint or groups != targets:
|
||||
raise PlanChanged("duplicate_prompts")
|
||||
changed, rows = 0, []
|
||||
kept = _kept_duplicates(paired, conflicted)
|
||||
for sorted_ids in groups:
|
||||
removed, _moved = self._merge_duplicate_group(conn, sorted_ids, strict=True)
|
||||
changed += removed
|
||||
@@ -846,7 +904,7 @@ class MaintenanceOperations:
|
||||
{"prompt_id": duplicate_id, "kept_id": sorted_ids[0]}
|
||||
for duplicate_id in sorted_ids[1:]
|
||||
)
|
||||
return {"changed": changed, "rows": rows[:MAX_RESULT_ROWS]}
|
||||
return {"changed": changed, "rows": rows[:MAX_RESULT_ROWS], "kept": kept}
|
||||
|
||||
def _apply_unused_components(self, conn, fingerprint, targets, _roots):
|
||||
ids, current = self._unused_fingerprint(conn)
|
||||
@@ -858,7 +916,7 @@ class MaintenanceOperations:
|
||||
{"prompt_id": prompt_id, "text": _excerpt(texts.get(prompt_id))}
|
||||
for prompt_id in ids[:MAX_RESULT_ROWS]
|
||||
]
|
||||
return {"changed": changed, "rows": rows}
|
||||
return {"changed": changed, "rows": rows, "kept": _kept_unused(conn)}
|
||||
|
||||
def _apply_missing_links(self, conn, fingerprint, targets, roots, marks):
|
||||
rows = []
|
||||
|
||||
+3
-4
@@ -14,6 +14,7 @@ from contextlib import closing
|
||||
from pathlib import Path
|
||||
from typing import Optional, Set, Tuple
|
||||
|
||||
from .sqlite_uri import readonly_uri
|
||||
from .managed_connection import (
|
||||
ManagedConnection,
|
||||
await_closed,
|
||||
@@ -1199,7 +1200,7 @@ class PromptModel:
|
||||
if header != SQLITE_HEADER:
|
||||
return False, "Not a SQLite database file"
|
||||
|
||||
uri = Path(os.path.abspath(path)).as_uri() + "?mode=ro"
|
||||
uri = readonly_uri(path)
|
||||
try:
|
||||
conn = sqlite3.connect(uri, uri=True)
|
||||
except sqlite3.Error as e:
|
||||
@@ -1306,9 +1307,7 @@ class PromptModel:
|
||||
replacement (for example, incompatible page sizes in WAL mode),
|
||||
propagate the error rather than copy files under other open owners.
|
||||
"""
|
||||
source = sqlite3.connect(
|
||||
Path(src_path).resolve().as_uri() + "?mode=ro", uri=True
|
||||
)
|
||||
source = sqlite3.connect(readonly_uri(Path(src_path).resolve()), uri=True)
|
||||
try:
|
||||
target = self.get_connection()
|
||||
# The target is written under backup's step too: mark it in use
|
||||
|
||||
@@ -42,6 +42,7 @@ from typing import Dict, Optional
|
||||
from . import workflow_documents as wd
|
||||
from .execution_history_schema import rotate_history_epoch
|
||||
from .safety_backup import take_safety_backup
|
||||
from .sqlite_uri import readonly_uri
|
||||
|
||||
logger = logging.getLogger("prompt_manager.database.restore_stage")
|
||||
|
||||
@@ -114,7 +115,7 @@ def file_digest(path: str) -> str:
|
||||
|
||||
|
||||
def _readonly(path: str) -> sqlite3.Connection:
|
||||
return sqlite3.connect(Path(path).resolve().as_uri() + "?mode=ro", uri=True)
|
||||
return sqlite3.connect(readonly_uri(Path(path).resolve()), uri=True)
|
||||
|
||||
|
||||
def _schema(path: str) -> list:
|
||||
|
||||
@@ -31,7 +31,15 @@ MAX_STATE_BYTES = 4096
|
||||
MAX_TAGS = 20
|
||||
MAX_TAG_LENGTH = 50
|
||||
_IMAGE_SORTS = ("newest", "oldest", "name", "name-desc", "rating")
|
||||
_LIBRARY_SORTS = ("newest", "oldest", "name", "rating")
|
||||
_LIBRARY_SORTS = (
|
||||
"newest",
|
||||
"oldest",
|
||||
"name",
|
||||
"name-desc",
|
||||
"most-used",
|
||||
"last-used",
|
||||
"rating",
|
||||
)
|
||||
_MEDIA = ("all", "image", "video")
|
||||
# How a Library view's Tags match (#444): every tag, any of them, or parents
|
||||
# without tags ("untagged" never goes with tags; the listing refuses both).
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
"""The read-only SQLite URI of a path, safe for Windows network shares."""
|
||||
|
||||
import ntpath
|
||||
import os
|
||||
from pathlib import PurePosixPath, PureWindowsPath
|
||||
from urllib.parse import quote
|
||||
|
||||
|
||||
def readonly_uri(db_path, *, windows=None):
|
||||
"""The ``mode=ro`` SQLite URI of *db_path*, on this platform or the given one.
|
||||
|
||||
pathlib spells a UNC path ``\\\\server\\share\\x.db`` as
|
||||
``file://server/share/x.db``, a remote authority SQLite refuses to open.
|
||||
With an empty authority (``file:////server/share/x.db``) SQLite hands
|
||||
Windows the ``//server/share`` path, which it opens as the UNC path.
|
||||
"""
|
||||
if windows is None:
|
||||
windows = os.name == "nt"
|
||||
db_path = os.path.abspath(db_path)
|
||||
if not windows:
|
||||
return PurePosixPath(db_path).as_uri() + "?mode=ro"
|
||||
path = PureWindowsPath(db_path)
|
||||
if ntpath.splitdrive(str(path))[0].startswith("\\\\"):
|
||||
return "file:" + quote("//" + path.as_posix()) + "?mode=ro"
|
||||
return path.as_uri() + "?mode=ro"
|
||||
@@ -112,20 +112,22 @@ QUERY_PLAN_STATEMENTS = {
|
||||
|
||||
|
||||
def _rewrite_row(conn, image_id, row, verified, checked):
|
||||
"""Store the row's inline texts as documents; returns (changed, protected, size).
|
||||
"""Store the row's inline texts as documents; returns (changed, kept, size).
|
||||
|
||||
``kept`` names the columns whose value stays inline (not text).
|
||||
|
||||
``verified`` maps document ids already read back in this batch to their
|
||||
text, and ``checked`` holds the reused ids ``store_text`` already
|
||||
checked, so a document shared by many rows is decompressed once.
|
||||
"""
|
||||
changes, replaced = {}, []
|
||||
protected = size = 0
|
||||
changes, replaced, kept = {}, [], []
|
||||
size = 0
|
||||
for index, (column, reference) in enumerate(TEXT_COLUMNS):
|
||||
value, current = row[index], row[index + len(TEXT_COLUMNS)]
|
||||
if value is None:
|
||||
continue
|
||||
if not storable(value):
|
||||
protected += 1
|
||||
kept.append(column)
|
||||
continue
|
||||
document_id = store_text(conn, value, checked)
|
||||
if document_id not in verified:
|
||||
@@ -143,7 +145,7 @@ def _rewrite_row(conn, image_id, row, verified, checked):
|
||||
[*changes.values(), image_id],
|
||||
)
|
||||
release_documents(conn, replaced)
|
||||
return bool(changes), protected, size
|
||||
return bool(changes), kept, size
|
||||
|
||||
|
||||
def migrate_batch(
|
||||
@@ -160,9 +162,10 @@ def migrate_batch(
|
||||
savepoint: a row that cannot be rewritten (``_row_error``) is rolled
|
||||
back alone, stays inline and is listed in ``errors``; anything else rolls
|
||||
the whole batch back and propagates. Returns ``{last_id, rows,
|
||||
protected, errors, done}``: the last row looked at, rows rewritten,
|
||||
inline values kept (not text), the ids kept by an error and whether
|
||||
there was nothing left to look at.
|
||||
protected, kept, errors, done}``: the last row looked at, rows
|
||||
rewritten, inline values kept (not text) and each of them as ``(image
|
||||
id, column)`` (#586), the ids kept by an error and whether there was
|
||||
nothing left to look at.
|
||||
"""
|
||||
if conn.in_transaction:
|
||||
raise RuntimeError("The migration needs its own transaction")
|
||||
@@ -174,6 +177,7 @@ def migrate_batch(
|
||||
"last_id": after_id,
|
||||
"rows": 0,
|
||||
"protected": 0,
|
||||
"kept": [],
|
||||
"errors": [],
|
||||
"done": not ids,
|
||||
}
|
||||
@@ -182,7 +186,7 @@ def migrate_batch(
|
||||
conn.execute("SAVEPOINT workflow_row")
|
||||
try:
|
||||
row = conn.execute(ROW_SQL, (image_id,)).fetchone()
|
||||
changed, protected, row_size = _rewrite_row(
|
||||
changed, kept, row_size = _rewrite_row(
|
||||
conn, image_id, row, verified, checked
|
||||
)
|
||||
except Exception as error:
|
||||
@@ -194,11 +198,12 @@ def migrate_batch(
|
||||
checked.clear()
|
||||
logger.warning(f"Image record {image_id} kept inline: {error}")
|
||||
result["errors"].append(image_id)
|
||||
changed, protected, row_size = False, 0, 0
|
||||
changed, kept, row_size = False, [], 0
|
||||
else:
|
||||
conn.execute("RELEASE workflow_row")
|
||||
result["rows"] += int(changed)
|
||||
result["protected"] += protected
|
||||
result["protected"] += len(kept)
|
||||
result["kept"].extend((image_id, column) for column in kept)
|
||||
result["last_id"] = image_id
|
||||
size += row_size
|
||||
if size >= max_bytes or _clock() - started >= max_seconds:
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 85 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 34 KiB |
@@ -3,7 +3,6 @@ PromptManager: Main custom node implementation that extends CLIPTextEncode
|
||||
with persistent prompt storage and search capabilities.
|
||||
"""
|
||||
|
||||
import time
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
if __package__:
|
||||
@@ -259,19 +258,6 @@ class PromptManager(PromptManagerBase, ComfyNodeABC):
|
||||
if not save_to_library:
|
||||
return result # #90: no metadata registration, no history text
|
||||
|
||||
# Register with ComfyUI integration for standard metadata compatibility
|
||||
node_id = f"promptmanager_{int(time.time() * 1000)}"
|
||||
self.comfyui_integration.register_prompt(
|
||||
node_id,
|
||||
encoding_text.strip(),
|
||||
{
|
||||
"category": category.strip() if category else None,
|
||||
"tags": extended_tags,
|
||||
"prompt_id": prompt_id,
|
||||
"prepend_text": prepend_text.strip() if prepend_text else None,
|
||||
"append_text": append_text.strip() if append_text else None,
|
||||
},
|
||||
)
|
||||
record_runtime_context(
|
||||
type(self).__name__, text, prepend_text, append_text, encoding_text
|
||||
)
|
||||
|
||||
@@ -19,7 +19,6 @@ except ImportError:
|
||||
|
||||
try:
|
||||
from .database.operations import PromptDatabase
|
||||
from .utils.comfyui_integration import get_comfyui_integration
|
||||
from .utils.image_monitor import get_image_monitor
|
||||
from .utils.prompt_tracker import get_prompt_tracker
|
||||
from .utils.hashing import generate_prompt_hash
|
||||
@@ -38,7 +37,6 @@ except ImportError:
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from database.operations import PromptDatabase
|
||||
from utils.comfyui_integration import get_comfyui_integration
|
||||
from utils.image_monitor import get_image_monitor
|
||||
from utils.prompt_tracker import get_prompt_tracker
|
||||
from utils.hashing import generate_prompt_hash
|
||||
@@ -86,7 +84,6 @@ class PromptManagerBase:
|
||||
self.db = PromptDatabase()
|
||||
self.prompt_tracker = get_prompt_tracker(self.db)
|
||||
self.image_monitor = get_image_monitor(self.db, self.prompt_tracker)
|
||||
self.comfyui_integration = get_comfyui_integration()
|
||||
|
||||
self._start_gallery_system()
|
||||
self.logger.debug(f"{self.__class__.__name__} node initialization completed")
|
||||
@@ -433,18 +430,6 @@ class PromptManagerBase:
|
||||
self.logger.error(f"Error searching prompts: {e}")
|
||||
return []
|
||||
|
||||
def search_prompts_api(self, search_text: str = "") -> List[Dict[str, Any]]:
|
||||
"""API method for JavaScript UI to search prompts."""
|
||||
return self._search_prompts(search_text=search_text)
|
||||
|
||||
def get_recent_prompts_api(self, limit: int = 20) -> List[Dict[str, Any]]:
|
||||
"""API method for JavaScript UI to get recent prompts."""
|
||||
try:
|
||||
return self.db.get_recent_prompts(limit=limit)
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error getting recent prompts: {e}")
|
||||
return []
|
||||
|
||||
def _start_gallery_system(self):
|
||||
"""Initialize and start the gallery monitoring system."""
|
||||
try:
|
||||
@@ -455,13 +440,6 @@ class PromptManagerBase:
|
||||
self.logger.error(f"Failed to start gallery system: {e}")
|
||||
self.logger.warning("Gallery features will be disabled")
|
||||
|
||||
def get_gallery_status(self) -> Dict[str, Any]:
|
||||
"""Get status of the gallery system."""
|
||||
return {
|
||||
"image_monitor": self.image_monitor.get_status(),
|
||||
"prompt_tracker": self.prompt_tracker.get_status(),
|
||||
}
|
||||
|
||||
def cleanup_gallery_system(self):
|
||||
"""Clean up gallery system resources.
|
||||
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
if __package__:
|
||||
@@ -173,19 +172,6 @@ class PromptManagerPair(PromptManagerFocused):
|
||||
self.logger.warning(f"Could not save or track paired prompt: {error}")
|
||||
if is_positive:
|
||||
self._mark_untracked(prompt, unique_id)
|
||||
try:
|
||||
self.comfyui_integration.register_prompt(
|
||||
f"promptmanager_pair_{time.time_ns()}",
|
||||
positive,
|
||||
{
|
||||
"prompt_id": prompt_id,
|
||||
"pair_id": pair_id if pair_used else None,
|
||||
"negative_text": negative_text,
|
||||
"category": category.strip() or None,
|
||||
},
|
||||
)
|
||||
except Exception as error:
|
||||
self.logger.warning(f"Could not register paired metadata: {error}")
|
||||
record_runtime_context(
|
||||
type(self).__name__,
|
||||
text,
|
||||
|
||||
@@ -3,7 +3,6 @@ PromptManagerText: A text-only version of PromptManager that outputs STRING
|
||||
without CLIP encoding, while maintaining all database and search features.
|
||||
"""
|
||||
|
||||
import time
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
if __package__:
|
||||
@@ -230,19 +229,6 @@ class PromptManagerText(PromptManagerBase, ComfyNodeABC):
|
||||
if not save_to_library:
|
||||
return result # #90: no metadata registration, no history text
|
||||
|
||||
# Register with ComfyUI integration for standard metadata compatibility
|
||||
node_id = f"promptmanagertext_{int(time.time() * 1000)}"
|
||||
self.comfyui_integration.register_prompt(
|
||||
node_id,
|
||||
final_text.strip(),
|
||||
{
|
||||
"category": category.strip() if category else None,
|
||||
"tags": extended_tags,
|
||||
"prompt_id": prompt_id,
|
||||
"prepend_text": prepend_text.strip() if prepend_text else None,
|
||||
"append_text": append_text.strip() if append_text else None,
|
||||
},
|
||||
)
|
||||
record_runtime_context(
|
||||
type(self).__name__, text, prepend_text, append_text, final_text
|
||||
)
|
||||
|
||||
+16
-60
@@ -11,6 +11,7 @@ Split into domain-specific mixins for maintainability:
|
||||
- JobRoutesMixin: the session job registry behind Activity (py/api/jobs.py)
|
||||
- ImageTagJobsMixin: bulk Add tags / Auto-tag as image-tags jobs
|
||||
- ThumbnailJobsMixin: thumbnail plan, thumbnail job and the gallery stream on it
|
||||
- ThumbnailLocationMixin: the thumbnail location check and payloads (#320)
|
||||
- ThumbnailConvertMixin: Convert outdated thumbnails (#317), a thumbnail job mode
|
||||
- AutoThumbnailMixin: previews made on first view, on capture and after a scan
|
||||
- ReviewRoutesMixin: the Needs review queue, item detail and decisions
|
||||
@@ -60,6 +61,11 @@ from .settings_routes import SettingsRoutesMixin
|
||||
from .job_routes import JobRoutesMixin
|
||||
from .image_tag_jobs import ImageTagJobsMixin
|
||||
from .thumbnail_jobs import ThumbnailJobsMixin
|
||||
from .thumbnail_location import ThumbnailLocationMixin
|
||||
from .thumbnail_move import ThumbnailMoveMixin
|
||||
from .thumbnail_move_plan import ThumbnailMovePlanMixin
|
||||
from .thumbnail_move_recovery import ThumbnailMoveRecoveryMixin
|
||||
from .thumbnail_prune import ThumbnailPruneMixin
|
||||
from .thumbnail_convert import ThumbnailConvertMixin
|
||||
from .thumbnail_auto import AutoThumbnailMixin
|
||||
from .system import SystemRoutesMixin
|
||||
@@ -83,7 +89,6 @@ from .source_rescope import SourceRescopeMixin
|
||||
|
||||
try:
|
||||
from ...database.operations import PromptDatabase
|
||||
from ...utils.file_prompt import parse_comfyui_prompt
|
||||
from ...utils.logging_config import get_logger
|
||||
from ...utils.native_memory import release_free_memory
|
||||
from ...utils.png_text import png_text
|
||||
@@ -95,7 +100,6 @@ except ImportError:
|
||||
0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
)
|
||||
from database.operations import PromptDatabase
|
||||
from utils.file_prompt import parse_comfyui_prompt
|
||||
from utils.logging_config import get_logger
|
||||
from utils.native_memory import release_free_memory
|
||||
from utils.png_text import png_text
|
||||
@@ -248,23 +252,6 @@ async def _read_json_body(request, max_bytes=JSON_BODY_MAX_BYTES):
|
||||
return data, None
|
||||
|
||||
|
||||
class _LongJobGuard:
|
||||
"""Async context manager handed out by ``_acquire_long_job``."""
|
||||
|
||||
def __init__(self, lock):
|
||||
self._lock = lock
|
||||
|
||||
async def __aenter__(self):
|
||||
# The caller checked ``locked()`` on this same loop iteration, so this
|
||||
# acquire never waits.
|
||||
await self._lock.acquire()
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
self._lock.release()
|
||||
return False
|
||||
|
||||
|
||||
# ── Gzip compression middleware ────────────────────────────────────────
|
||||
_GZIP_MIN_SIZE = 1024 # Only compress bodies larger than 1 KB
|
||||
_GZIP_TYPES = frozenset(
|
||||
@@ -338,6 +325,11 @@ class PromptManagerAPI(
|
||||
JobRoutesMixin,
|
||||
ImageTagJobsMixin,
|
||||
ThumbnailJobsMixin,
|
||||
ThumbnailLocationMixin,
|
||||
ThumbnailMovePlanMixin,
|
||||
ThumbnailMoveMixin,
|
||||
ThumbnailMoveRecoveryMixin,
|
||||
ThumbnailPruneMixin,
|
||||
ThumbnailConvertMixin,
|
||||
AutoThumbnailMixin,
|
||||
SystemRoutesMixin,
|
||||
@@ -415,10 +407,6 @@ class PromptManagerAPI(
|
||||
self._workspace_card_cache = {}
|
||||
# Favorite keys of each root's cached scan (images._favorite_listing).
|
||||
self._favorite_listing_cache = {}
|
||||
# Single-flight locks for long jobs, keyed by job family; see
|
||||
# _acquire_long_job(). Created lazily so no lock binds to an event
|
||||
# loop before the server's loop exists.
|
||||
self._long_jobs: dict = {}
|
||||
# This session's jobs (py/api/jobs.py): the metadata scan, and later
|
||||
# every other long operation the Activity page tracks.
|
||||
self._jobs = JobRegistry(self.logger)
|
||||
@@ -475,26 +463,6 @@ class PromptManagerAPI(
|
||||
_public_error = staticmethod(_public_error)
|
||||
_read_json_body = staticmethod(_read_json_body)
|
||||
|
||||
def _acquire_long_job(self, name):
|
||||
"""Claim the single-flight slot for a long job family, or None when busy.
|
||||
|
||||
Usage in a handler::
|
||||
|
||||
job = self._acquire_long_job("duplicates")
|
||||
if job is None:
|
||||
return self._long_job_busy_response("Duplicate scan")
|
||||
async with job:
|
||||
...
|
||||
|
||||
Families are free-form strings ("duplicates", "thumbnails",
|
||||
"autotag", ...); the lock is released when the ``async with`` block
|
||||
exits, including on exceptions. Must be called from the event loop.
|
||||
"""
|
||||
lock = self._long_jobs.setdefault(name, asyncio.Lock())
|
||||
if lock.locked():
|
||||
return None
|
||||
return _LongJobGuard(lock)
|
||||
|
||||
@staticmethod
|
||||
def _long_job_busy_response(label):
|
||||
"""409 envelope for a long job that is already running."""
|
||||
@@ -716,6 +684,7 @@ class PromptManagerAPI(
|
||||
# ── Register domain-specific routes from mixins ───────────────
|
||||
|
||||
self._register_pair_routes(routes)
|
||||
self._register_thumbnail_move_routes(routes)
|
||||
self._register_library_routes(routes)
|
||||
self._register_library_delete_routes(routes)
|
||||
self._register_vocabulary_routes(routes)
|
||||
@@ -852,20 +821,11 @@ class PromptManagerAPI(
|
||||
f"{url_quote(rel_path.as_posix(), safe='/')}"
|
||||
)
|
||||
|
||||
# Any preview the file owns, the current format first (#317)
|
||||
from . import thumbnail_settings
|
||||
from .prompts import thumbnail_url_for
|
||||
# Any preview the file owns, the current format first
|
||||
# (#317); no store I/O here in folder mode (#320).
|
||||
from .images import listing_thumbnail_url
|
||||
|
||||
existing = thumbnail_settings.existing_thumbnail(
|
||||
output_path, rel_path
|
||||
)
|
||||
thumbnail_url = None
|
||||
if existing is not None:
|
||||
thumb_rel = (
|
||||
thumbnail_settings.thumbnail_stem_rel(rel_path)
|
||||
+ existing.suffix
|
||||
)
|
||||
thumbnail_url = thumbnail_url_for(thumb_rel, existing)
|
||||
thumbnail_url = listing_thumbnail_url(output_path, rel_path)
|
||||
if thumbnail_url:
|
||||
image["thumbnail_url"] = thumbnail_url
|
||||
break # Found matching root, stop searching
|
||||
@@ -1076,7 +1036,3 @@ class PromptManagerAPI(
|
||||
f"Unreadable image {os.path.basename(str(image_path))}: {e}"
|
||||
)
|
||||
return {}
|
||||
|
||||
def _parse_comfyui_prompt(self, metadata):
|
||||
"""Parse ComfyUI and A1111 prompt data from metadata."""
|
||||
return parse_comfyui_prompt(metadata)
|
||||
|
||||
+3
-1
@@ -30,8 +30,10 @@ from .jobs import (
|
||||
)
|
||||
|
||||
try:
|
||||
from ...database.sqlite_uri import readonly_uri
|
||||
from ...utils.parallel import map_parallel
|
||||
except ImportError:
|
||||
from database.sqlite_uri import readonly_uri
|
||||
from utils.parallel import map_parallel
|
||||
|
||||
try:
|
||||
@@ -399,7 +401,7 @@ def _inspect_restore_upload(path):
|
||||
Returns:
|
||||
None when acceptable, otherwise a human-readable reason (blocking).
|
||||
"""
|
||||
uri = Path(os.path.abspath(path)).as_uri() + "?mode=ro"
|
||||
uri = readonly_uri(path)
|
||||
try:
|
||||
conn = sqlite3.connect(uri, uri=True)
|
||||
except sqlite3.Error as e:
|
||||
|
||||
+13
-24
@@ -19,8 +19,8 @@ from .jobs import (
|
||||
)
|
||||
|
||||
from .maintenance import disk_plan
|
||||
from .prompts import thumbnail_url_for
|
||||
from . import thumbnail_settings
|
||||
from .images import listing_thumbnail_url
|
||||
from . import preview_store
|
||||
from .. import model_gate
|
||||
|
||||
try:
|
||||
@@ -943,16 +943,6 @@ class AutotagRoutesMixin:
|
||||
return refusal.response
|
||||
return web.json_response({"success": True, "job": job.summary()}, status=202)
|
||||
|
||||
def _gated_runner(self, owner):
|
||||
"""Executor calls under the model gate *owner* holds (plain when None)."""
|
||||
if owner is None:
|
||||
return self._run_in_executor
|
||||
|
||||
async def run(func, *args, **kwargs):
|
||||
return await model_gate.run(owner, func, *args, **kwargs)
|
||||
|
||||
return run
|
||||
|
||||
def _resolve_allowed_image_path(self, raw_path: str) -> Optional[str]:
|
||||
"""Return the real path when it is a media file inside an output directory."""
|
||||
if Path(str(raw_path)).suffix.lower() not in _MEDIA_EXTENSIONS:
|
||||
@@ -1421,7 +1411,8 @@ class AutotagRoutesMixin:
|
||||
)
|
||||
|
||||
model_type = service.get_loaded_model_type()
|
||||
service.unload_model()
|
||||
# Off the loop; a dropped request keeps the gate until it ends.
|
||||
await model_gate.run(owner, service.unload_model)
|
||||
finally:
|
||||
model_gate.release(owner)
|
||||
|
||||
@@ -1486,23 +1477,21 @@ class AutotagRoutesMixin:
|
||||
files = await self._run_in_executor(
|
||||
self._without_removed, output_path, files
|
||||
)
|
||||
has_thumbnails = (output_path / "thumbnails").is_dir()
|
||||
location = preview_store.current()
|
||||
if location.mode == preview_store.FOLDER_MODE:
|
||||
has_thumbnails = location.available # no store I/O here (#320)
|
||||
else:
|
||||
store = preview_store.store_dir(output_path)
|
||||
has_thumbnails = store is not None and store.is_dir()
|
||||
|
||||
images = []
|
||||
for image_path in files:
|
||||
rel_path = image_path.relative_to(output_path)
|
||||
thumbnail_url = None
|
||||
if has_thumbnails:
|
||||
# Any preview the file owns, the current format first (#317)
|
||||
existing = thumbnail_settings.existing_thumbnail(
|
||||
output_path, rel_path
|
||||
)
|
||||
if existing is not None:
|
||||
thumb_rel = (
|
||||
thumbnail_settings.thumbnail_stem_rel(rel_path)
|
||||
+ existing.suffix
|
||||
)
|
||||
thumbnail_url = thumbnail_url_for(thumb_rel, existing)
|
||||
# Any preview the file owns, the current format first
|
||||
# (#317); in folder mode without store I/O (#320).
|
||||
thumbnail_url = listing_thumbnail_url(output_path, rel_path)
|
||||
images.append(
|
||||
{
|
||||
"filename": image_path.name,
|
||||
|
||||
@@ -45,7 +45,6 @@ import os
|
||||
import sqlite3
|
||||
import tempfile
|
||||
from contextlib import closing
|
||||
from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
@@ -56,6 +55,7 @@ from .maintenance_plans import PlanExpired, PlanUsed
|
||||
from .restore_plans import RestorePlanStore
|
||||
|
||||
try:
|
||||
from ...database.sqlite_uri import readonly_uri
|
||||
from ...database import restore_stage
|
||||
from ...database.operations import PromptDatabase
|
||||
from ...database.safety_backup import (
|
||||
@@ -66,6 +66,7 @@ try:
|
||||
safety_temp,
|
||||
)
|
||||
except ImportError:
|
||||
from database.sqlite_uri import readonly_uri
|
||||
from database import restore_stage
|
||||
from database.operations import PromptDatabase
|
||||
from database.safety_backup import (
|
||||
@@ -186,7 +187,7 @@ def snapshot_corruption(path):
|
||||
with open(path, "rb") as handle:
|
||||
if handle.read(len(SQLITE_HEADER)) != SQLITE_HEADER:
|
||||
return "Not a SQLite database file"
|
||||
uri = Path(os.path.abspath(path)).as_uri() + "?mode=ro"
|
||||
uri = readonly_uri(path)
|
||||
with closing(sqlite3.connect(uri, uri=True)) as conn:
|
||||
row = conn.execute("PRAGMA integrity_check").fetchone()
|
||||
except (OSError, sqlite3.Error) as error:
|
||||
|
||||
@@ -65,6 +65,7 @@ from .duplicate_review import ( # noqa: F401 (CHANGED, OUTSIDE_SOURCE: public)
|
||||
NOT_ATTEMPTED,
|
||||
OUTSIDE_SOURCE,
|
||||
RECORDS_FAILED,
|
||||
REVIEW_KEPT_DETAILS,
|
||||
ROW_DETAILS,
|
||||
SAME_FILE,
|
||||
SOURCES_CHANGED,
|
||||
@@ -434,6 +435,7 @@ class DuplicateFilesMixin:
|
||||
{
|
||||
"targets": review["targets"],
|
||||
"kept": review["kept"],
|
||||
"held": review["held"],
|
||||
"protected": review["public"]["protected"]["count"],
|
||||
},
|
||||
)
|
||||
@@ -446,7 +448,7 @@ class DuplicateFilesMixin:
|
||||
job hashes each file again right before it unlinks it.
|
||||
"""
|
||||
listed = self._listed_sources()
|
||||
targets, kept, protected = [], [], {}
|
||||
targets, kept, protected, held = [], [], {}, []
|
||||
for group, keepers in selection.values():
|
||||
removals = [r for r in group["files"] if r not in keepers]
|
||||
for record in removals:
|
||||
@@ -467,6 +469,7 @@ class DuplicateFilesMixin:
|
||||
problem = _bound_problem(record, group, keeper)
|
||||
if problem:
|
||||
protected[problem] = protected.get(problem, 0) + 1
|
||||
held.append(self._copy_row(record, REVIEW_KEPT_DETAILS[problem]))
|
||||
continue
|
||||
reviewed.append(self._target(group, record, keeper))
|
||||
if reviewed:
|
||||
@@ -488,6 +491,7 @@ class DuplicateFilesMixin:
|
||||
return {
|
||||
"targets": targets,
|
||||
"kept": kept,
|
||||
"held": held,
|
||||
"fingerprint": _fingerprint(targets),
|
||||
"public": _public_plan(targets, protected, records),
|
||||
}
|
||||
@@ -584,6 +588,13 @@ class DuplicateFilesMixin:
|
||||
|
||||
async def _remove_duplicates(self, job, db, plan, backup):
|
||||
targets = plan.payload["targets"]
|
||||
# #586: every copy the review protected, with its reason, and the
|
||||
# whole group in the report.
|
||||
held = plan.payload["held"]
|
||||
job.record_counted("kept", len(held), held, complete=True)
|
||||
if held:
|
||||
copies = _plural(len(held), "copy", "copies")
|
||||
job.log_line(f"Kept {copies} the review protected")
|
||||
for row in plan.payload["kept"]:
|
||||
job.record("kept", row, f"= {row['root']}/{row['path']} · {KEPT_COPY}")
|
||||
job.update(total=len(targets), status_line="Removing duplicate files")
|
||||
|
||||
@@ -70,6 +70,18 @@ PROTECTED_REASONS = {
|
||||
"files kept: their kept copy changed or is gone",
|
||||
),
|
||||
}
|
||||
# The job's row for a copy the review protected (#586): the review's reason.
|
||||
REVIEW_KEPT_DETAILS = {
|
||||
MAYBE_SAME: "Kept: may be the same file reached through another mount or share",
|
||||
NO_FILE_ID: "Kept: can't confirm this is a separate file on this drive",
|
||||
SAME: "Kept: the same file as its kept copy",
|
||||
OUTSIDE_SOURCE: "Kept: not inside a configured source",
|
||||
LINK: "Kept: a link, not a regular file",
|
||||
CHANGED_CONTENT: "Kept: changed since the search",
|
||||
GONE: "Kept: no longer exists",
|
||||
UNREADABLE: "Kept: could not be read",
|
||||
KEEPER: "Kept: its kept copy changed or is gone",
|
||||
}
|
||||
# Job outcome rows (never an OS error text).
|
||||
CHANGED = "The file changed since the review; it was kept."
|
||||
KEEPER_CHANGED = "The kept copy changed or is gone; this file was kept."
|
||||
|
||||
+40
-15
@@ -15,10 +15,10 @@ from typing import BinaryIO
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from . import thumbnail_settings
|
||||
from . import preview_store, thumbnail_settings
|
||||
from .images import _canonical, _public_path, SERVABLE_EXTENSIONS, VIDEO_EXTENSIONS
|
||||
from .images import in_trash_below_roots
|
||||
from .images import existing_media_thumbnail, media_thumbnail_rel
|
||||
from .images import existing_media_thumbnail, media_preview_path
|
||||
|
||||
try:
|
||||
from ...utils.execution_outputs import (
|
||||
@@ -244,19 +244,36 @@ def resolve_current_candidate(
|
||||
return _unavailable("temporary_unavailable", error=True)
|
||||
|
||||
|
||||
def _target_authorized(target, namespace, scope, store_root=None) -> bool:
|
||||
"""A preview in the thumbnail folder (#320, folder mode) is authorized by
|
||||
the store of its source (``preview_store.owns``: inside the key folder,
|
||||
no link); everything else by the output namespace and the roots."""
|
||||
if (
|
||||
store_root is not None
|
||||
and preview_store.current().mode == preview_store.FOLDER_MODE
|
||||
):
|
||||
return preview_store.owns(store_root, target)
|
||||
return _authorized(target, namespace, scope)
|
||||
|
||||
|
||||
def _open_current_target(
|
||||
target: Path, namespace: Path, scope: CurrentOutputScope, preview: bool = False
|
||||
target: Path,
|
||||
namespace: Path,
|
||||
scope: CurrentOutputScope,
|
||||
preview: bool = False,
|
||||
store_root: Path | None = None,
|
||||
) -> BinaryIO:
|
||||
"""Open only the canonical target and verify authority against the actual handle.
|
||||
|
||||
A *preview* (a file under ``thumbnails/``) may also be JPEG XL (#317)."""
|
||||
A *preview* (a file under ``thumbnails/``) may also be JPEG XL (#317); in
|
||||
folder mode it is authorized by the store of *store_root* (#320)."""
|
||||
if _canonical(target) != target:
|
||||
raise CurrentLocationUnavailable("not_authorized", stale=True)
|
||||
if in_trash_below_roots(target, [namespace]):
|
||||
# Also at the open: the cover chooses its preview after the candidate
|
||||
# check, and a preview can be a link into a trash folder (#543).
|
||||
raise CurrentLocationUnavailable("missing")
|
||||
if not _authorized(target, namespace, scope):
|
||||
if not _target_authorized(target, namespace, scope, store_root):
|
||||
raise CurrentLocationUnavailable("not_authorized")
|
||||
suffix = target.suffix.lower()
|
||||
if suffix not in SERVABLE_EXTENSIONS and not (
|
||||
@@ -289,7 +306,7 @@ def _open_current_target(
|
||||
if (
|
||||
_canonical(target) != target
|
||||
or (observed.st_dev, observed.st_ino) != (current.st_dev, current.st_ino)
|
||||
or not _authorized(target, namespace, scope)
|
||||
or not _target_authorized(target, namespace, scope, store_root)
|
||||
):
|
||||
raise CurrentLocationUnavailable("not_authorized", stale=True)
|
||||
return os.fdopen(fd, "rb")
|
||||
@@ -326,12 +343,10 @@ class _OpenedCurrentMedia:
|
||||
|
||||
|
||||
def _preview_writable(root: Path, preview: Path) -> bool:
|
||||
"""A preview may be written: ``thumbnails`` and every link segment of the
|
||||
preview's folder resolve inside *root* (the serve route's rule, #249)."""
|
||||
folder = root / "thumbnails"
|
||||
return _canonical(folder).is_relative_to(_canonical(root)) and _canonical(
|
||||
preview
|
||||
).is_relative_to(_canonical(folder))
|
||||
"""A preview may be written: the store of *root* and every link segment
|
||||
of the preview's folder resolve inside it (the serve route's rule, #249;
|
||||
``preview_store.owns``)."""
|
||||
return preview_store.owns(root, preview)
|
||||
|
||||
|
||||
def _is_jxl(path) -> bool:
|
||||
@@ -381,8 +396,12 @@ def _cover_target(resolved, ensure_preview, placeholder, accepts_jxl):
|
||||
return _canonical(preview)
|
||||
if preview is not None:
|
||||
return _canonical(preview)
|
||||
preview = resolved.gallery_root / media_thumbnail_rel(relative)
|
||||
if ensure_preview is not None and _preview_writable(resolved.gallery_root, preview):
|
||||
preview = media_preview_path(resolved.gallery_root, relative, create=True)
|
||||
if (
|
||||
ensure_preview is not None
|
||||
and preview is not None
|
||||
and _preview_writable(resolved.gallery_root, preview)
|
||||
):
|
||||
ensure_preview(target, preview, is_video)
|
||||
if is_video and placeholder is not None:
|
||||
return _placeholder_media(placeholder)
|
||||
@@ -418,7 +437,13 @@ def _prepare_current_media(
|
||||
return chosen
|
||||
if chosen is not None:
|
||||
target, is_preview = chosen, True
|
||||
handle = _open_current_target(target, resolved.namespace, scope, is_preview)
|
||||
handle = _open_current_target(
|
||||
target,
|
||||
resolved.namespace,
|
||||
scope,
|
||||
is_preview,
|
||||
resolved.gallery_root if is_preview else None,
|
||||
)
|
||||
try:
|
||||
return _OpenedCurrentMedia(
|
||||
handle,
|
||||
|
||||
+173
-20
@@ -34,7 +34,8 @@ from .gallery_listing import (
|
||||
WalkOutcome,
|
||||
walk_root,
|
||||
)
|
||||
from . import thumbnail_settings
|
||||
from . import preview_store, thumbnail_settings
|
||||
from .thumbnail_location import location_refusal
|
||||
from .thumbnail_encode import (
|
||||
PREVIEW_HELD,
|
||||
EncoderClosed,
|
||||
@@ -1068,13 +1069,24 @@ def _iter_media_files(output_path, max_files):
|
||||
|
||||
|
||||
def media_thumbnail_rel(rel_path, settings=None):
|
||||
"""``thumbnails/<rel>_thumb<ext>`` (POSIX) for the media file *rel_path*:
|
||||
the name the preview gets when it is written now (#317), images and
|
||||
videos alike (``writer_settings``' extension)."""
|
||||
"""``thumbnails/<rel>_thumb<ext>`` (POSIX URL text) for the media file
|
||||
*rel_path*: the name the preview gets when it is written now (#317),
|
||||
images and videos alike (``writer_settings``' extension). Where it is on
|
||||
disk is ``media_preview_path``."""
|
||||
settings = settings or writer_settings()
|
||||
return thumbnail_settings.thumbnail_rel(rel_path, settings.extension)
|
||||
|
||||
|
||||
def media_preview_path(output_path, rel_path, settings=None, *, create=False):
|
||||
"""Where the preview of *rel_path* is written now (``preview_store``);
|
||||
*create* for a writer (a new source gets its key). None when nothing
|
||||
may be written (the thumbnail folder is unavailable)."""
|
||||
settings = settings or writer_settings()
|
||||
return preview_store.preview_path(
|
||||
output_path, rel_path, settings.extension, create=create
|
||||
)
|
||||
|
||||
|
||||
def existing_media_thumbnail(output_path, rel_path):
|
||||
"""The existing preview *rel_path* owns that a listing or cover shows
|
||||
(the format written now first, then older ones), or None. The Images
|
||||
@@ -1084,6 +1096,27 @@ def existing_media_thumbnail(output_path, rel_path):
|
||||
)
|
||||
|
||||
|
||||
def listing_thumbnail_url(output_path, rel_path):
|
||||
"""The thumbnail URL of an older listing (``_enrich_images``, the
|
||||
auto-tag scan), or None. These run on the event loop: in folder mode
|
||||
(#320) the store is never touched there, so the URL names the preview
|
||||
as it is written now and the serve route finds or makes it off the loop
|
||||
(None while the thumbnail folder is unavailable)."""
|
||||
from .prompts import thumbnail_url_for
|
||||
|
||||
location = preview_store.current()
|
||||
if location.mode == preview_store.FOLDER_MODE:
|
||||
if not location.available:
|
||||
return None
|
||||
quoted = urllib.parse.quote(media_thumbnail_rel(rel_path), safe="/")
|
||||
return f"/prompt_manager/images/serve/{quoted}"
|
||||
existing = thumbnail_settings.existing_thumbnail(output_path, rel_path)
|
||||
if existing is None:
|
||||
return None
|
||||
thumb_rel = thumbnail_settings.thumbnail_stem_rel(rel_path) + existing.suffix
|
||||
return thumbnail_url_for(thumb_rel, existing)
|
||||
|
||||
|
||||
def media_thumbnail(output_path, rel_path, root_index, *, root_scope=None):
|
||||
"""``(thumbnail_url, ready)`` for *rel_path* under *output_path* (a few
|
||||
blocking stats). The Images listing, the Needs review queue cards and the
|
||||
@@ -2798,13 +2831,56 @@ class ImageRoutesMixin:
|
||||
output_dirs[0], filepath, int(root_param), len(outputs)
|
||||
)
|
||||
|
||||
# Preview text is mapped by the resolver (#320); any other text,
|
||||
# ``Thumbnails/...`` media on a case-sensitive disk included,
|
||||
# keeps the plain join.
|
||||
preview_parts = (
|
||||
tuple(filepath.split("/"))
|
||||
if preview_store.is_preview_text(filepath)
|
||||
else None
|
||||
)
|
||||
location = preview_store.current()
|
||||
if location.mode == preview_store.FOLDER_MODE:
|
||||
if preview_parts is None and await self._run_image(
|
||||
self._in_source_thumbnails, output_dirs, filepath
|
||||
):
|
||||
# Previews live in the thumbnail folder: anything else
|
||||
# under a source's thumbnails/ is a leftover (#320), in
|
||||
# any case on a case-insensitive disk.
|
||||
return _forbidden()
|
||||
if preview_parts is not None:
|
||||
# Store I/O never runs on the event loop (a share may
|
||||
# stall): the lookup runs on the image pool.
|
||||
found = await self._run_image(
|
||||
self._find_preview, output_dirs, preview_parts, location
|
||||
)
|
||||
if found == "forbidden":
|
||||
return _forbidden()
|
||||
if found is not None:
|
||||
return self._file_response(found)
|
||||
for output_path in output_dirs:
|
||||
image_path = output_path / filepath
|
||||
if (
|
||||
preview_parts is not None
|
||||
and location.mode == preview_store.FOLDER_MODE
|
||||
):
|
||||
break # looked up above
|
||||
if preview_parts is None:
|
||||
image_path = output_path / filepath
|
||||
else:
|
||||
image_path = preview_store.request_path(output_path, preview_parts)
|
||||
if image_path is None:
|
||||
continue
|
||||
if not image_path.exists():
|
||||
continue
|
||||
# A symlink that points outside the root is a hard deny, not
|
||||
# a fall-through to the next root.
|
||||
if not _is_within(image_path, output_path):
|
||||
# A symlink that points outside the root (a preview: outside
|
||||
# the folder holding its store) is a hard deny, not a
|
||||
# fall-through to the next root.
|
||||
container = (
|
||||
output_path
|
||||
if preview_parts is None
|
||||
else preview_store.store_container(output_path)
|
||||
)
|
||||
if not _is_within(image_path, container):
|
||||
return _forbidden()
|
||||
if in_trash_below_roots(image_path, output_dirs):
|
||||
return _file_not_found()
|
||||
@@ -2830,6 +2906,48 @@ class ImageRoutesMixin:
|
||||
{"success": False, "error": _safe_error(e)}, status=500
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _in_source_thumbnails(output_dirs, filepath):
|
||||
"""Whether *filepath* lies in a source's ``thumbnails`` folder: its
|
||||
first part is ``thumbnails`` exactly, or another case of it that is
|
||||
the same folder (a case-insensitive disk). ``Thumbnails/`` media on
|
||||
a case-sensitive disk is another folder and stays served."""
|
||||
first = filepath.split("/", 1)[0]
|
||||
if first == "thumbnails":
|
||||
return True
|
||||
if first.casefold() != "thumbnails":
|
||||
return False
|
||||
for output_path in output_dirs:
|
||||
spelled = output_path / first
|
||||
cache = preview_store.source_folder(output_path)
|
||||
try:
|
||||
if spelled.is_dir() and cache.is_dir() and spelled.samefile(cache):
|
||||
return True
|
||||
except OSError:
|
||||
continue
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _find_preview(output_dirs, parts, location):
|
||||
"""The preview file *parts* names in folder mode, ``"forbidden"``
|
||||
(it resolves out of its store or through a link) or None (blocking;
|
||||
the first root whose store holds it, in ``_select_root`` order)."""
|
||||
for output_path in output_dirs:
|
||||
image_path = preview_store.request_path(output_path, parts, location)
|
||||
if image_path is None:
|
||||
continue
|
||||
try:
|
||||
with preview_store.store_io():
|
||||
if not os.path.lexists(image_path):
|
||||
continue
|
||||
if not preview_store.owns(output_path, image_path, location):
|
||||
return "forbidden"
|
||||
if image_path.is_file():
|
||||
return image_path
|
||||
except OSError:
|
||||
return None
|
||||
return None
|
||||
|
||||
def _is_recorded_media(self, path) -> bool:
|
||||
"""Whether the database records ``path`` as a linked image.
|
||||
|
||||
@@ -2956,6 +3074,9 @@ class ImageRoutesMixin:
|
||||
data, error = await _json_body_or_error(request)
|
||||
if error is not None:
|
||||
return error
|
||||
refused = location_refusal()
|
||||
if refused is not None:
|
||||
return refused
|
||||
return await self._run_held(
|
||||
THUMBNAILS_JOB_KIND,
|
||||
THUMBNAIL_GENERATION_TITLE,
|
||||
@@ -2988,6 +3109,7 @@ class ImageRoutesMixin:
|
||||
"""Blocking thumbnail generation loop (run in executor)."""
|
||||
settings = writer_settings(settings)
|
||||
start = _time.monotonic()
|
||||
thumbnails_dir = preview_store.given_store(output_path, thumbnails_dir)
|
||||
plan = self._plan_thumbnails(output_path, thumbnails_dir, settings=settings)
|
||||
payload = self._execute_thumbnail_plan(plan, settings, start)
|
||||
payload["thumbnails_path"] = _public_path(thumbnails_dir)
|
||||
@@ -3000,7 +3122,8 @@ class ImageRoutesMixin:
|
||||
start = _time.monotonic()
|
||||
plan = self._plan_gallery_thumbnails(output_dirs, settings=settings)
|
||||
payload = self._execute_thumbnail_plan(plan, settings, start)
|
||||
payload["thumbnails_path"] = _public_path(output_dirs[0] / "thumbnails")
|
||||
store = preview_store.store_dir(output_dirs[0])
|
||||
payload["thumbnails_path"] = None if store is None else _public_path(store)
|
||||
return payload
|
||||
|
||||
def _execute_thumbnail_plan(self, plan: dict, settings, start: float) -> dict:
|
||||
@@ -3043,9 +3166,12 @@ class ImageRoutesMixin:
|
||||
if stop is not None and stop():
|
||||
break
|
||||
root = Path(root)
|
||||
store = preview_store.ensure_store(root)
|
||||
if store is None:
|
||||
continue # the thumbnail folder is unavailable
|
||||
plan = self._plan_thumbnails(
|
||||
root,
|
||||
root / "thumbnails",
|
||||
store,
|
||||
mode=mode,
|
||||
stop=stop,
|
||||
settings=settings,
|
||||
@@ -3084,6 +3210,8 @@ class ImageRoutesMixin:
|
||||
"format": writer_settings().format,
|
||||
# Files whose automatic preview failed this session (#249).
|
||||
"failed": self._auto_thumbnail_failures(),
|
||||
# #320: where previews live (no path here, D6A).
|
||||
"location": self._thumbnail_location_payload(with_path=False),
|
||||
"collection_id": hashlib.sha256(collection.encode()).hexdigest()[
|
||||
:16
|
||||
],
|
||||
@@ -3129,9 +3257,11 @@ class ImageRoutesMixin:
|
||||
"""
|
||||
settings = writer_settings(settings)
|
||||
output_path = Path(output_path)
|
||||
thumbnails_dir = Path(thumbnails_dir)
|
||||
thumbnails_dir = preview_store.given_store(output_path, thumbnails_dir)
|
||||
plan = {key: 0 for key in _PLAN_COUNTS}
|
||||
plan.update(targets=[], fresh_files=[])
|
||||
if thumbnails_dir is None:
|
||||
return plan # the thumbnail folder is unavailable
|
||||
folders = {}
|
||||
removed = self._removal_check_under(output_path)
|
||||
for src in _iter_media_files(output_path, MAX_GALLERY_FILES):
|
||||
@@ -3221,6 +3351,9 @@ class ImageRoutesMixin:
|
||||
"dir": src.parent.name,
|
||||
"type": "video" if is_video else "image",
|
||||
}
|
||||
# Taken when the write starts: a thumbnail location switched since
|
||||
# then drops the write at its rename (#320).
|
||||
guard = preview_store.write_guard(guard)
|
||||
dst = Path(dst)
|
||||
if dst.suffix != settings.extension:
|
||||
dst = dst.with_suffix(settings.extension)
|
||||
@@ -3231,7 +3364,7 @@ class ImageRoutesMixin:
|
||||
return {**result, "action": "error", "error": "File no longer exists"}
|
||||
if not rebuild and _thumbnail_is_fresh(src, dst):
|
||||
return {**result, "action": "skipped", "path": str(dst)}
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
preview_store.make_parent(dst)
|
||||
if is_video:
|
||||
made = self._generate_video_thumbnail(
|
||||
src,
|
||||
@@ -3436,8 +3569,12 @@ class ImageRoutesMixin:
|
||||
|
||||
Invalid thumbnail directories return 400; unavailable gallery roots
|
||||
return 404, and unexpected filesystem failures return 500. Holds the
|
||||
registry as guarded work, like generation.
|
||||
registry as guarded work, like generation. Refused (409) while the
|
||||
thumbnail folder is unavailable (#320).
|
||||
"""
|
||||
refused = location_refusal()
|
||||
if refused is not None:
|
||||
return refused
|
||||
return await self._run_held(
|
||||
THUMBNAILS_JOB_KIND,
|
||||
THUMBNAIL_CLEAR_TITLE,
|
||||
@@ -3501,8 +3638,12 @@ class ImageRoutesMixin:
|
||||
self.logger.debug(f"No roots to sweep for preview temp files: {exc}")
|
||||
return 0
|
||||
for output_path in output_dirs:
|
||||
folder = Path(output_path) / "thumbnails"
|
||||
if folder.is_symlink() or not folder.is_dir():
|
||||
folder = preview_store.store_dir(output_path)
|
||||
if (
|
||||
folder is None
|
||||
or preview_store.store_is_link(folder)
|
||||
or not folder.is_dir()
|
||||
):
|
||||
continue
|
||||
removed += sweep_stray_temps(folder)[0]
|
||||
if removed:
|
||||
@@ -3520,13 +3661,16 @@ class ImageRoutesMixin:
|
||||
directories = []
|
||||
for output_path in output_dirs:
|
||||
output_path = Path(output_path)
|
||||
thumbnails_dir = output_path / "thumbnails"
|
||||
thumbnails_dir = preview_store.store_dir(output_path)
|
||||
if thumbnails_dir is None:
|
||||
continue
|
||||
if not thumbnails_dir.exists() and not thumbnails_dir.is_symlink():
|
||||
continue
|
||||
container = preview_store.store_container(output_path)
|
||||
if (
|
||||
thumbnails_dir.is_symlink()
|
||||
or not _is_within(thumbnails_dir, output_path)
|
||||
or _canonical(thumbnails_dir) == _canonical(output_path)
|
||||
preview_store.store_is_link(thumbnails_dir)
|
||||
or not _is_within(thumbnails_dir, container)
|
||||
or _canonical(thumbnails_dir) == _canonical(container)
|
||||
):
|
||||
raise ValueError(
|
||||
"Safety check failed: invalid thumbnails directory path"
|
||||
@@ -3541,7 +3685,15 @@ class ImageRoutesMixin:
|
||||
temps, temp_bytes = sweep_stray_temps(thumbnails_dir, min_age=0)
|
||||
cleared_count += temps
|
||||
cleared_size += temp_bytes
|
||||
for root, _dirs, files in os.walk(thumbnails_dir):
|
||||
folder_mode = preview_store.current().mode == preview_store.FOLDER_MODE
|
||||
for root, dirs, files in os.walk(thumbnails_dir):
|
||||
if folder_mode:
|
||||
# Never into a junction or reparse point (#320).
|
||||
dirs[:] = [
|
||||
d
|
||||
for d in dirs
|
||||
if not preview_store.store_is_link(Path(root) / d)
|
||||
]
|
||||
for name in files:
|
||||
path = Path(root) / name
|
||||
if (
|
||||
@@ -3549,6 +3701,7 @@ class ImageRoutesMixin:
|
||||
or path.suffix.lower()
|
||||
not in thumbnail_settings.THUMBNAIL_EXTENSIONS
|
||||
or path.is_symlink()
|
||||
or not preview_store.may_write(path)
|
||||
):
|
||||
continue
|
||||
try:
|
||||
|
||||
+9
-8
@@ -288,15 +288,16 @@ class Job:
|
||||
index = row.get("root_index")
|
||||
if isinstance(index, bool) or not isinstance(index, int) or index < 0:
|
||||
index = None
|
||||
shaped = {
|
||||
"path": path,
|
||||
"root": str(row.get("root") or ""),
|
||||
"root_index": index,
|
||||
"detail": str(row.get("detail") or ""),
|
||||
}
|
||||
if group in self.complete: # a group kept whole for the report (#586)
|
||||
self.complete[group].append(dict(shaped))
|
||||
if len(rows) < MAX_OUTCOME_ROWS:
|
||||
rows.append(
|
||||
{
|
||||
"path": path,
|
||||
"root": str(row.get("root") or ""),
|
||||
"root_index": index,
|
||||
"detail": str(row.get("detail") or ""),
|
||||
}
|
||||
)
|
||||
rows.append(shaped)
|
||||
self.log_line(log_line)
|
||||
|
||||
def record_counted(self, group, count, rows=(), *, complete=False):
|
||||
|
||||
+3
-1
@@ -377,7 +377,9 @@ class LibraryRoutesMixin:
|
||||
[parent["id"] for parent in result["items"]]
|
||||
)
|
||||
result["next_cursor"] = _encode(
|
||||
result.pop("next_key"), scope, name=options["sort"] == "name"
|
||||
result.pop("next_key"),
|
||||
scope,
|
||||
name=options["sort"] in ("name", "name-desc"),
|
||||
)
|
||||
return result, records
|
||||
|
||||
|
||||
@@ -50,6 +50,7 @@ _REFUSAL_STATUS = {
|
||||
"not-movable": 400,
|
||||
"same-folder": 400,
|
||||
"target-invalid": 400,
|
||||
"data-dir-served": 400,
|
||||
"target-has-library": 409,
|
||||
"target-conflict": 409,
|
||||
"insufficient-space": 507,
|
||||
@@ -74,6 +75,24 @@ def _failure(message, status, code):
|
||||
)
|
||||
|
||||
|
||||
def _thumbnail_folder():
|
||||
"""The thumbnail folder in use, or None (previews inside each source)."""
|
||||
from . import preview_store
|
||||
|
||||
setting = preview_store.setting()
|
||||
return setting.path if setting.mode == preview_store.FOLDER_MODE else None
|
||||
|
||||
|
||||
def _inside_thumbnail_folder(path):
|
||||
"""Whether *path* lies in the thumbnail folder in use (#320)."""
|
||||
from . import preview_store
|
||||
|
||||
setting = preview_store.setting()
|
||||
if setting.mode != preview_store.FOLDER_MODE or not setting.path:
|
||||
return False
|
||||
return ll._inside_any(path, [setting.path])
|
||||
|
||||
|
||||
def _plural(count, one, many=None):
|
||||
return f"{count:,} {one if count == 1 else (many or one + 's')}"
|
||||
|
||||
@@ -225,6 +244,8 @@ class LibraryLocationRoutesMixin:
|
||||
),
|
||||
"problem": location.problem or self._node_folder_risk(location),
|
||||
"size_bytes": size,
|
||||
# #320 D201: thumbnails kept in one folder stay there (D8A).
|
||||
"thumbnails_folder": _thumbnail_folder(),
|
||||
}
|
||||
|
||||
# The node folder (tests point it at a temp tree).
|
||||
@@ -272,6 +293,8 @@ class LibraryLocationRoutesMixin:
|
||||
def _plan_library_move(self):
|
||||
"""The checked plan (blocking); raises MoveFailed."""
|
||||
plan = lm.plan_move(self._live_location())
|
||||
if _inside_thumbnail_folder(plan.target_dir):
|
||||
raise lm.MoveFailed("data-dir-served")
|
||||
if lm.space_problem(plan) is not None:
|
||||
raise lm.MoveFailed("insufficient-space")
|
||||
return plan
|
||||
|
||||
+23
-2
@@ -68,6 +68,23 @@ def _count(count, singular, plural):
|
||||
return f"{count} {singular if count == 1 else plural}"
|
||||
|
||||
|
||||
# Each kept entry's row in the job (#586), in the review's "Kept: ..." words.
|
||||
KEPT_DETAILS = {
|
||||
"active": "Kept: the active log",
|
||||
"link": "Kept: a link",
|
||||
"other": "Kept: another file",
|
||||
}
|
||||
|
||||
|
||||
def _kept_files(rows):
|
||||
"""``{name, kind}`` of each entry the cleanup keeps, in folder order."""
|
||||
return [
|
||||
{"name": row["name"], "kind": row["kind"]}
|
||||
for row in rows
|
||||
if row["kind"] in KEPT_DETAILS
|
||||
]
|
||||
|
||||
|
||||
def _protected_reasons(rows):
|
||||
"""The plan's Protected reasons: what the cleanup keeps, zero counts left out."""
|
||||
active = sum(1 for row in rows if row["kind"] == "active")
|
||||
@@ -129,7 +146,7 @@ class LogCleanupMixin:
|
||||
token = store.issue(
|
||||
ROTATED_LOGS,
|
||||
json.dumps(files, sort_keys=True),
|
||||
{"files": files, "protected": protected},
|
||||
{"files": files, "protected": protected, "kept": _kept_files(rows)},
|
||||
(),
|
||||
)
|
||||
except ScopeTooLarge:
|
||||
@@ -198,7 +215,11 @@ class LogCleanupMixin:
|
||||
job.record(
|
||||
"errors", {"path": name, "detail": detail}, f"! {name} · {detail}"
|
||||
)
|
||||
job.record_counted("protected", plan.payload["protected"])
|
||||
kept = [
|
||||
{"path": row["name"], "detail": KEPT_DETAILS[row["kind"]]}
|
||||
for row in plan.payload["kept"]
|
||||
]
|
||||
job.record_counted("protected", len(kept), kept, complete=True)
|
||||
removed, failed = len(result["removed"]), len(result["failed"])
|
||||
line = f"Deleted {_count(removed, 'rotated log copy', 'rotated log copies')}"
|
||||
if failed:
|
||||
|
||||
+47
-12
@@ -697,19 +697,13 @@ class MaintenanceRoutesMixin:
|
||||
rows,
|
||||
complete=plan.operation in COMPLETE_RECORD,
|
||||
)
|
||||
if "kept" in result:
|
||||
# #550: each kept record with its reason, like the changed ones.
|
||||
kept = [
|
||||
{
|
||||
"path": "",
|
||||
"detail": f"Prompt {item['prompt_id']}: kept, "
|
||||
+ "; ".join(item["reasons"]),
|
||||
}
|
||||
for item in result["kept"]
|
||||
]
|
||||
job.record_counted("protected", len(kept), kept, complete=True)
|
||||
kept = await self._run_in_executor(self._kept_rows, db, plan, result)
|
||||
if kept is None:
|
||||
job.record_counted("protected", payload["protected"]) # none kept
|
||||
else:
|
||||
job.record_counted("protected", payload["protected"])
|
||||
# #550, #586: each kept record with its reason, like the changed
|
||||
# ones, and every one of them in the report.
|
||||
job.record_counted("protected", len(kept), kept, complete=True)
|
||||
line = _changed_line(
|
||||
plan.operation, result["changed"], result.get("marks", 0), result
|
||||
)
|
||||
@@ -717,6 +711,47 @@ class MaintenanceRoutesMixin:
|
||||
job.update(progress=100, status_line=f"{line} · {backup_note(backup)}")
|
||||
return None
|
||||
|
||||
def _kept_rows(self, db, plan, result):
|
||||
"""Outcome rows for the records the apply kept (blocking), or None.
|
||||
|
||||
Prompts are records (no path); a missing-links plan's kept links
|
||||
name their file, relative to its output root, so Activity can open
|
||||
and locate it (#586); one under a source that is gone keeps its path
|
||||
under that source's label. None for an operation that keeps nothing
|
||||
(Optimize).
|
||||
"""
|
||||
if plan.operation == "missing_links":
|
||||
roots = _scan_roots(self._get_all_output_dirs())
|
||||
judged = plan.payload["roots"]
|
||||
rows = []
|
||||
for item in db.kept_links(judged):
|
||||
path, label, index = _scan_location(item["image_path"], roots)
|
||||
if index is None and item["owner"] is not None:
|
||||
# A source that is gone is not listed: name the file
|
||||
# under it, with no listing index to open it by.
|
||||
source = judged[item["owner"]]
|
||||
where = [(os.path.abspath(source), self._public_path(source), None)]
|
||||
path, label, index = _scan_location(item["image_path"], where)
|
||||
rows.append(
|
||||
{
|
||||
"path": path,
|
||||
"root": label,
|
||||
"root_index": index,
|
||||
"detail": f"Kept: {item['reason']}",
|
||||
}
|
||||
)
|
||||
return rows
|
||||
if "kept" not in result:
|
||||
return None
|
||||
return [
|
||||
{
|
||||
"path": "",
|
||||
"detail": f"Prompt {item['prompt_id']}: kept, "
|
||||
+ "; ".join(item["reasons"]),
|
||||
}
|
||||
for item in result["kept"]
|
||||
]
|
||||
|
||||
def _maintenance_rows(self, operation, described):
|
||||
"""Outcome rows for the changed records (root-relative paths only)."""
|
||||
if operation == "missing_links":
|
||||
|
||||
@@ -0,0 +1,728 @@
|
||||
"""The thumbnail location setting (#320): parsing, validation, source keys
|
||||
and the thumbnail folder's marker.
|
||||
|
||||
``preview_store`` holds the runtime state and maps paths; this module holds
|
||||
the rules it applies. Nothing here keeps state.
|
||||
|
||||
Schema (config.json ``thumbnails.location``)::
|
||||
|
||||
{"mode": "folder", "path": "D:\\\\Thumbs", "id": "<uuid4>",
|
||||
"install": "<uuid4>", "cleanup": []}
|
||||
|
||||
The default (and a missing key) is ``{"mode": "source"}``: previews inside
|
||||
each source. An invalid value never falls back to the sources: it parses as
|
||||
a folder location with problem ``invalid``, so nothing is written anywhere.
|
||||
|
||||
Layout in folder mode: ``<folder>/<source key>/<rel dir>/<stem>_thumb<ext>``.
|
||||
A source key is a readable slug plus a short hash of the source's identity
|
||||
(``output-3f9a2c1b``), assigned once and stored in the marker
|
||||
``.promptmanager-thumbnails.json`` with the identity it belongs to.
|
||||
"""
|
||||
|
||||
import errno
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import ntpath
|
||||
import os
|
||||
import posixpath
|
||||
import re
|
||||
import stat as stat_module
|
||||
import threading
|
||||
import time
|
||||
import unicodedata
|
||||
import uuid
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
try:
|
||||
from ...utils import library_location
|
||||
except ImportError: # tests import the package top-level
|
||||
from utils import library_location
|
||||
|
||||
logger = logging.getLogger("prompt_manager.thumbnails")
|
||||
|
||||
SOURCE_MODE = "source"
|
||||
FOLDER_MODE = "folder"
|
||||
MARKER_NAME = ".promptmanager-thumbnails.json"
|
||||
MARKER_VERSION = 1
|
||||
# Beside the marker while one process re-reads, changes and replaces it.
|
||||
MARKER_LOCK_NAME = ".promptmanager-thumbnails.lock"
|
||||
MARKER_LOCK_TIMEOUT = 1.5 # a writer waits this long (under preview_store.IO_BUDGET)
|
||||
MARKER_LOCK_STALE = 30.0 # a lock this old was left by a writer that died
|
||||
MARKER_LOCK_POLL = 0.02
|
||||
# A source key: one folder name, a slug and a 8-16 hex hash (``source_key``).
|
||||
KEY_PATTERN = re.compile(r"^[a-z0-9_-]{1,24}-[0-9a-f]{8,16}$")
|
||||
# A marker is a few keys; anything larger is not ours.
|
||||
MAX_MARKER_BYTES = 4 * 2**20
|
||||
# OS metadata files a folder may hold and still count as empty.
|
||||
OS_METADATA = frozenset({"thumbs.db", "desktop.ini", ".ds_store"})
|
||||
KEY_HASH_LENGTHS = (8, 12, 16)
|
||||
SLUG_LENGTH = 24
|
||||
# Folder + the longest key (24 + 1 + 16) + this margin must fit (Windows).
|
||||
WINDOWS_FOLDER_LIMIT = 247
|
||||
LONGEST_KEY = SLUG_LENGTH + 1 + KEY_HASH_LENGTHS[-1]
|
||||
PATH_MARGIN = 40
|
||||
# A validation stat on a share that does not answer in time is unavailable.
|
||||
VALIDATION_TIMEOUT = 5.0
|
||||
FILE_ATTRIBUTE_REPARSE_POINT = 0x400
|
||||
|
||||
_RESERVED = re.compile(r"^(con|prn|aux|nul|com[1-9]|lpt[1-9])(\..*)?$", re.IGNORECASE)
|
||||
_BAD_CHARS = re.compile(r'[<>:"|?*\x00-\x1f]')
|
||||
|
||||
# Problem codes, their HTTP status and the text the UI shows (no path).
|
||||
MESSAGES = {
|
||||
"location-not-absolute": (
|
||||
400,
|
||||
"Enter a full folder path, such as D:\\Thumbnails or /mnt/fast/thumbnails.",
|
||||
),
|
||||
"location-device-path": (
|
||||
400,
|
||||
"Enter the folder as a drive letter or \\\\server\\share path, "
|
||||
"without \\\\?\\.",
|
||||
),
|
||||
"location-invalid-name": (
|
||||
400,
|
||||
"This folder name can't be used on Windows. Choose another name.",
|
||||
),
|
||||
"location-too-broad": (
|
||||
400,
|
||||
"Choose a folder inside a drive or share, not the drive itself.",
|
||||
),
|
||||
"location-served": (
|
||||
400,
|
||||
"Choose a folder outside your image sources and ComfyUI's output, input "
|
||||
"and temp folders.",
|
||||
),
|
||||
"location-node-folder": (
|
||||
400,
|
||||
"Choose a folder outside the Prompt Manager node folder.",
|
||||
),
|
||||
"location-contains-source": (
|
||||
400,
|
||||
"This folder contains an image source. Choose a folder beside it instead.",
|
||||
),
|
||||
"location-same": (400, "Thumbnails are already stored here."),
|
||||
"location-inside-current": (
|
||||
400,
|
||||
"This folder is inside the thumbnail folder in use. Choose a folder "
|
||||
"outside it.",
|
||||
),
|
||||
"location-parent-missing": (
|
||||
400,
|
||||
"The folder that would hold it doesn't exist. Check the path and that its "
|
||||
"drive is connected.",
|
||||
),
|
||||
"location-unavailable": (
|
||||
409,
|
||||
"This folder can't be reached right now. Check that its drive or share is "
|
||||
"connected.",
|
||||
),
|
||||
"location-not-empty": (
|
||||
409,
|
||||
"This folder already holds other files. Choose an empty folder or one "
|
||||
"Prompt Manager used before.",
|
||||
),
|
||||
"location-not-writable": (
|
||||
409,
|
||||
"Prompt Manager can't write to this folder. Check its permissions.",
|
||||
),
|
||||
"location-path-too-long": (
|
||||
400,
|
||||
"This path is too long for Windows. Choose a shorter folder path.",
|
||||
),
|
||||
"insufficient-space": (
|
||||
507,
|
||||
"There is not enough space for the thumbnails. Free some space or choose "
|
||||
"another folder.",
|
||||
),
|
||||
"location-needs-move": (
|
||||
409,
|
||||
"Use Move thumbnails in System › Sources & scans.",
|
||||
),
|
||||
}
|
||||
HOLDS_THUMBNAILS_REASON = "This folder holds Prompt Manager's thumbnails."
|
||||
|
||||
|
||||
def message(code):
|
||||
"""``(status, text)`` of a problem code."""
|
||||
return MESSAGES[code]
|
||||
|
||||
|
||||
# ── parsing ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ParsedLocation:
|
||||
"""A ``thumbnails.location`` value, checked as data (no filesystem)."""
|
||||
|
||||
mode: str = SOURCE_MODE
|
||||
path: str | None = None
|
||||
id: str | None = None
|
||||
install: str | None = None
|
||||
cleanup: tuple = field(default_factory=tuple)
|
||||
problem: str | None = None
|
||||
|
||||
def to_config(self):
|
||||
"""The config.json form (``invalid`` values are never written)."""
|
||||
if self.mode == SOURCE_MODE:
|
||||
value = {"mode": SOURCE_MODE}
|
||||
else:
|
||||
value = {"mode": FOLDER_MODE, "path": self.path, "id": self.id}
|
||||
if self.install:
|
||||
value["install"] = self.install
|
||||
value["cleanup"] = list(self.cleanup)
|
||||
return value
|
||||
|
||||
|
||||
def _cleanup(value):
|
||||
if not isinstance(value, list):
|
||||
return ()
|
||||
return tuple(item for item in value if isinstance(item, dict))
|
||||
|
||||
|
||||
def parse_location(value, *, flavor=os.path):
|
||||
"""The location a ``thumbnails.location`` value names.
|
||||
|
||||
Missing: inside each source. Anything else that is not a well-formed
|
||||
location (an unknown mode, a relative path, a missing id) becomes a
|
||||
folder location with problem ``invalid``: never a silent fall-back.
|
||||
"""
|
||||
if value is None:
|
||||
return ParsedLocation()
|
||||
if not isinstance(value, dict):
|
||||
return ParsedLocation(mode=FOLDER_MODE, problem="invalid")
|
||||
mode = value.get("mode", SOURCE_MODE)
|
||||
install = value.get("install") if isinstance(value.get("install"), str) else None
|
||||
cleanup = _cleanup(value.get("cleanup"))
|
||||
if mode == SOURCE_MODE:
|
||||
return ParsedLocation(install=install, cleanup=cleanup)
|
||||
if mode != FOLDER_MODE:
|
||||
return ParsedLocation(mode=FOLDER_MODE, problem="invalid", cleanup=cleanup)
|
||||
path, marker_id = value.get("path"), value.get("id")
|
||||
if (
|
||||
not isinstance(path, str)
|
||||
or form_problem(path, flavor=flavor) is not None
|
||||
or not isinstance(marker_id, str)
|
||||
or not marker_id.strip()
|
||||
):
|
||||
return ParsedLocation(
|
||||
mode=FOLDER_MODE,
|
||||
path=path if isinstance(path, str) else None,
|
||||
problem="invalid",
|
||||
cleanup=cleanup,
|
||||
)
|
||||
return ParsedLocation(
|
||||
mode=FOLDER_MODE,
|
||||
path=flavor.abspath(path),
|
||||
id=marker_id,
|
||||
install=install,
|
||||
cleanup=cleanup,
|
||||
)
|
||||
|
||||
|
||||
# ── form (text only) ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _is_windows(flavor):
|
||||
return flavor is ntpath or getattr(flavor, "__name__", "") == "ntpath"
|
||||
|
||||
|
||||
def form_problem(path, *, flavor=os.path):
|
||||
"""The text-only problem of *path*, or None (no filesystem access).
|
||||
|
||||
``\\\\?\\`` and ``\\\\.\\`` prefixes, relative, drive-relative (``C:x``)
|
||||
and root-relative (``\\x``) spellings, and names Windows cannot hold
|
||||
(checked on every OS: the folder may be moved to Windows).
|
||||
"""
|
||||
if not isinstance(path, str) or not path.strip() or path != path.lstrip():
|
||||
return "location-not-absolute"
|
||||
if path.replace("/", "\\").startswith(("\\\\?\\", "\\\\.\\")):
|
||||
return "location-device-path"
|
||||
if _is_windows(flavor):
|
||||
drive, rest = ntpath.splitdrive(path)
|
||||
if not drive or not rest.startswith(("\\", "/")):
|
||||
return "location-not-absolute"
|
||||
elif not posixpath.isabs(path):
|
||||
return "location-not-absolute"
|
||||
drive, rest = flavor.splitdrive(path)
|
||||
for part in re.split(r"[\\/]+", rest):
|
||||
if not part:
|
||||
continue
|
||||
if (
|
||||
_RESERVED.match(part)
|
||||
or part.endswith((".", " "))
|
||||
or _BAD_CHARS.search(part)
|
||||
):
|
||||
return "location-invalid-name"
|
||||
return None
|
||||
|
||||
|
||||
def is_too_broad(path, *, home=None, flavor=os.path):
|
||||
"""A filesystem or share root, or the home folder itself."""
|
||||
normalized = flavor.normpath(path)
|
||||
drive, rest = flavor.splitdrive(normalized)
|
||||
if rest in ("", "\\", "/"):
|
||||
return True
|
||||
home = home if home is not None else os.path.expanduser("~")
|
||||
return bool(home) and flavor.normcase(normalized) == flavor.normcase(
|
||||
flavor.normpath(home)
|
||||
)
|
||||
|
||||
|
||||
def too_long_for_windows(path, long_paths=None):
|
||||
"""Folder + the longest key + a margin over the folder limit (Windows
|
||||
without long paths; *long_paths* injects the registry answer)."""
|
||||
if long_paths is None:
|
||||
long_paths = long_paths_enabled()
|
||||
if long_paths:
|
||||
return False
|
||||
return len(path) + 1 + LONGEST_KEY + PATH_MARGIN > WINDOWS_FOLDER_LIMIT
|
||||
|
||||
|
||||
def long_paths_enabled():
|
||||
"""``LongPathsEnabled`` on Windows (True on other systems)."""
|
||||
if os.name != "nt":
|
||||
return True
|
||||
try:
|
||||
import winreg
|
||||
|
||||
with winreg.OpenKey(
|
||||
winreg.HKEY_LOCAL_MACHINE,
|
||||
r"SYSTEM\CurrentControlSet\Control\FileSystem",
|
||||
) as key:
|
||||
return winreg.QueryValueEx(key, "LongPathsEnabled")[0] == 1
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
# ── links ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def is_link_like(path):
|
||||
"""A symlink, a junction or any reparse point (one lstat), or missing.
|
||||
|
||||
Raises OSError other than "not found" (an unreachable share).
|
||||
"""
|
||||
try:
|
||||
info = os.lstat(path)
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
if stat_module.S_ISLNK(info.st_mode):
|
||||
return True
|
||||
attributes = getattr(info, "st_file_attributes", 0)
|
||||
if attributes & FILE_ATTRIBUTE_REPARSE_POINT:
|
||||
return True
|
||||
is_junction = getattr(os.path, "isjunction", None)
|
||||
return bool(is_junction and is_junction(path))
|
||||
|
||||
|
||||
# ── source keys ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def slug(name):
|
||||
"""``NFKD`` to ASCII, lower case, runs outside ``[a-z0-9_]`` as ``-``,
|
||||
trimmed and cut to 24 characters; empty becomes ``source``."""
|
||||
text = unicodedata.normalize("NFKD", str(name))
|
||||
text = text.encode("ascii", "ignore").decode("ascii").lower()
|
||||
text = re.sub(r"[^a-z0-9_]+", "-", text).strip("-")
|
||||
text = text[:SLUG_LENGTH].strip("-")
|
||||
return text or "source"
|
||||
|
||||
|
||||
def source_key(spelling, identity, taken):
|
||||
"""The key of a source: ``slug(basename) + "-" + sha256(identity)[:n]``.
|
||||
|
||||
*taken* maps existing keys to their identity; a key taken by another
|
||||
identity grows its hash to 12, then 16 characters.
|
||||
"""
|
||||
base = slug(ntpath.basename(str(spelling).rstrip("\\/")))
|
||||
digest = hashlib.sha256(str(identity).encode("utf-8")).hexdigest()
|
||||
for length in KEY_HASH_LENGTHS:
|
||||
key = f"{base}-{digest[:length]}"
|
||||
owner = taken.get(key)
|
||||
if owner is None or owner == identity:
|
||||
return key
|
||||
raise ValueError("no free source key")
|
||||
|
||||
|
||||
def file_identity(path):
|
||||
"""``[st_dev, st_ino]`` of an existing folder, or None (no file ids)."""
|
||||
try:
|
||||
info = os.stat(path)
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
if not info.st_ino:
|
||||
return None
|
||||
return [info.st_dev, info.st_ino]
|
||||
|
||||
|
||||
def find_key(keys, spelling, identity, dev_ino):
|
||||
"""The marker key of a source: the same configured spelling, then the
|
||||
same identity text, then the same file id; or None."""
|
||||
keys = {k: v for k, v in keys.items() if valid_key(k) and isinstance(v, dict)}
|
||||
for key, entry in keys.items():
|
||||
if entry.get("path") == spelling:
|
||||
return key
|
||||
for key, entry in keys.items():
|
||||
if entry.get("identity") == identity:
|
||||
return key
|
||||
if dev_ino:
|
||||
for key, entry in keys.items():
|
||||
if entry.get("dev_ino") == list(dev_ino):
|
||||
return key
|
||||
return None
|
||||
|
||||
|
||||
# ── the marker ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def marker_path(folder):
|
||||
return os.path.join(folder, MARKER_NAME)
|
||||
|
||||
|
||||
def read_marker(folder):
|
||||
"""The marker in *folder* as a dict, or None (absent or unreadable as
|
||||
ours). OSError other than "not found" propagates (an unreachable share)."""
|
||||
try:
|
||||
with open(marker_path(folder), "r", encoding="utf-8") as handle:
|
||||
text = handle.read(MAX_MARKER_BYTES + 1)
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
except (ValueError, UnicodeDecodeError):
|
||||
return None
|
||||
if len(text) > MAX_MARKER_BYTES:
|
||||
return None
|
||||
try:
|
||||
data = json.loads(text)
|
||||
except ValueError:
|
||||
return None
|
||||
if not isinstance(data, dict) or not isinstance(data.get("id"), str):
|
||||
return None
|
||||
data["keys"] = valid_keys(data.get("keys"))
|
||||
return data
|
||||
|
||||
|
||||
def valid_key(key):
|
||||
"""Whether *key* can name a source's folder: one folder name of the
|
||||
``source_key`` form, never a separator, ``..`` or a drive."""
|
||||
return isinstance(key, str) and KEY_PATTERN.fullmatch(key) is not None
|
||||
|
||||
|
||||
def valid_keys(keys):
|
||||
"""The marker's keys that are safe to join onto the folder; a hand-edited
|
||||
or foreign entry that is not is dropped (and logged), never followed."""
|
||||
if not isinstance(keys, dict):
|
||||
return {}
|
||||
kept = {}
|
||||
for key, entry in keys.items():
|
||||
if valid_key(key) and isinstance(entry, dict):
|
||||
kept[key] = entry
|
||||
else:
|
||||
logger.warning(
|
||||
"Ignoring a thumbnail folder marker entry that is not a source " "key"
|
||||
)
|
||||
return kept
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _marker_lock(folder):
|
||||
"""Hold the folder's marker lock: a file made with ``O_CREAT | O_EXCL``
|
||||
(atomic on local disks and SMB/NFS shares, unlike ``flock``), so installs
|
||||
sharing the folder change the marker one at a time. A lock older than
|
||||
MARKER_LOCK_STALE was left by a writer that died and is taken over;
|
||||
waiting longer than MARKER_LOCK_TIMEOUT raises TimeoutError."""
|
||||
lock = os.path.join(folder, MARKER_LOCK_NAME)
|
||||
deadline = time.monotonic() + MARKER_LOCK_TIMEOUT
|
||||
while True:
|
||||
try:
|
||||
handle = os.open(lock, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
|
||||
except FileExistsError:
|
||||
_break_stale_lock(lock)
|
||||
if time.monotonic() >= deadline:
|
||||
raise TimeoutError(
|
||||
errno.ETIMEDOUT, "the thumbnail folder marker is locked", lock
|
||||
)
|
||||
time.sleep(MARKER_LOCK_POLL)
|
||||
continue
|
||||
try:
|
||||
os.write(handle, f"{os.getpid()}\n".encode("ascii"))
|
||||
finally:
|
||||
os.close(handle) # closed before any unlink (Windows)
|
||||
break
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
try:
|
||||
os.unlink(lock)
|
||||
except FileNotFoundError:
|
||||
pass # taken over as stale by another writer: nothing to release
|
||||
|
||||
|
||||
def _break_stale_lock(lock):
|
||||
"""Remove *lock* when it is older than MARKER_LOCK_STALE (its writer
|
||||
died); a lock that changed since it was looked at is left alone."""
|
||||
try:
|
||||
before = os.stat(lock)
|
||||
except FileNotFoundError:
|
||||
return
|
||||
if time.time() - before.st_mtime < MARKER_LOCK_STALE:
|
||||
return
|
||||
logger.warning(f"Taking over a stale thumbnail folder lock: {lock}")
|
||||
try:
|
||||
after = os.stat(lock)
|
||||
if (after.st_ino, after.st_mtime_ns) == (before.st_ino, before.st_mtime_ns):
|
||||
os.unlink(lock)
|
||||
except FileNotFoundError:
|
||||
pass # another writer took it over first
|
||||
|
||||
|
||||
def write_marker(folder, mutate):
|
||||
"""Re-read the marker, apply ``mutate(marker)`` and write it atomically
|
||||
(a temp sibling renamed over it), all under the folder's marker lock so a
|
||||
second install sharing the folder never drops this one's change (and
|
||||
the other way round). Callers hold ``preview_store.write_lock`` (threads
|
||||
of this process). Returns the marker written; TimeoutError when another
|
||||
writer holds the lock too long."""
|
||||
with _marker_lock(folder):
|
||||
return _write_marker_locked(folder, mutate)
|
||||
|
||||
|
||||
def _write_marker_locked(folder, mutate):
|
||||
current = read_marker(folder) or {}
|
||||
marker = {
|
||||
"version": MARKER_VERSION,
|
||||
"id": current.get("id"),
|
||||
"created_by_pm": bool(current.get("created_by_pm", False)),
|
||||
"keys": dict(current.get("keys") or {}),
|
||||
}
|
||||
mutate(marker)
|
||||
if not marker.get("id"):
|
||||
raise ValueError("a marker needs an id")
|
||||
target = marker_path(folder)
|
||||
temp = os.path.join(
|
||||
folder, f".{MARKER_NAME}.{os.getpid()}-{threading.get_ident()}.tmp"
|
||||
)
|
||||
try:
|
||||
with open(temp, "w", encoding="utf-8") as handle:
|
||||
json.dump(marker, handle, indent=2, sort_keys=True)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temp, target)
|
||||
finally:
|
||||
try:
|
||||
os.unlink(temp)
|
||||
except OSError:
|
||||
pass # gone with the rename; a leftover is swept with the temps
|
||||
return marker
|
||||
|
||||
|
||||
def new_id():
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
def init_folder(folder, *, created_by_pm=False):
|
||||
"""The marker id of *folder*: an existing marker is adopted; a folder
|
||||
without one gets a new marker (new id, no keys)."""
|
||||
existing = read_marker(folder)
|
||||
if existing is not None:
|
||||
return existing["id"]
|
||||
|
||||
def create(marker):
|
||||
if not marker.get("id"):
|
||||
marker["id"] = new_id()
|
||||
marker["created_by_pm"] = created_by_pm
|
||||
|
||||
return write_marker(folder, create)["id"]
|
||||
|
||||
|
||||
def folder_state(path, expected_id):
|
||||
"""``(available, problem, marker)`` of a configured thumbnail folder.
|
||||
|
||||
Available: the folder exists, is a directory and not a link, junction or
|
||||
reparse point, and holds our marker with *expected_id*. Raises OSError
|
||||
when the folder cannot be reached (an I/O error other than not found).
|
||||
"""
|
||||
try:
|
||||
info = os.lstat(path)
|
||||
except FileNotFoundError:
|
||||
return False, "missing", None
|
||||
except OSError as error:
|
||||
if error.errno in (errno.ENOENT, errno.ENOTDIR):
|
||||
return False, "missing", None
|
||||
raise
|
||||
if is_link_like(path):
|
||||
return False, "invalid", None
|
||||
if not stat_module.S_ISDIR(info.st_mode):
|
||||
return False, "invalid", None
|
||||
marker = read_marker(path)
|
||||
if marker is None:
|
||||
return False, "missing", None
|
||||
if marker["id"] != expected_id:
|
||||
return False, "marker-mismatch", marker
|
||||
return True, None, marker
|
||||
|
||||
|
||||
def holds_only_metadata(path):
|
||||
"""Whether an existing folder is empty but for OS metadata files."""
|
||||
with os.scandir(path) as entries:
|
||||
for entry in entries:
|
||||
name = entry.name.lower()
|
||||
if name in OS_METADATA or name.startswith("._"):
|
||||
continue
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
# ── validation ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Context:
|
||||
"""What a location is checked against (gathered by the caller)."""
|
||||
|
||||
roots: tuple = () # every image source of every root list
|
||||
contained: tuple = () # ComfyUI output/input/temp and the library folder
|
||||
node_folder: str | None = None
|
||||
default_path: str | None = None
|
||||
current_path: str | None = None # the folder in use (folder mode)
|
||||
home: str | None = None
|
||||
flavor: object = os.path
|
||||
long_paths: bool | None = None
|
||||
environ: object = None
|
||||
# ComfyUI's folder_paths module (None: none; unset: imported).
|
||||
folder_paths: object = library_location._UNSET
|
||||
|
||||
|
||||
def _timed(func, timeout):
|
||||
"""``func()`` on a daemon thread; raises TimeoutError past *timeout*.
|
||||
A stuck call is left to finish on its own thread."""
|
||||
result = {}
|
||||
done = threading.Event()
|
||||
|
||||
def run():
|
||||
try:
|
||||
result["value"] = func()
|
||||
except BaseException as error: # _timed() re-raises it in the caller
|
||||
result["error"] = error
|
||||
finally:
|
||||
done.set()
|
||||
|
||||
threading.Thread(target=run, name="pm-preview-stat", daemon=True).start()
|
||||
if not done.wait(timeout):
|
||||
raise TimeoutError("the folder did not answer in time")
|
||||
if "error" in result:
|
||||
raise result["error"]
|
||||
return result.get("value")
|
||||
|
||||
|
||||
def _inside(path, folders):
|
||||
return library_location._inside_any(path, folders)
|
||||
|
||||
|
||||
def _same(first, second):
|
||||
return library_location.same_folder(first, second) or os.path.normcase(
|
||||
os.path.realpath(first)
|
||||
) == os.path.normcase(os.path.realpath(second))
|
||||
|
||||
|
||||
def _served(path, context):
|
||||
environ = os.environ if context.environ is None else context.environ
|
||||
return (
|
||||
library_location.data_dir_problem(
|
||||
path,
|
||||
folder_paths=context.folder_paths,
|
||||
environ=environ,
|
||||
gallery_roots=list(context.roots),
|
||||
)
|
||||
== "data-dir-served"
|
||||
)
|
||||
|
||||
|
||||
def _probe_write(path):
|
||||
"""Create and delete a probe file in *path*; False when refused."""
|
||||
probe = os.path.join(path, f".pm-probe-{uuid.uuid4().hex[:8]}.tmp")
|
||||
try:
|
||||
with open(probe, "wb") as handle:
|
||||
handle.write(b"")
|
||||
except OSError:
|
||||
return False
|
||||
try:
|
||||
os.unlink(probe)
|
||||
except OSError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def location_problem(path, *, purpose, context, timeout=VALIDATION_TIMEOUT):
|
||||
"""Why *path* cannot be the thumbnail folder, as a code, or None.
|
||||
|
||||
*purpose*: ``move`` (every rule), ``startup`` (no ``same``, ``empty``,
|
||||
``writable`` or parent rules: availability covers them) or ``recreate``
|
||||
(the folder is missing and is made again).
|
||||
"""
|
||||
flavor = context.flavor
|
||||
code = form_problem(path, flavor=flavor)
|
||||
if code is not None:
|
||||
return code
|
||||
path = flavor.abspath(path)
|
||||
default = bool(context.default_path) and flavor.normcase(path) == flavor.normcase(
|
||||
flavor.abspath(context.default_path)
|
||||
)
|
||||
if is_too_broad(path, home=context.home, flavor=flavor):
|
||||
return "location-too-broad"
|
||||
try:
|
||||
return _timed(
|
||||
lambda: _filesystem_problem(path, purpose, context, default), timeout
|
||||
)
|
||||
except (TimeoutError, OSError):
|
||||
return "location-unavailable"
|
||||
|
||||
|
||||
def _filesystem_problem(path, purpose, context, default):
|
||||
if _served(path, context):
|
||||
return "location-served"
|
||||
if context.node_folder and not default and _inside(path, [context.node_folder]):
|
||||
return "location-node-folder"
|
||||
inner = list(context.roots) + list(context.contained)
|
||||
if any(folder and _inside(folder, [path]) for folder in inner):
|
||||
if not default:
|
||||
return "location-contains-source"
|
||||
if (
|
||||
purpose == "move"
|
||||
and context.current_path
|
||||
and (
|
||||
os.path.normcase(path) == os.path.normcase(context.current_path)
|
||||
or (os.path.isdir(path) and _same(path, context.current_path))
|
||||
)
|
||||
):
|
||||
return "location-same"
|
||||
if (
|
||||
purpose == "move"
|
||||
and context.current_path
|
||||
and _inside(path, [context.current_path])
|
||||
):
|
||||
# The old key folders are removed after the move, and with them a
|
||||
# target nested inside one of them.
|
||||
return "location-inside-current"
|
||||
# Windows, or a test injecting its limit (``long_paths=False``).
|
||||
if os.name == "nt" or context.long_paths is False:
|
||||
if too_long_for_windows(path, context.long_paths):
|
||||
return "location-path-too-long"
|
||||
if purpose == "startup":
|
||||
return None
|
||||
if is_link_like(path):
|
||||
# Never through a link: the folder itself must be a real folder.
|
||||
return "location-not-empty"
|
||||
exists = os.path.isdir(path)
|
||||
if not exists:
|
||||
if os.path.lexists(path):
|
||||
return "location-not-empty"
|
||||
if not default and not os.path.isdir(os.path.dirname(path)):
|
||||
return "location-parent-missing"
|
||||
return None
|
||||
if read_marker(path) is None and not holds_only_metadata(path):
|
||||
return "location-not-empty"
|
||||
if not _probe_write(path):
|
||||
return "location-not-writable"
|
||||
return None
|
||||
@@ -0,0 +1,728 @@
|
||||
"""Where previews live on disk (#320): the one resolver.
|
||||
|
||||
Every mapping between an image source (a "root"), one of its media files and
|
||||
that file's preview on disk goes through this module. URL text stays in
|
||||
``thumbnail_settings`` (``thumbnail_rel``: ``thumbnails/<rel>_thumb<ext>``,
|
||||
the shape the serve route, ``inspection-state.mjs`` and ``admin.js`` share);
|
||||
this module answers where such a preview is on disk.
|
||||
|
||||
The store of a source is where its previews live. With the default location
|
||||
(mode ``source``, "Inside each source") it is ``<source>/thumbnails``, exactly
|
||||
the folder every caller joined by hand before, so nothing moves.
|
||||
|
||||
Writes and the switch: every preview writer takes ``write_guard`` when it
|
||||
starts (it records the location's ``generation``); the final rename runs
|
||||
under ``write_lock`` and only when the generation is unchanged and no hold
|
||||
applies (``thumbnail_encode.replace_unless_held``). Changing the location
|
||||
bumps the generation under the same lock, so no write that began before can
|
||||
land in the old store afterwards. Nothing reads or commits config while
|
||||
``write_lock`` is held. The hold check runs under it too, so hold checks take
|
||||
no lock (the job registry's ``running()`` is a lock-free snapshot) and
|
||||
``bump_generation`` is never called with a registry or config lock held.
|
||||
|
||||
All functions are blocking: call them off the event loop.
|
||||
"""
|
||||
|
||||
import errno
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, replace
|
||||
from pathlib import Path
|
||||
|
||||
from . import preview_location, thumbnail_settings
|
||||
|
||||
try:
|
||||
from ...utils.media_folders import PREVIEW_FOLDER
|
||||
except ImportError: # tests import the package top-level
|
||||
from utils.media_folders import PREVIEW_FOLDER
|
||||
|
||||
logger = logging.getLogger("prompt_manager.thumbnails")
|
||||
|
||||
SOURCE_MODE = preview_location.SOURCE_MODE
|
||||
FOLDER_MODE = preview_location.FOLDER_MODE
|
||||
# The probe checks the folder this often, and a check (or a store call)
|
||||
# taking longer than the budget marks it unreachable.
|
||||
PROBE_INTERVAL = 5.0
|
||||
IO_BUDGET = 2.0
|
||||
PROBE_THREAD = "pm-preview-probe"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Location:
|
||||
"""One snapshot of the thumbnail location (take one per request or job)."""
|
||||
|
||||
mode: str = SOURCE_MODE
|
||||
path: str | None = None
|
||||
available: bool = True
|
||||
problem: str | None = None
|
||||
generation: int = 0
|
||||
|
||||
|
||||
write_lock = threading.Lock()
|
||||
_generation = 0
|
||||
# The runtime location: the parsed setting, availability and the marker keys
|
||||
# last read. Swapped as a whole; readers take one snapshot.
|
||||
_state_lock = threading.Lock()
|
||||
_parsed = preview_location.ParsedLocation()
|
||||
_available = True
|
||||
_problem = None
|
||||
# The newest check whose answer was published.
|
||||
_published_order = 0
|
||||
_keys = {}
|
||||
_key_cache = {}
|
||||
_clock = time.monotonic
|
||||
|
||||
|
||||
def current():
|
||||
"""The thumbnail location now."""
|
||||
with _state_lock:
|
||||
parsed, available, problem, generation_now = (
|
||||
_parsed,
|
||||
_available,
|
||||
_problem,
|
||||
_generation,
|
||||
)
|
||||
if parsed.mode == SOURCE_MODE:
|
||||
return Location(generation=generation_now)
|
||||
return Location(FOLDER_MODE, parsed.path, available, problem, generation_now)
|
||||
|
||||
|
||||
def setting():
|
||||
"""The parsed setting in use (``preview_location.ParsedLocation``)."""
|
||||
return _parsed
|
||||
|
||||
|
||||
def generation():
|
||||
return _generation
|
||||
|
||||
|
||||
def bump_generation():
|
||||
"""Invalidate every write that started before (under ``write_lock``)."""
|
||||
global _generation
|
||||
with write_lock:
|
||||
_generation += 1
|
||||
return _generation
|
||||
|
||||
|
||||
def _canonical(path):
|
||||
"""Resolved and case-folded, so paths compare on every OS."""
|
||||
return Path(os.path.normcase(os.path.realpath(str(path))))
|
||||
|
||||
|
||||
def _within(path, folder):
|
||||
return _canonical(path).is_relative_to(_canonical(folder))
|
||||
|
||||
|
||||
# ── the setting and availability ─────────────────────────────────────
|
||||
|
||||
|
||||
def configure(value, *, context=None, check=True):
|
||||
"""Use the ``thumbnails.location`` *value* (config.json) from now on.
|
||||
|
||||
A folder location is validated against *context* (``startup`` rules: a
|
||||
problem other than an unreachable folder makes it ``invalid``), then
|
||||
checked once; the probe keeps checking it. Swapped under ``write_lock``
|
||||
with a generation bump, so no write that started before lands.
|
||||
"""
|
||||
global _parsed, _available, _problem, _generation
|
||||
parsed = preview_location.parse_location(value)
|
||||
problem = parsed.problem
|
||||
if parsed.mode == FOLDER_MODE and problem is None and context is not None:
|
||||
code = preview_location.location_problem(
|
||||
parsed.path, purpose="startup", context=context
|
||||
)
|
||||
if code == "location-unavailable":
|
||||
problem = "unreachable"
|
||||
elif code is not None:
|
||||
problem = "invalid"
|
||||
parsed = preview_location.ParsedLocation(
|
||||
mode=FOLDER_MODE,
|
||||
path=parsed.path,
|
||||
id=parsed.id,
|
||||
install=parsed.install,
|
||||
cleanup=parsed.cleanup,
|
||||
problem="invalid",
|
||||
)
|
||||
with write_lock:
|
||||
with _state_lock:
|
||||
_parsed = parsed
|
||||
_available = parsed.mode == SOURCE_MODE
|
||||
_problem = problem if parsed.mode == FOLDER_MODE else None
|
||||
_keys.clear()
|
||||
_key_cache.clear()
|
||||
_generation += 1
|
||||
if parsed.mode == FOLDER_MODE:
|
||||
if check and parsed.problem is None:
|
||||
refresh()
|
||||
_probe.start()
|
||||
else:
|
||||
_probe.stop()
|
||||
return current()
|
||||
|
||||
|
||||
def refresh():
|
||||
"""Check the folder now (bounded by IO_BUDGET); returns availability."""
|
||||
parsed = _parsed
|
||||
if parsed.mode != FOLDER_MODE or parsed.problem is not None:
|
||||
return parsed.mode == SOURCE_MODE
|
||||
try:
|
||||
(available, problem, marker), order = _probe.ordered_check(
|
||||
parsed.path, parsed.id
|
||||
)
|
||||
except (OSError, TimeoutError, RuntimeError):
|
||||
available, problem, marker = False, "unreachable", None
|
||||
order = _probe.latest_order()
|
||||
_set_state(parsed, available, problem, marker, order)
|
||||
return available
|
||||
|
||||
|
||||
def _set_keys(parsed, marker):
|
||||
"""Take the keys of a marker just written (availability unchanged)."""
|
||||
with _state_lock:
|
||||
if _parsed is not parsed:
|
||||
return
|
||||
keys = dict((marker or {}).get("keys") or {})
|
||||
if keys != _keys:
|
||||
_keys.clear()
|
||||
_keys.update(keys)
|
||||
_key_cache.clear()
|
||||
|
||||
|
||||
def set_cleanup(entries):
|
||||
"""The setting's ``cleanup`` after a removal was committed."""
|
||||
global _parsed
|
||||
with _state_lock:
|
||||
_parsed = replace(_parsed, cleanup=tuple(entries))
|
||||
|
||||
|
||||
def mark_invalid():
|
||||
"""The setting cannot be used (a startup check that failed): folder
|
||||
mode stays in force, unavailable, so nothing is written anywhere."""
|
||||
global _parsed, _available, _problem
|
||||
with _state_lock:
|
||||
if _parsed.mode != FOLDER_MODE:
|
||||
_parsed = preview_location.ParsedLocation(
|
||||
mode=FOLDER_MODE, problem="invalid"
|
||||
)
|
||||
else:
|
||||
_parsed = replace(_parsed, problem="invalid")
|
||||
_available, _problem = False, "invalid"
|
||||
|
||||
|
||||
def _set_state(parsed, available, problem, marker, order):
|
||||
"""Publish a check's answer, unless a check that started later already
|
||||
published (two callers can finish out of order, #611)."""
|
||||
global _available, _problem, _published_order
|
||||
with _state_lock:
|
||||
if _parsed is not parsed:
|
||||
return # the setting changed meanwhile
|
||||
if order < _published_order:
|
||||
return
|
||||
_published_order = order
|
||||
_available, _problem = available, problem
|
||||
keys = dict((marker or {}).get("keys") or {})
|
||||
if keys != _keys:
|
||||
_keys.clear()
|
||||
_keys.update(keys)
|
||||
_key_cache.clear()
|
||||
|
||||
|
||||
def note_failure():
|
||||
"""A store call failed or stalled: unreachable until the probe finds the
|
||||
folder again (requests do no store I/O meanwhile)."""
|
||||
global _available, _problem
|
||||
if _parsed.mode != FOLDER_MODE:
|
||||
return
|
||||
with _state_lock:
|
||||
_available, _problem = False, "unreachable"
|
||||
_probe.wake()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def store_io():
|
||||
"""Around a store call in folder mode: an I/O error (other than a file
|
||||
that is not there) or a call over IO_BUDGET marks the folder
|
||||
unreachable."""
|
||||
if _parsed.mode != FOLDER_MODE:
|
||||
yield
|
||||
return
|
||||
started = _clock()
|
||||
try:
|
||||
yield
|
||||
except OSError as error:
|
||||
if not isinstance(error, (FileNotFoundError, NotADirectoryError)):
|
||||
note_failure()
|
||||
raise
|
||||
finally:
|
||||
if _clock() - started > IO_BUDGET:
|
||||
note_failure()
|
||||
|
||||
|
||||
class _Probe:
|
||||
"""The background check of the thumbnail folder (one thread, one check
|
||||
at a time; a stuck check is left to finish on its own thread)."""
|
||||
|
||||
def __init__(self):
|
||||
self._lock = threading.Lock()
|
||||
self._wake = threading.Event()
|
||||
self._stop = threading.Event()
|
||||
self._thread = None
|
||||
self._checking = None
|
||||
self._order = 0
|
||||
|
||||
def check(self, path, expected_id):
|
||||
"""``folder_state`` within IO_BUDGET; see :meth:`ordered_check`."""
|
||||
return self.ordered_check(path, expected_id)[0]
|
||||
|
||||
def latest_order(self):
|
||||
"""The order number of the check started last."""
|
||||
with self._lock:
|
||||
return self._order
|
||||
|
||||
def ordered_check(self, path, expected_id):
|
||||
"""``(folder_state, order)`` within IO_BUDGET; raises TimeoutError.
|
||||
|
||||
``order`` grows with each check started, so answers can be published
|
||||
in the order they looked. The answer comes from a check that started
|
||||
after this call: one already running may have looked before what the
|
||||
caller just did, so it is waited for (never run twice at once; a stuck
|
||||
one times out here) and followed by a new one."""
|
||||
key = (path, expected_id)
|
||||
with self._lock:
|
||||
earlier = self._running(key)
|
||||
pending = None if earlier else self._start_check(path, expected_id)
|
||||
if earlier:
|
||||
self._wait(earlier)
|
||||
with self._lock:
|
||||
# One another caller started meanwhile is new enough to join.
|
||||
pending = self._running(key) or self._start_check(path, expected_id)
|
||||
self._wait(pending)
|
||||
with self._lock:
|
||||
if self._checking is pending:
|
||||
self._checking = None
|
||||
if "error" in pending[2]:
|
||||
raise pending[2]["error"]
|
||||
return pending[2]["value"], pending[2]["order"]
|
||||
|
||||
def _running(self, key):
|
||||
"""The check of *key* still running (lock held), else None."""
|
||||
pending = self._checking
|
||||
if pending is None or pending[0] != key or pending[1].is_set():
|
||||
return None
|
||||
return pending
|
||||
|
||||
@staticmethod
|
||||
def _wait(pending):
|
||||
if not pending[1].wait(IO_BUDGET):
|
||||
raise TimeoutError("the thumbnail folder did not answer in time")
|
||||
|
||||
def _start_check(self, path, expected_id):
|
||||
"""Start one check on its own thread (lock held)."""
|
||||
self._order += 1
|
||||
result = {"order": self._order}
|
||||
done = threading.Event()
|
||||
|
||||
def run():
|
||||
try:
|
||||
result["value"] = preview_location.folder_state(path, expected_id)
|
||||
except BaseException as error: # check() re-raises it in the caller
|
||||
result["error"] = error
|
||||
finally:
|
||||
done.set()
|
||||
|
||||
pending = ((path, expected_id), done, result)
|
||||
self._checking = pending
|
||||
threading.Thread(target=run, name=PROBE_THREAD + "-check", daemon=True).start()
|
||||
return pending
|
||||
|
||||
def start(self):
|
||||
with self._lock:
|
||||
if self._thread is not None and self._thread.is_alive():
|
||||
return
|
||||
# Each thread has its own stop: a stopped one never resumes.
|
||||
stop = self._stop = threading.Event()
|
||||
thread = threading.Thread(
|
||||
target=self._run, args=(stop,), name=PROBE_THREAD, daemon=True
|
||||
)
|
||||
try:
|
||||
thread.start()
|
||||
except RuntimeError:
|
||||
return # refused at interpreter exit: no probe
|
||||
self._thread = thread
|
||||
|
||||
def _run(self, stop):
|
||||
while not stop.is_set():
|
||||
self._wake.wait(PROBE_INTERVAL)
|
||||
self._wake.clear()
|
||||
if stop.is_set():
|
||||
return
|
||||
try:
|
||||
refresh()
|
||||
except Exception: # one failed check never stops the probe
|
||||
logger.debug("Thumbnail folder check failed", exc_info=True)
|
||||
|
||||
def wake(self):
|
||||
self._wake.set()
|
||||
|
||||
def stop(self, timeout=2.0):
|
||||
"""Stop the probe; True when its thread ended."""
|
||||
with self._lock:
|
||||
thread, self._thread = self._thread, None
|
||||
self._stop.set()
|
||||
self._wake.set()
|
||||
if thread is None or thread is threading.current_thread():
|
||||
return True
|
||||
thread.join(timeout)
|
||||
return not thread.is_alive()
|
||||
|
||||
|
||||
_probe = _Probe()
|
||||
|
||||
|
||||
def stop_probe(timeout=2.0):
|
||||
"""Stop the folder probe (the package teardown, tests)."""
|
||||
return _probe.stop(timeout)
|
||||
|
||||
|
||||
# ── stores ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def source_folder(root):
|
||||
"""``<source>/thumbnails``: the store in source mode, a leftover in
|
||||
folder mode."""
|
||||
return Path(root) / PREVIEW_FOLDER
|
||||
|
||||
|
||||
def store_container(root, location=None):
|
||||
"""The folder a store of *root* must stay inside: the source itself, or
|
||||
the thumbnail folder in folder mode."""
|
||||
location = location or current()
|
||||
if location.mode == FOLDER_MODE:
|
||||
return Path(location.path) if location.path else None
|
||||
return Path(root)
|
||||
|
||||
|
||||
def _identity(root):
|
||||
return os.path.normcase(os.path.realpath(str(root)))
|
||||
|
||||
|
||||
def _lookup_key(root):
|
||||
spelling = str(root)
|
||||
with _state_lock:
|
||||
if spelling in _key_cache:
|
||||
return _key_cache[spelling]
|
||||
keys = dict(_keys)
|
||||
key = preview_location.find_key(
|
||||
keys,
|
||||
spelling,
|
||||
_identity(root),
|
||||
preview_location.file_identity(root),
|
||||
)
|
||||
if key is not None:
|
||||
with _state_lock:
|
||||
_key_cache[spelling] = key
|
||||
return key
|
||||
|
||||
|
||||
def store_dir(root, location=None):
|
||||
"""The store of *root*: ``<source>/thumbnails``, or in folder mode
|
||||
``<folder>/<key>`` (None while the folder is unavailable or before the
|
||||
source has a key: a reader then finds no preview)."""
|
||||
location = location or current()
|
||||
if location.mode == SOURCE_MODE:
|
||||
return Path(root) / PREVIEW_FOLDER
|
||||
if not location.available or not location.path:
|
||||
return None
|
||||
key = _lookup_key(root)
|
||||
return None if key is None else _key_folder(location.path, key)
|
||||
|
||||
|
||||
def _key_folder(folder, key):
|
||||
"""``<folder>/<key>``, only for a key that is one folder name directly
|
||||
below the thumbnail folder (a marker is data, never a path)."""
|
||||
if not preview_location.valid_key(key):
|
||||
return None
|
||||
store = Path(folder) / key
|
||||
if store.parent != Path(folder) or store.name != key:
|
||||
return None
|
||||
return store
|
||||
|
||||
|
||||
def ensure_store(root, location=None, *, create=False):
|
||||
"""``store_dir`` for a writer: in folder mode a source without a key gets
|
||||
one (recorded in the marker); with *create* its key folder is made (never
|
||||
the thumbnail folder itself). None while the folder is unavailable, and
|
||||
for a key folder that is a link, junction or reparse point: nothing is
|
||||
planned, written or removed through it."""
|
||||
location = location or current()
|
||||
store = store_dir(root, location)
|
||||
if location.mode == SOURCE_MODE or (store is None and not location.available):
|
||||
return store
|
||||
if store is None:
|
||||
store = _assign_key(root, location)
|
||||
if store is None:
|
||||
return None
|
||||
try:
|
||||
with store_io():
|
||||
linked = preview_location.is_link_like(store)
|
||||
except OSError:
|
||||
return None
|
||||
if linked:
|
||||
logger.warning(f"Skipping the thumbnail folder {store}: it is a link")
|
||||
return None
|
||||
if create:
|
||||
try:
|
||||
with store_io():
|
||||
os.mkdir(store)
|
||||
except FileExistsError:
|
||||
pass # already there: another writer made the key folder
|
||||
except OSError:
|
||||
return None
|
||||
return store
|
||||
|
||||
|
||||
def _assign_key(root, location):
|
||||
parsed = _parsed
|
||||
if location.path != parsed.path:
|
||||
return None
|
||||
spelling = str(root)
|
||||
identity = _identity(root)
|
||||
dev_ino = preview_location.file_identity(root)
|
||||
assigned = {}
|
||||
|
||||
def add(marker):
|
||||
if marker.get("id") != parsed.id:
|
||||
raise ValueError("another folder's marker") # marker-mismatch
|
||||
keys = marker["keys"]
|
||||
key = preview_location.find_key(keys, spelling, identity, dev_ino)
|
||||
if key is None:
|
||||
taken = {k: v.get("identity") for k, v in keys.items()}
|
||||
key = preview_location.source_key(spelling, identity, taken)
|
||||
keys[key] = {
|
||||
"path": spelling,
|
||||
"identity": identity,
|
||||
"dev_ino": dev_ino,
|
||||
"install": parsed.install,
|
||||
}
|
||||
assigned["key"] = key
|
||||
|
||||
try:
|
||||
with write_lock, store_io():
|
||||
marker = preview_location.write_marker(location.path, add)
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
_set_keys(parsed, marker)
|
||||
return _key_folder(location.path, assigned["key"])
|
||||
|
||||
|
||||
def may_write(path, location=None):
|
||||
"""The one ownership check every preview write and delete passes.
|
||||
|
||||
Source mode: True (the callers' own root rules apply, as before). Folder
|
||||
mode: *path* lies below a source key folder of the thumbnail folder,
|
||||
lexically and resolved, and no link, junction or reparse point sits
|
||||
anywhere from the key folder down to it (the file itself included).
|
||||
"""
|
||||
location = location or current()
|
||||
if location.mode == SOURCE_MODE:
|
||||
return True
|
||||
if not location.path:
|
||||
return False
|
||||
folder = Path(os.path.abspath(location.path))
|
||||
path = Path(os.path.abspath(path))
|
||||
try:
|
||||
relative = path.relative_to(folder)
|
||||
except ValueError:
|
||||
return False
|
||||
if len(relative.parts) < 2 or ".." in relative.parts:
|
||||
return False
|
||||
store = _key_folder(folder, relative.parts[0])
|
||||
if store is None or not _within(path, folder):
|
||||
return False
|
||||
try:
|
||||
return _no_link_below(store, path)
|
||||
except OSError:
|
||||
note_failure()
|
||||
return False
|
||||
|
||||
|
||||
def make_parent(dst, location=None):
|
||||
"""Make the folders a preview *dst* is written into. In folder mode only
|
||||
folders below the thumbnail folder are made, never the folder itself,
|
||||
and only while it holds this install's marker (an unplugged drive or an
|
||||
unmounted mount point raises FileNotFoundError); a *dst* that does not
|
||||
pass ``may_write`` raises PermissionError."""
|
||||
location = location or current()
|
||||
dst = Path(dst)
|
||||
if location.mode == SOURCE_MODE or not location.path:
|
||||
if location.mode != SOURCE_MODE:
|
||||
raise FileNotFoundError(errno.ENOENT, "thumbnail folder unavailable")
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
return
|
||||
folder = Path(os.path.abspath(location.path))
|
||||
with store_io():
|
||||
if not location.available or not folder.is_dir():
|
||||
raise FileNotFoundError(errno.ENOENT, "thumbnail folder unavailable")
|
||||
marker = preview_location.read_marker(str(folder))
|
||||
if marker is None or marker.get("id") != _parsed.id:
|
||||
note_failure()
|
||||
raise FileNotFoundError(errno.ENOENT, "thumbnail folder unavailable")
|
||||
if not may_write(dst, location):
|
||||
raise PermissionError(errno.EACCES, "outside the thumbnail folder")
|
||||
current_folder = folder
|
||||
for part in Path(os.path.abspath(dst.parent)).relative_to(folder).parts:
|
||||
current_folder = current_folder / part
|
||||
try:
|
||||
os.mkdir(current_folder)
|
||||
except FileExistsError:
|
||||
pass # already there: another writer made this folder
|
||||
if preview_location.is_link_like(current_folder):
|
||||
raise PermissionError(errno.EACCES, "a link in the thumbnail folder")
|
||||
|
||||
|
||||
def store_is_link(store, location=None):
|
||||
"""Whether a store folder is a link (a junction or reparse point too in
|
||||
folder mode): it is skipped by sweeps and refused by clears."""
|
||||
location = location or current()
|
||||
if location.mode == SOURCE_MODE:
|
||||
return Path(store).is_symlink()
|
||||
return preview_location.is_link_like(store)
|
||||
|
||||
|
||||
def given_store(root, store):
|
||||
"""A caller's explicit store argument: the default spelling
|
||||
(``<root>/thumbnails``) means the resolver's store; another folder (only
|
||||
tests pass one) is used as given."""
|
||||
store = Path(store)
|
||||
if store == Path(root) / PREVIEW_FOLDER:
|
||||
return store_dir(root)
|
||||
return store
|
||||
|
||||
|
||||
def preview_path(root, media_rel, ext=None, location=None, *, create=False):
|
||||
"""Where the preview of *media_rel* (relative to *root*) is, or is to be
|
||||
written (*create*: a writer, which gives a new source its key), in *ext*
|
||||
(the extension written now when None)."""
|
||||
if create:
|
||||
store = ensure_store(root, location)
|
||||
else:
|
||||
store = store_dir(root, location)
|
||||
if store is None:
|
||||
return None
|
||||
if ext is None:
|
||||
from .thumbnail_encode import writer_settings
|
||||
|
||||
ext = writer_settings().extension
|
||||
return store / (thumbnail_settings.preview_stem(media_rel) + ext)
|
||||
|
||||
|
||||
def request_path(root, parts, location=None):
|
||||
"""The disk path of the URL parts ``("thumbnails", ...)``, or None when
|
||||
*parts* is not under an exact ``thumbnails`` first part or the store has
|
||||
no place for it. The text is checked before (``is_preview_text``)."""
|
||||
parts = tuple(parts)
|
||||
if len(parts) < 2 or parts[0] != PREVIEW_FOLDER:
|
||||
return None
|
||||
store = store_dir(root, location)
|
||||
if store is None:
|
||||
return None
|
||||
return store.joinpath(*parts[1:])
|
||||
|
||||
|
||||
def _no_link_below(store, path):
|
||||
"""No folder from *store* down to *path* (and *store* itself) is a link,
|
||||
junction or reparse point (lexical walk, one lstat each)."""
|
||||
store = Path(os.path.abspath(store))
|
||||
path = Path(os.path.abspath(path))
|
||||
try:
|
||||
relative = path.relative_to(store)
|
||||
except ValueError:
|
||||
return False
|
||||
current = store
|
||||
for part in ("",) + relative.parts:
|
||||
current = current / part if part else current
|
||||
if preview_location.is_link_like(current):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def owns(root, path, location=None):
|
||||
"""Whether *path* lies inside the store of *root*, resolved: the store
|
||||
stays inside its container and no link takes *path* out of the store.
|
||||
In folder mode no link, junction or reparse point may sit anywhere from
|
||||
the store down to *path*."""
|
||||
location = location or current()
|
||||
store = store_dir(root, location)
|
||||
if store is None:
|
||||
return False
|
||||
container = store_container(root, location)
|
||||
if not (_within(store, container) and _within(path, store)):
|
||||
return False
|
||||
if location.mode == SOURCE_MODE:
|
||||
return True
|
||||
return may_write(path, location)
|
||||
|
||||
|
||||
# ── preview text ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def is_preview_text(filepath):
|
||||
"""URL text naming a preview: ``thumbnails/<...>/<name>_thumb<ext>``.
|
||||
|
||||
Text only, before any join: no backslash, drive or stream colon, and no
|
||||
empty, ``.`` or ``..`` segment; ``thumbnails`` exactly as the first part
|
||||
and nowhere else among the folders; the extension, in any case, one a
|
||||
preview may carry (``X_thumb.JPG`` of an older release included).
|
||||
"""
|
||||
if not filepath or "\\" in filepath or ":" in filepath:
|
||||
return False
|
||||
parts = filepath.split("/")
|
||||
if len(parts) < 2 or parts[0] != PREVIEW_FOLDER:
|
||||
return False
|
||||
if any(part in ("", ".", "..") for part in parts):
|
||||
return False
|
||||
if PREVIEW_FOLDER in parts[1:-1]:
|
||||
return False
|
||||
stem, extension = os.path.splitext(parts[-1])
|
||||
return (
|
||||
stem.endswith(thumbnail_settings.THUMB_SUFFIX)
|
||||
and len(stem) > len(thumbnail_settings.THUMB_SUFFIX)
|
||||
and extension.lower() in thumbnail_settings.THUMBNAIL_EXTENSIONS
|
||||
)
|
||||
|
||||
|
||||
# ── writes ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class _WriteGuard:
|
||||
"""``guard()`` for one preview write: true while the location has not
|
||||
changed since the write started and *hold* (if any) allows it."""
|
||||
|
||||
__slots__ = ("generation", "_hold")
|
||||
|
||||
def __init__(self, hold=None):
|
||||
self.generation = _generation
|
||||
self._hold = hold
|
||||
|
||||
def __call__(self):
|
||||
if self.generation != _generation:
|
||||
return False
|
||||
return self._hold is None or bool(self._hold())
|
||||
|
||||
|
||||
def write_guard(hold=None):
|
||||
"""Take when a preview write starts; hand to its final rename."""
|
||||
if isinstance(hold, _WriteGuard):
|
||||
return hold
|
||||
return _WriteGuard(hold)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def committed(guard=None):
|
||||
"""Around a preview's final rename: holds ``write_lock`` and yields
|
||||
whether the write may land (``guard()``, re-checked under the lock)."""
|
||||
with write_lock:
|
||||
yield guard is None or bool(guard())
|
||||
+9
-1
@@ -727,7 +727,15 @@ class PromptRoutesMixin:
|
||||
async def save_prompt(self, request):
|
||||
"""Save a new prompt with metadata and duplicate detection."""
|
||||
try:
|
||||
data = await request.json()
|
||||
data, error = await self._read_json_body(request)
|
||||
if error is not None:
|
||||
return error
|
||||
for field in ("text", "category", "notes"):
|
||||
if data.get(field) is not None and not isinstance(data[field], str):
|
||||
return web.json_response(
|
||||
{"success": False, "error": f"{field} must be text"},
|
||||
status=400,
|
||||
)
|
||||
|
||||
text = (data.get("text") or "").strip()
|
||||
if not text:
|
||||
|
||||
@@ -30,6 +30,7 @@ from . import thumbnail_settings
|
||||
from .admin import _current_image_monitor
|
||||
from .job_routes import busy_response
|
||||
from .jobs import THUMBNAILS_JOB_KIND, JobBusy
|
||||
from .thumbnail_location import needs_move_response
|
||||
|
||||
try:
|
||||
from ...utils.validators import validate_result_timeout, validate_worker_threads
|
||||
@@ -342,6 +343,12 @@ class SettingsRoutesMixin:
|
||||
"sources": source_status,
|
||||
"monitor": monitor,
|
||||
"thumbnail_encoders": encoders,
|
||||
# #320: the full path only here (D6A).
|
||||
"thumbnail_location": await self._run_in_executor(
|
||||
functools.partial(
|
||||
self._thumbnail_location_payload, with_path=True
|
||||
)
|
||||
),
|
||||
"settings": {
|
||||
"monitoring_enabled": GalleryConfig.MONITORING_ENABLED,
|
||||
"save_foreign_runs": GalleryConfig.SAVE_FOREIGN_RUNS,
|
||||
@@ -468,6 +475,9 @@ class SettingsRoutesMixin:
|
||||
if error_response is not None:
|
||||
return error_response
|
||||
|
||||
if "thumbnail_location" in data:
|
||||
# #320: only the Move thumbnails job changes the location.
|
||||
return needs_move_response()
|
||||
expected_revision = data.get("expected_revision")
|
||||
if expected_revision is not None and not isinstance(expected_revision, str):
|
||||
return web.json_response(
|
||||
@@ -576,7 +586,8 @@ class SettingsRoutesMixin:
|
||||
for key in _THUMBNAIL_SETTINGS
|
||||
)
|
||||
job = self._jobs.active(THUMBNAILS_JOB_KIND) if changed else None
|
||||
if job is None or job.scope.get("mode") != "convert":
|
||||
# A move copies previews of the current settings (#320).
|
||||
if job is None or job.scope.get("mode") not in ("convert", "move"):
|
||||
return None
|
||||
return web.json_response(
|
||||
{
|
||||
|
||||
+22
-1
@@ -30,6 +30,8 @@ import ntpath
|
||||
import os
|
||||
import re
|
||||
|
||||
from .preview_location import HOLDS_THUMBNAILS_REASON
|
||||
|
||||
logger = logging.getLogger("prompt_manager.api")
|
||||
|
||||
MAX_SOURCE_PATH = 4096 # characters
|
||||
@@ -114,7 +116,26 @@ def source_problem(text):
|
||||
code = source_form_error(text)
|
||||
if code is not None:
|
||||
return code, FORM_REASONS[code]
|
||||
return GalleryConfig.gallery_root_problem(text)
|
||||
code, reason = GalleryConfig.gallery_root_problem(text)
|
||||
if code is None and _holds_thumbnails(GalleryConfig.resolve_gallery_root(text)):
|
||||
return "holds-thumbnails", HOLDS_THUMBNAILS_REASON
|
||||
return code, reason
|
||||
|
||||
|
||||
def _holds_thumbnails(path):
|
||||
"""Whether *path* is inside or around the thumbnail folder in use (#320):
|
||||
a source there would list Prompt Manager's own previews."""
|
||||
from . import preview_store
|
||||
|
||||
try:
|
||||
from ...utils.library_location import _inside_any
|
||||
except ImportError: # tests import the package top-level
|
||||
from utils.library_location import _inside_any
|
||||
|
||||
setting = preview_store.setting()
|
||||
if setting.mode != preview_store.FOLDER_MODE or not setting.path:
|
||||
return False
|
||||
return _inside_any(path, [setting.path]) or _inside_any(setting.path, [path])
|
||||
|
||||
|
||||
def public_spelling(text):
|
||||
|
||||
+96
-29
@@ -67,7 +67,7 @@ from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from . import thumbnail_settings
|
||||
from . import preview_store, thumbnail_settings
|
||||
from .images import (
|
||||
in_trash_below_roots,
|
||||
MEDIA_EXTENSIONS,
|
||||
@@ -78,7 +78,7 @@ from .images import (
|
||||
_thumbnail_is_fresh,
|
||||
_video_placeholder_image,
|
||||
_worker_threads,
|
||||
media_thumbnail_rel,
|
||||
media_preview_path,
|
||||
)
|
||||
from .jobs import (
|
||||
DUPLICATE_CLEANUP_JOB_KIND,
|
||||
@@ -158,6 +158,36 @@ def _dst_key(dst):
|
||||
return flight_key(Path(dst).parent, (Path(dst).name,))
|
||||
|
||||
|
||||
class DeferredPreview:
|
||||
"""The preview of *rel* under *root*, resolved when its write starts
|
||||
(#320): a queued capture follows the thumbnail location at write time,
|
||||
not the one when it was queued."""
|
||||
|
||||
__slots__ = ("root", "rel", "strict")
|
||||
|
||||
def __init__(self, root, rel, *, strict=True):
|
||||
self.root = Path(root)
|
||||
self.rel = Path(rel)
|
||||
# A capture refuses a store linked out of its container (the serve
|
||||
# route's rule); a planned target keeps the plan's rule: inside the
|
||||
# store, as the thumbnail job writes it.
|
||||
self.strict = strict
|
||||
|
||||
def resolve(self):
|
||||
"""The preview's path now, or None when nothing may be written."""
|
||||
dst = media_preview_path(self.root, self.rel, create=True)
|
||||
if dst is None:
|
||||
return None
|
||||
if self.strict:
|
||||
return dst if preview_store.owns(self.root, dst) else None
|
||||
store = preview_store.store_dir(self.root)
|
||||
return dst if _is_within(dst, store) else None
|
||||
|
||||
|
||||
def _resolved(dst):
|
||||
return dst.resolve() if isinstance(dst, DeferredPreview) else Path(dst)
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def video_placeholder_jpeg():
|
||||
"""The placeholder frame as JPEG bytes (a video whose preview is missing)."""
|
||||
@@ -215,10 +245,19 @@ def _candidates(parts):
|
||||
def _current_preview(root, parts):
|
||||
"""The preview *parts* names, in the format written now (#317): a request
|
||||
for an older name (``_thumb.png`` before a conversion) is answered with
|
||||
the current file."""
|
||||
the current file. None when the store of *root* has no place for it."""
|
||||
_folder, base, _wanted = _candidates(parts)
|
||||
name = base + thumbnail_settings.THUMB_SUFFIX + writer_settings().extension
|
||||
return Path(root).joinpath(*parts[:-1], name)
|
||||
return preview_store.request_path(root, (*parts[:-1], name))
|
||||
|
||||
|
||||
def _request_key(root, parts):
|
||||
"""The flight key of the preview *parts* under *root*: the key of its
|
||||
disk path (so the serve and history routes share one flight)."""
|
||||
path = preview_store.request_path(root, parts)
|
||||
if path is None:
|
||||
return flight_key(root, parts)
|
||||
return _dst_key(path)
|
||||
|
||||
|
||||
def _quick_source(root, parts):
|
||||
@@ -259,12 +298,14 @@ def _thumbnail_source(root, parts):
|
||||
if in_trash_below_roots(found, [root]):
|
||||
return None
|
||||
rel = Path(*folder_parts, found.name)
|
||||
dst = root / media_thumbnail_rel(rel)
|
||||
dst = media_preview_path(root, rel, create=True)
|
||||
if dst is None:
|
||||
# Nowhere to write (the thumbnail folder is unavailable): the
|
||||
# original is served and nothing is written.
|
||||
return found, None, found.suffix.lower() in VIDEO_EXTENSIONS
|
||||
if dst != _current_preview(root, parts):
|
||||
return None
|
||||
if not _is_within(root / "thumbnails", root) or not _is_within(
|
||||
dst, root / "thumbnails"
|
||||
):
|
||||
if not preview_store.owns(root, dst):
|
||||
dst = None
|
||||
return found, dst, found.suffix.lower() in VIDEO_EXTENSIONS
|
||||
|
||||
@@ -284,6 +325,21 @@ def _scan_folder(folder, base, extensions):
|
||||
return None
|
||||
|
||||
|
||||
def _deferred(roots, src, dst):
|
||||
"""The ``DeferredPreview`` of a planned ``(src, dst)``: the root whose
|
||||
store the plan wrote *dst* into (nested roots plan one each); *dst* as
|
||||
planned when none matches."""
|
||||
src = Path(src)
|
||||
for root in roots:
|
||||
root = Path(root)
|
||||
if not src.is_relative_to(root):
|
||||
continue
|
||||
rel = src.relative_to(root)
|
||||
if media_preview_path(root, rel) == Path(dst):
|
||||
return DeferredPreview(root, rel, strict=False)
|
||||
return dst
|
||||
|
||||
|
||||
def _stop_stale_workers():
|
||||
"""Stop capture workers a previous import of this module started."""
|
||||
for thread in threading.enumerate():
|
||||
@@ -425,16 +481,23 @@ class AutoThumbnailer:
|
||||
|
||||
def enqueue(self, src, dst, is_video):
|
||||
"""Queue *dst*'s preview for the worker; False when already queued,
|
||||
full, closed or its worker is refused at interpreter exit. Never
|
||||
blocks on a write."""
|
||||
key = _dst_key(dst)
|
||||
full, closed, nowhere to write it or its worker is refused at
|
||||
interpreter exit. Never blocks on a write. *dst* may be a
|
||||
``DeferredPreview``: its path is resolved again when the write
|
||||
starts."""
|
||||
now = _resolved(dst)
|
||||
if now is None:
|
||||
return False
|
||||
key = _dst_key(now)
|
||||
if not isinstance(dst, DeferredPreview):
|
||||
dst = now
|
||||
with self._changed:
|
||||
if self._closed or key in self._queued:
|
||||
return False
|
||||
if len(self._queue) >= self._max_queued:
|
||||
return False
|
||||
self._queued.add(key)
|
||||
self._queue.append((key, Path(src), Path(dst), bool(is_video)))
|
||||
self._queue.append((key, Path(src), dst, bool(is_video)))
|
||||
self._busy = True
|
||||
started = True
|
||||
if self._start_worker and not self._worker_running:
|
||||
@@ -514,6 +577,9 @@ class AutoThumbnailer:
|
||||
def _write_queued(self, item):
|
||||
_key, src, dst, is_video = item
|
||||
try:
|
||||
dst = _resolved(dst)
|
||||
if dst is None:
|
||||
return # nowhere to write it now: made on its first view
|
||||
self.ensure(src, dst, is_video)
|
||||
except Exception: # one file never stops the worker
|
||||
self._logger.warning(
|
||||
@@ -566,10 +632,6 @@ class AutoThumbnailer:
|
||||
"""True once the worker has nothing queued or in hand."""
|
||||
return self._wait_for(lambda: not self._busy, timeout)
|
||||
|
||||
def wait_batch(self, size, timeout=None):
|
||||
"""True once the worker holds a batch of *size* files."""
|
||||
return self._wait_for(lambda: self._batch == size, timeout)
|
||||
|
||||
|
||||
class _PreviewGate:
|
||||
"""Admission of view requests on the event loop (#249 I2).
|
||||
@@ -683,6 +745,8 @@ class AutoThumbnailMixin:
|
||||
pool.shutdown(wait=False, cancel_futures=True)
|
||||
if final:
|
||||
shutdown_encoders() # kill JPEG XL / ffmpeg encoders still running
|
||||
# The thumbnail folder's probe (#320), bounded like the rest.
|
||||
closed = preview_store.stop_probe() and closed
|
||||
return closed
|
||||
|
||||
def _auto_thumbnail_failures(self):
|
||||
@@ -707,6 +771,9 @@ class AutoThumbnailMixin:
|
||||
return job
|
||||
if job.kind == THUMBNAILS_JOB_KIND and job.title == THUMBNAIL_CLEAR_TITLE:
|
||||
return job
|
||||
# Move thumbnails holds automatic previews for its whole run (D2A).
|
||||
if job.kind == THUMBNAILS_JOB_KIND and job.scope.get("mode") == "move":
|
||||
return job
|
||||
return None
|
||||
|
||||
# ── the view admission ────────────────────────────────────────────
|
||||
@@ -847,7 +914,7 @@ class AutoThumbnailMixin:
|
||||
for root in output_dirs:
|
||||
root = Path(root)
|
||||
outcome = await gate.run(
|
||||
flight_key(root, parts),
|
||||
_request_key(root, parts),
|
||||
functools.partial(self._preview_on_demand, root, parts),
|
||||
pool,
|
||||
)
|
||||
@@ -860,7 +927,7 @@ class AutoThumbnailMixin:
|
||||
continue
|
||||
return self._preview_response(BUSY, quick[0], None, quick[1])
|
||||
dst = _current_preview(root, parts)
|
||||
if outcome == READY:
|
||||
if outcome == READY and dst is not None:
|
||||
return self._file_response(dst)
|
||||
located = await gate.loop.run_in_executor(
|
||||
pool, _thumbnail_source, root, parts
|
||||
@@ -883,17 +950,22 @@ class AutoThumbnailMixin:
|
||||
"""
|
||||
if self._auto_thumbnails_shut:
|
||||
return "skipped"
|
||||
located = self._capture_target(image_path)
|
||||
located = self._capture_source(image_path)
|
||||
if located is None:
|
||||
return "skipped"
|
||||
src, dst, is_video = located
|
||||
src, root, rel, is_video = located
|
||||
dst = DeferredPreview(root, rel).resolve()
|
||||
if dst is None:
|
||||
return "skipped"
|
||||
if _thumbnail_is_fresh(src, dst):
|
||||
return "fresh"
|
||||
if self._auto_thumbnails().enqueue(src, dst, is_video):
|
||||
if self._auto_thumbnails().enqueue(src, DeferredPreview(root, rel), is_video):
|
||||
return "queued"
|
||||
return "skipped"
|
||||
|
||||
def _capture_target(self, image_path):
|
||||
def _capture_source(self, image_path):
|
||||
"""``(src, root, rel, is_video)`` of a media file under a current
|
||||
root (not a preview, a link or in a trash folder), or None."""
|
||||
path = Path(os.path.abspath(str(image_path)))
|
||||
if path.suffix.lower() not in MEDIA_EXTENSIONS or not _regular_file(path):
|
||||
return None
|
||||
@@ -906,12 +978,7 @@ class AutoThumbnailMixin:
|
||||
rel = Path(os.path.relpath(path, root))
|
||||
if "thumbnails" in rel.parts[:-1] or in_trash_below_roots(path, [root]):
|
||||
return None
|
||||
dst = root / media_thumbnail_rel(rel)
|
||||
if not _is_within(root / "thumbnails", root) or not _is_within(
|
||||
dst, root / "thumbnails"
|
||||
):
|
||||
return None
|
||||
return path, dst, path.suffix.lower() in VIDEO_EXTENSIONS
|
||||
return path, root, rel, path.suffix.lower() in VIDEO_EXTENSIONS
|
||||
return None
|
||||
|
||||
# ── after a scan ──────────────────────────────────────────────────
|
||||
@@ -953,7 +1020,7 @@ class AutoThumbnailMixin:
|
||||
return "none"
|
||||
thumbnailer = self._auto_thumbnails()
|
||||
for src, dst, is_video in plan["targets"]:
|
||||
thumbnailer.enqueue(src, dst, is_video)
|
||||
thumbnailer.enqueue(src, _deferred(output_dirs, src, dst), is_video)
|
||||
self.logger.info(f"{missing:,} previews missing after the scan; queued")
|
||||
return "queued"
|
||||
|
||||
|
||||
@@ -44,7 +44,7 @@ import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from . import thumbnail_settings
|
||||
from . import preview_store, thumbnail_settings
|
||||
from .thumbnail_encode import ThumbnailVerifyError, verify_thumbnail, writer_settings
|
||||
from .admin import _plural, _scan_location, _scan_roots, _scan_status_line
|
||||
from .images import _thumbnail_is_fresh, _worker_threads, _thumbnail_batch_size
|
||||
@@ -393,6 +393,12 @@ class ThumbnailConvertMixin:
|
||||
removed, with no bytes (another run or a sibling removed it)."""
|
||||
removed, kept = 0, []
|
||||
for path, size in old:
|
||||
if not preview_store.may_write(path):
|
||||
# Folder mode: never a file outside a key folder, or one
|
||||
# reached through a link (checked at the moment of the delete).
|
||||
self.logger.warning(f"Not removing {path}: outside the thumbnails")
|
||||
kept.append(path)
|
||||
continue
|
||||
try:
|
||||
os.remove(path)
|
||||
except FileNotFoundError:
|
||||
|
||||
+47
-10
@@ -46,7 +46,7 @@ from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from . import thumbnail_settings
|
||||
from . import preview_location, preview_store, thumbnail_settings
|
||||
|
||||
try:
|
||||
from ...utils.logging_config import get_logger
|
||||
@@ -101,14 +101,20 @@ def temp_sibling(dst, tag="part"):
|
||||
|
||||
|
||||
def replace_unless_held(tmp, dst, guard=None):
|
||||
"""Rename *tmp* over *dst*, unless ``guard()`` is false (PreviewWriteHeld)."""
|
||||
if guard is not None and not guard():
|
||||
raise PreviewWriteHeld(PREVIEW_HELD)
|
||||
os.replace(tmp, dst)
|
||||
"""Rename *tmp* over *dst*, unless ``guard()`` is false (PreviewWriteHeld).
|
||||
|
||||
The check and the rename run under ``preview_store.write_lock`` (#320): a
|
||||
guard from ``preview_store.write_guard`` also refuses a write that started
|
||||
before the thumbnail location changed."""
|
||||
with preview_store.committed(guard) as allowed:
|
||||
if not allowed:
|
||||
raise PreviewWriteHeld(PREVIEW_HELD)
|
||||
os.replace(tmp, dst)
|
||||
|
||||
|
||||
# ``temp_sibling``'s names: ``.<stem>.<tag>-<pid>-<thread id><ext>``.
|
||||
_TEMP_NAME = re.compile(r"^\..+\.(?:part|src)-(\d+)-\d+\.[A-Za-z0-9]+$")
|
||||
_MOVE_TEMP_NAME = re.compile(r"\.pm-move-[0-9a-f]{8}\.tmp$")
|
||||
# A temp at least this old (seconds) outlived its write: no encode runs longer.
|
||||
STRAY_TEMP_AGE = 4 * ENCODE_TIMEOUT
|
||||
# When this process started (this module's import): a temp carrying its PID
|
||||
@@ -120,7 +126,10 @@ def sweep_stray_temps(folder, min_age=None):
|
||||
"""Remove the temp files a crashed write left under *folder* (a preview
|
||||
folder, walked without following links); returns ``(count, bytes)``.
|
||||
|
||||
Only ``temp_sibling`` names are touched, never a link. With *min_age*
|
||||
Only ``temp_sibling`` names are touched, never a link: a folder that is a
|
||||
link, junction or reparse point is not walked into (Windows walks into a
|
||||
junction even without ``followlinks``), and each file passes
|
||||
``preview_store.may_write`` before it goes. With *min_age*
|
||||
(seconds, the startup sweep) a temp younger than that, or one this
|
||||
process wrote since it started (it may still be writing it), is left;
|
||||
``min_age=0`` (the clear, which holds every preview write) removes them
|
||||
@@ -130,19 +139,39 @@ def sweep_stray_temps(folder, min_age=None):
|
||||
now = time.time()
|
||||
own = str(os.getpid())
|
||||
count = size = 0
|
||||
for root, _dirs, files in os.walk(folder):
|
||||
try:
|
||||
if preview_location.is_link_like(folder):
|
||||
return count, size
|
||||
except OSError as exc:
|
||||
logger.debug(f"Not sweeping {folder}: {exc}")
|
||||
return count, size
|
||||
for root, dirs, files in os.walk(folder):
|
||||
dirs[:] = [d for d in dirs if not _link_like(Path(root) / d)]
|
||||
for name in files:
|
||||
match = _TEMP_NAME.match(name)
|
||||
if match is None:
|
||||
# A move's copy temp (#320): ``<name>.pm-move-<8 hex>.tmp``.
|
||||
move_temp = match is None and _MOVE_TEMP_NAME.search(name)
|
||||
if match is None and not move_temp:
|
||||
continue
|
||||
path = Path(root) / name
|
||||
try:
|
||||
stat = path.lstat()
|
||||
if path.is_symlink() or not path.is_file():
|
||||
if (
|
||||
path.is_symlink()
|
||||
or not path.is_file()
|
||||
or _link_like(path)
|
||||
or not preview_store.may_write(path)
|
||||
):
|
||||
continue
|
||||
if min_age and (
|
||||
now - stat.st_mtime < min_age
|
||||
or (match.group(1) == own and stat.st_mtime >= PROCESS_STARTED)
|
||||
or move_temp
|
||||
and stat.st_mtime >= PROCESS_STARTED
|
||||
or (
|
||||
match is not None
|
||||
and match.group(1) == own
|
||||
and stat.st_mtime >= PROCESS_STARTED
|
||||
)
|
||||
):
|
||||
continue
|
||||
path.unlink()
|
||||
@@ -154,6 +183,14 @@ def sweep_stray_temps(folder, min_age=None):
|
||||
return count, size
|
||||
|
||||
|
||||
def _link_like(path):
|
||||
"""``is_link_like``, with a path that cannot be checked counted as one."""
|
||||
try:
|
||||
return preview_location.is_link_like(path)
|
||||
except OSError:
|
||||
return True
|
||||
|
||||
|
||||
def _unlink_quietly(path):
|
||||
try:
|
||||
Path(path).unlink()
|
||||
|
||||
@@ -49,6 +49,7 @@ from .images import (
|
||||
_worker_threads,
|
||||
)
|
||||
from .job_routes import busy_response
|
||||
from .thumbnail_location import location_refusal
|
||||
from .jobs import THUMBNAIL_GROUP_NAMES, THUMBNAILS_JOB_KIND, JobBusy
|
||||
from .thumbnail_convert import CONVERT_JOB_TITLE
|
||||
from .thumbnail_encode import writer_settings
|
||||
@@ -89,7 +90,7 @@ class ThumbnailOptions:
|
||||
}
|
||||
|
||||
|
||||
def thumbnail_options(raw, *, from_query=False):
|
||||
def thumbnail_options(raw):
|
||||
"""``(ThumbnailOptions, None)`` from a body or query, or ``(None, message)``.
|
||||
|
||||
An absent mode is ``missing``; the settings are the server's now.
|
||||
@@ -131,8 +132,11 @@ class ThumbnailJobsMixin:
|
||||
"""The thumbnail plan, the thumbnail job and the legacy stream on it."""
|
||||
|
||||
async def get_thumbnail_plan(self, request):
|
||||
"""What a thumbnail run would do (read-only; capped like a run)."""
|
||||
options, error = thumbnail_options(request.query, from_query=True)
|
||||
"""What a thumbnail run would do (read-only; capped like a run).
|
||||
``mode=prune`` lists the removed sources' thumbnails (#320)."""
|
||||
if request.query.get("mode") == "prune":
|
||||
return await self.get_prune_plan()
|
||||
options, error = thumbnail_options(request.query)
|
||||
if error:
|
||||
return _invalid(error)
|
||||
try:
|
||||
@@ -179,15 +183,39 @@ class ThumbnailJobsMixin:
|
||||
)
|
||||
except (ValueError, UnicodeDecodeError):
|
||||
return _invalid(_OPTIONS_ERROR)
|
||||
if isinstance(body, dict) and body.get("mode") in ("move", "prune"):
|
||||
return await self._start_location_job(body)
|
||||
options, error = thumbnail_options(body)
|
||||
if error:
|
||||
return _invalid(error)
|
||||
refused = location_refusal()
|
||||
if refused is not None:
|
||||
return refused
|
||||
try:
|
||||
job = self._start_thumbnail_job(options)
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
return web.json_response({"success": True, "job": job.summary()}, status=202)
|
||||
|
||||
async def _start_location_job(self, body):
|
||||
"""Move thumbnails or Remove thumbnails of removed sources (#320)."""
|
||||
if body["mode"] == "prune":
|
||||
refused = location_refusal()
|
||||
if refused is not None:
|
||||
return refused
|
||||
try:
|
||||
if body["mode"] == "move":
|
||||
started = self.start_thumbnail_move(body)
|
||||
else:
|
||||
started = self.start_thumbnail_prune()
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
if isinstance(started, web.Response):
|
||||
return started
|
||||
return web.json_response(
|
||||
{"success": True, "job": started.summary()}, status=202
|
||||
)
|
||||
|
||||
def _start_thumbnail_job(self, options):
|
||||
"""Register the thumbnail job synchronously; raises JobBusy.
|
||||
|
||||
@@ -459,8 +487,12 @@ class ThumbnailJobsMixin:
|
||||
or attaches to the running thumbnail job, and streams its frames. A
|
||||
legacy hold (the blocking generate or clear) or a record-changing
|
||||
job refuses it with the stream's error frame. The job goes on when
|
||||
the client goes away.
|
||||
the client goes away. Refused (409) while the thumbnail folder is
|
||||
unavailable (#320).
|
||||
"""
|
||||
refused = location_refusal()
|
||||
if refused is not None:
|
||||
return refused
|
||||
response = web.StreamResponse(
|
||||
status=200,
|
||||
headers={
|
||||
@@ -507,8 +539,12 @@ class ThumbnailJobsMixin:
|
||||
the way.
|
||||
"""
|
||||
running = self._jobs.active(THUMBNAILS_JOB_KIND)
|
||||
if running is not None and running.scope.get("mode") == "convert":
|
||||
return None, False, running # no legacy frames to follow (#317)
|
||||
if running is not None and running.scope.get("mode") in (
|
||||
"convert",
|
||||
"move",
|
||||
"prune",
|
||||
):
|
||||
return None, False, running # no legacy frames to follow (#317, #320)
|
||||
if running is not None and running.cancellable:
|
||||
return running, False, None
|
||||
busy = self._jobs.blocker(THUMBNAILS_JOB_KIND, admission="guarded")
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
"""The thumbnail location on the API (#320): startup, the settings payload
|
||||
and the refusals while the thumbnail folder is unavailable.
|
||||
|
||||
The setting itself is config.json ``thumbnails.location``
|
||||
(``preview_location``); the runtime state is ``preview_store``. POST
|
||||
``/settings`` never changes it (409 ``location-needs-move``): the Move
|
||||
thumbnails job is its only writer.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from . import preview_location, preview_store
|
||||
|
||||
try:
|
||||
from ...utils import library_location
|
||||
except ImportError: # tests import the package top-level
|
||||
from utils import library_location
|
||||
|
||||
# The node folder (its web/ is served and a reinstall wipes it).
|
||||
NODE_FOLDER = os.path.dirname(
|
||||
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
)
|
||||
|
||||
|
||||
def default_path():
|
||||
"""``<data folder>/data/thumbnails`` (D7B), or None when unknown."""
|
||||
try:
|
||||
return library_location.cache_dir("thumbnails")
|
||||
except Exception: # no data folder can be decided: no default
|
||||
return None
|
||||
|
||||
|
||||
def location_refusal(location=None):
|
||||
"""The 409 ``location-unavailable`` response while the thumbnail folder
|
||||
is unavailable, else None (thumbnail generate, convert, clear, prune)."""
|
||||
location = location or preview_store.current()
|
||||
if location.mode != preview_store.FOLDER_MODE or location.available:
|
||||
return None
|
||||
status, text = preview_location.message("location-unavailable")
|
||||
return web.json_response(
|
||||
{"success": False, "code": "location-unavailable", "error": text},
|
||||
status=status,
|
||||
)
|
||||
|
||||
|
||||
def needs_move_response():
|
||||
status, text = preview_location.message("location-needs-move")
|
||||
return web.json_response(
|
||||
{"success": False, "code": "location-needs-move", "error": text},
|
||||
status=status,
|
||||
)
|
||||
|
||||
|
||||
class ThumbnailLocationMixin:
|
||||
"""Startup check and payloads of the thumbnail location."""
|
||||
|
||||
def _all_source_roots(self):
|
||||
"""Every image source of every root list: the Images roots, the
|
||||
configured sources and the extra gallery roots (as strings)."""
|
||||
from ..config import GalleryConfig, _extra_gallery_roots
|
||||
|
||||
roots = [str(root) for root in self._get_all_output_dirs()]
|
||||
roots += [str(root) for root in GalleryConfig.MONITORING_DIRECTORIES]
|
||||
roots += list(_extra_gallery_roots())
|
||||
return tuple(dict.fromkeys(root for root in roots if root))
|
||||
|
||||
def _thumbnail_location_context(self):
|
||||
"""What a thumbnail folder is validated against (blocking)."""
|
||||
contained = []
|
||||
try:
|
||||
import folder_paths
|
||||
|
||||
for name in (
|
||||
"get_output_directory",
|
||||
"get_input_directory",
|
||||
"get_temp_directory",
|
||||
):
|
||||
getter = getattr(folder_paths, name, None)
|
||||
folder = getter() if getter is not None else None
|
||||
if folder:
|
||||
contained.append(str(folder))
|
||||
except ImportError:
|
||||
pass # folder_paths exists only inside ComfyUI (tests run without it)
|
||||
try:
|
||||
contained.append(library_location.config_location()[1])
|
||||
except Exception: # no data folder can be decided
|
||||
self.logger.debug("No library folder for the thumbnail check")
|
||||
setting = preview_store.setting()
|
||||
return preview_location.Context(
|
||||
roots=self._all_source_roots(),
|
||||
contained=tuple(contained),
|
||||
node_folder=NODE_FOLDER,
|
||||
default_path=default_path(),
|
||||
current_path=(
|
||||
setting.path if setting.mode == preview_store.FOLDER_MODE else None
|
||||
),
|
||||
)
|
||||
|
||||
def use_preview_location(self):
|
||||
"""Put the configured location in force at once, unchecked (the
|
||||
package's import, before any request): folder mode is unavailable
|
||||
until ``start_preview_location`` has checked it, so nothing is ever
|
||||
written to the sources meanwhile. No filesystem access."""
|
||||
from ..config import PromptManagerConfig
|
||||
|
||||
try:
|
||||
return preview_store.configure(
|
||||
PromptManagerConfig.THUMBNAIL_LOCATION, check=False
|
||||
)
|
||||
except Exception:
|
||||
self.logger.warning("The thumbnail location is unusable", exc_info=True)
|
||||
preview_store.mark_invalid()
|
||||
return preview_store.current()
|
||||
|
||||
def start_preview_location(self):
|
||||
"""Parse and check the configured location once (startup thread);
|
||||
the probe takes over. Never raises; a check that fails leaves a
|
||||
folder location unavailable (``invalid``), never the sources."""
|
||||
from ..config import PromptManagerConfig
|
||||
|
||||
try:
|
||||
value = PromptManagerConfig.THUMBNAIL_LOCATION
|
||||
parsed = preview_location.parse_location(value)
|
||||
context = None
|
||||
if parsed.mode == preview_store.FOLDER_MODE:
|
||||
context = self._thumbnail_location_context()
|
||||
location = preview_store.configure(value, context=context)
|
||||
except Exception:
|
||||
self.logger.warning("The thumbnail location check failed", exc_info=True)
|
||||
if (
|
||||
preview_location.parse_location(
|
||||
PromptManagerConfig.THUMBNAIL_LOCATION
|
||||
).mode
|
||||
== preview_store.FOLDER_MODE
|
||||
):
|
||||
preview_store.mark_invalid()
|
||||
return preview_store.current()
|
||||
if location.mode == preview_store.FOLDER_MODE and not location.available:
|
||||
self.logger.warning(
|
||||
"The thumbnail folder is unavailable "
|
||||
f"({location.problem or 'checking'}): originals are shown and "
|
||||
"no thumbnails are written until it is back"
|
||||
)
|
||||
return location
|
||||
|
||||
def _thumbnail_location_payload(self, *, with_path):
|
||||
"""``{mode, available, problem}`` plus, for the settings GET only,
|
||||
``path`` and ``default_path`` (D6A)."""
|
||||
location = preview_store.current()
|
||||
payload = {
|
||||
"mode": location.mode,
|
||||
"available": location.available,
|
||||
"problem": location.problem,
|
||||
}
|
||||
if with_path:
|
||||
payload["path"] = location.path
|
||||
payload["default_path"] = default_path()
|
||||
# Old copies a move still has to remove (#320 Part 3).
|
||||
payload["cleanup_pending"] = len(preview_store.setting().cleanup)
|
||||
return payload
|
||||
@@ -0,0 +1,579 @@
|
||||
"""Move thumbnails (#320 Part 3): the reviewed job that changes where previews
|
||||
live, the removal of old copies after it, recovery at startup and the
|
||||
"Remove thumbnails of removed sources" scope (Part 5).
|
||||
|
||||
Kind ``thumbnails`` with ``mode: "move"`` (D1A): admission ``guarded``, so
|
||||
scans, deletes, tagging, source changes and every other thumbnail job wait
|
||||
for it, while the monitor's Needs review recording runs beside it.
|
||||
Automatic previews are held for the whole job (D2A, ``_auto_thumbnail_blocker``).
|
||||
|
||||
Phases: review (``POST /images/thumbnails/plan``, no writes but the
|
||||
validation probe), start (a plan token), hold (generation bump, journal,
|
||||
target folder), copy (cancellable at every file boundary, on the job pool
|
||||
in small chunks, D3A), switch (one config commit sets the location and
|
||||
records the old stores under ``cleanup``; not cancellable from here),
|
||||
remove old copies, end. Per-file errors switch anyway (D10A); a folder-wide
|
||||
error or a cancel before the switch removes what the job wrote and changes
|
||||
nothing. The move is the only writer of the setting (D4B).
|
||||
|
||||
Moving away from a thumbnail folder also removes the previews of this
|
||||
install's sources that are no longer listed (Q320-8: remove).
|
||||
"""
|
||||
|
||||
import functools
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from . import preview_location, preview_store
|
||||
from . import thumbnail_move_files as files
|
||||
from .jobs import THUMBNAILS_JOB_KIND
|
||||
from .thumbnail_move_plan import (
|
||||
MOVE_GROUPS,
|
||||
MOVE_TITLE,
|
||||
PLAN_TTL,
|
||||
_number,
|
||||
_plural,
|
||||
_remove,
|
||||
_roots_revision,
|
||||
_volume,
|
||||
_is_network,
|
||||
_write_json,
|
||||
)
|
||||
from . import thumbnail_move_plan as move_plan
|
||||
|
||||
try:
|
||||
from ...utils import exit_signal
|
||||
from ...utils.parallel import map_parallel
|
||||
except ImportError: # tests import the package top-level
|
||||
from utils import exit_signal
|
||||
from utils.parallel import map_parallel
|
||||
|
||||
|
||||
class ThumbnailMoveMixin:
|
||||
"""Start and run the move job; the removal of old copies."""
|
||||
|
||||
# ── start ─────────────────────────────────────────────────────────
|
||||
|
||||
def _take_move_plan(self, body):
|
||||
"""The plan a start names, or ``(status, code)``."""
|
||||
token = body.get("token")
|
||||
plan = (self._move_plans or {}).get(token) if isinstance(token, str) else None
|
||||
if plan is None or time.monotonic() - plan.created > PLAN_TTL:
|
||||
return 409, "plan-stale"
|
||||
roots = [str(root) for root in self._get_all_output_dirs()]
|
||||
if _roots_revision(roots) != plan.revision:
|
||||
return 409, "plan-stale"
|
||||
if preview_store.setting() != plan.old:
|
||||
return 409, "plan-stale"
|
||||
if plan.confirm_required and body.get("confirmed") is not True:
|
||||
return 400, "confirmation-required"
|
||||
return plan
|
||||
|
||||
def start_thumbnail_move(self, body):
|
||||
"""Start the move job (sync, on the loop); raises JobBusy. Returns
|
||||
the job or a refusal response."""
|
||||
if not self._move_recovered:
|
||||
blocker = self._jobs.active(THUMBNAILS_JOB_KIND)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"code": "busy",
|
||||
"error": "Prompt Manager is still checking an earlier move. "
|
||||
"Try again in a moment.",
|
||||
"running": None if blocker is None else {"id": blocker.id},
|
||||
},
|
||||
status=409,
|
||||
)
|
||||
taken = self._take_move_plan(body)
|
||||
if isinstance(taken, tuple):
|
||||
status, code = taken
|
||||
text = {
|
||||
"plan-stale": "The review is out of date. Review the move again.",
|
||||
"confirmation-required": "Confirm that Prompt Manager may "
|
||||
"create its folders there.",
|
||||
}[code]
|
||||
return web.json_response(
|
||||
{"success": False, "code": code, "error": text}, status=status
|
||||
)
|
||||
plan = taken
|
||||
self._move_plans.pop(plan.token, None)
|
||||
return self._jobs.start(
|
||||
THUMBNAILS_JOB_KIND,
|
||||
MOVE_TITLE,
|
||||
admission="guarded",
|
||||
cancellable=True,
|
||||
scope={
|
||||
"mode": "move",
|
||||
"from": plan.old.mode,
|
||||
"to": plan.target["mode"],
|
||||
"to_path": plan.target["path"],
|
||||
},
|
||||
groups=MOVE_GROUPS,
|
||||
runner=functools.partial(self._run_thumbnail_move, plan=plan),
|
||||
)
|
||||
|
||||
# ── the job ───────────────────────────────────────────────────────
|
||||
|
||||
async def _run_thumbnail_move(self, job, *, plan):
|
||||
state = {"written": [], "created": [], "created_folder": False}
|
||||
try:
|
||||
await self._run_in_executor(self._move_hold, job, plan, state)
|
||||
except (OSError, ValueError) as error:
|
||||
await self._run_in_executor(self._move_rollback, plan, state)
|
||||
self._end_move(
|
||||
job, "failed", "The new folder could not be made; nothing changed."
|
||||
)
|
||||
self.logger.warning(f"Move thumbnails could not start: {error}")
|
||||
return
|
||||
left = [
|
||||
{
|
||||
"path": "/".join(item),
|
||||
"root": s.label,
|
||||
"detail": "Not a thumbnail; left in place",
|
||||
}
|
||||
for s in plan.sources
|
||||
if s.scan is not None
|
||||
for item in s.scan.left
|
||||
]
|
||||
if left:
|
||||
job.record_counted("left", len(left), left)
|
||||
items = [
|
||||
(source, item)
|
||||
for source in plan.sources
|
||||
if source.scan is not None
|
||||
for item in source.scan.copy
|
||||
]
|
||||
total = len(items)
|
||||
job.update(total=total, processed=0, progress=0)
|
||||
chunk = await self._run_in_executor(self._move_chunk, plan)
|
||||
workers = self._move_workers()
|
||||
done = 0
|
||||
for start in range(0, total, chunk):
|
||||
if job.cancelled:
|
||||
await self._move_cancelled(job, plan, state)
|
||||
return
|
||||
batch = items[start : start + chunk]
|
||||
try:
|
||||
results = await self._run_in_executor(
|
||||
map_parallel,
|
||||
functools.partial(self._move_copy_one, state=state),
|
||||
batch,
|
||||
workers,
|
||||
)
|
||||
except exit_signal.ExitRequested:
|
||||
return # exiting: the next start undoes the copies (journal)
|
||||
except files.FolderGone as error:
|
||||
self.logger.warning(f"Move thumbnails stopped: {error}")
|
||||
await self._run_in_executor(self._move_rollback, plan, state)
|
||||
self._end_move(
|
||||
job,
|
||||
"failed",
|
||||
"The new folder could not be reached or is full; nothing changed.",
|
||||
)
|
||||
return
|
||||
except Exception:
|
||||
# Anything else before the switch: undo, never half a move.
|
||||
self.logger.exception("Move thumbnails failed while copying")
|
||||
await self._run_in_executor(self._move_rollback, plan, state)
|
||||
self._end_move(job, "failed", "The copy failed; nothing changed.")
|
||||
return
|
||||
for (source, item), (outcome, detail) in zip(batch, results):
|
||||
row = {
|
||||
"path": "/".join(item[0]),
|
||||
"root": source.label,
|
||||
"detail": detail,
|
||||
}
|
||||
job.record(
|
||||
"moved" if outcome == "moved" else outcome,
|
||||
row,
|
||||
f"{outcome}: {source.label} · {'/'.join(item[0])}",
|
||||
)
|
||||
done += len(batch)
|
||||
label = batch[-1][0].label
|
||||
job.update(
|
||||
processed=done,
|
||||
progress=int(done * 90 / total) if total else 90,
|
||||
status_line=(
|
||||
f"Copying {_number(done)} of {_number(total)} previews · {label}"
|
||||
),
|
||||
)
|
||||
job.publish({"type": "progress", "summary": job.summary()})
|
||||
if job.cancelled:
|
||||
await self._move_cancelled(job, plan, state)
|
||||
return
|
||||
# The switch: not cancellable from here (an exit still stops between
|
||||
# files and leaves the cleanup to the next start).
|
||||
job.cancellable = False
|
||||
job.update(status_line="Switching to the new location", progress=92)
|
||||
job.publish({"type": "progress", "summary": job.summary()})
|
||||
try:
|
||||
await self._run_in_executor(self._move_switch, plan)
|
||||
except Exception:
|
||||
self.logger.exception("Move thumbnails could not switch")
|
||||
await self._run_in_executor(self._move_rollback, plan, state)
|
||||
self._end_move(
|
||||
job, "failed", "The setting could not be saved; nothing changed."
|
||||
)
|
||||
return
|
||||
await self._remove_old_copies(job, plan)
|
||||
moved = job.counters.get("moved", 0) + job.counters.get("skipped", 0)
|
||||
errors = job.counters.get("errors", 0)
|
||||
where = plan.target["path"] or "inside each source"
|
||||
parts = [f"{_number(moved)} moved"]
|
||||
if errors:
|
||||
parts.append(_plural(errors, "error"))
|
||||
parts.append(
|
||||
"old copies removed"
|
||||
if not job.scope.get("cleanup_left")
|
||||
else "some old copies are removed at the next start"
|
||||
)
|
||||
self._end_move(
|
||||
job, "finished", f"Thumbnails moved to {where}: " + " · ".join(parts) + "."
|
||||
)
|
||||
|
||||
def _end_move(self, job, state, line):
|
||||
job.update(status_line=line, progress=100)
|
||||
job.log_line(line)
|
||||
job.publish({"type": "complete", "summary": job.summary()})
|
||||
job.finish(state)
|
||||
|
||||
def _move_workers(self):
|
||||
from .images import _worker_threads
|
||||
|
||||
return max(1, int(_worker_threads() or 1))
|
||||
|
||||
def _move_chunk(self, plan):
|
||||
"""2 files at a time across volumes or to/from a share, else 4, never
|
||||
more than Worker threads (D3A: the pool keeps its size)."""
|
||||
sides = [plan.target["path"] or ""] + [s.old_store or "" for s in plan.sources]
|
||||
sides += [s.new_store for s in plan.sources]
|
||||
network = any(_is_network(side) for side in sides if side)
|
||||
volumes = {_volume(side) for side in sides if side}
|
||||
size = 2 if network or len(volumes) > 1 else 4
|
||||
return max(1, min(size, self._move_workers()))
|
||||
|
||||
def _move_hold(self, job, plan, state):
|
||||
"""Phase 3: bump the generation, write the journal, make the target."""
|
||||
preview_store.bump_generation()
|
||||
if plan.to_folder:
|
||||
folder = plan.target["path"]
|
||||
if not os.path.isdir(folder):
|
||||
if plan.target_default:
|
||||
os.makedirs(folder, exist_ok=True)
|
||||
else:
|
||||
os.mkdir(folder)
|
||||
state["created_folder"] = True
|
||||
plan.target_id = preview_location.init_folder(
|
||||
folder, created_by_pm=state["created_folder"]
|
||||
)
|
||||
install = plan.install
|
||||
|
||||
def add_keys(marker):
|
||||
for source in plan.sources:
|
||||
if source.key and source.key not in marker["keys"]:
|
||||
marker["keys"][source.key] = {
|
||||
"path": source.root,
|
||||
"identity": os.path.normcase(os.path.realpath(source.root)),
|
||||
"dev_ino": preview_location.file_identity(source.root),
|
||||
"install": install,
|
||||
}
|
||||
|
||||
with preview_store.write_lock:
|
||||
preview_location.write_marker(folder, add_keys)
|
||||
_write_json(
|
||||
move_plan.journal_path(),
|
||||
{
|
||||
"state": "copying",
|
||||
"from": plan.old.to_config(),
|
||||
"to": plan.target,
|
||||
"stores": [s.new_store for s in plan.sources],
|
||||
# Stores that do not exist yet are made by this job only:
|
||||
# after a crash everything in them goes (B4).
|
||||
"created_dirs": [
|
||||
s.new_store for s in plan.sources if not os.path.isdir(s.new_store)
|
||||
],
|
||||
"created_folder": state["created_folder"],
|
||||
"install": plan.install,
|
||||
},
|
||||
)
|
||||
job.log_line(
|
||||
f"Move thumbnails started: {plan.old.path or 'inside each source'} → "
|
||||
f"{plan.target['path'] or 'inside each source'}"
|
||||
)
|
||||
|
||||
def _move_copy_one(self, pair, *, state):
|
||||
"""Copy one preview (a pool thread); ``(group, detail)``."""
|
||||
source, (relative, size, mtime_ns) = pair
|
||||
destination = Path(source.new_store).joinpath(*relative)
|
||||
try:
|
||||
if not os.path.isdir(destination.parent):
|
||||
base = Path(source.new_store)
|
||||
if not base.is_dir():
|
||||
if plan_parent_missing(base):
|
||||
raise files.FolderGone("the new folder is gone")
|
||||
created = []
|
||||
files._make_folders(base.parent, (base.name,), created)
|
||||
state["created"].extend(created)
|
||||
created = []
|
||||
files._make_folders(base, relative[:-1], created)
|
||||
state["created"].extend(created)
|
||||
# Also when the folders already exist (Codex P1 on #592).
|
||||
files.refuse_links(source.new_store, relative[:-1])
|
||||
outcome = files.copy_one(
|
||||
Path(source.old_store).joinpath(*relative),
|
||||
destination,
|
||||
size,
|
||||
mtime_ns,
|
||||
state["created"],
|
||||
)
|
||||
except files.CopyFailed as error:
|
||||
return "errors", str(error)
|
||||
if outcome == "moved":
|
||||
state["written"].append(str(destination))
|
||||
return "moved", "Copied and checked"
|
||||
return "skipped", "Already there"
|
||||
|
||||
async def _move_cancelled(self, job, plan, state):
|
||||
if exit_signal.requested():
|
||||
return # the next start removes the partial copies (journal)
|
||||
await self._run_in_executor(self._move_rollback, plan, state)
|
||||
self._end_move(job, "cancelled", "Move cancelled: nothing changed.")
|
||||
|
||||
def _move_rollback(self, plan, state):
|
||||
"""Remove what this job wrote (files, folders it made, a folder and
|
||||
marker it created) and the journal: nothing changed."""
|
||||
for path in state["written"]:
|
||||
files.remove_file(path)
|
||||
for folder in reversed(state["created"]):
|
||||
files.remove_empty_folders(folder)
|
||||
if plan.to_folder and state.get("created_folder"):
|
||||
folder = plan.target["path"]
|
||||
try:
|
||||
names = os.listdir(folder)
|
||||
except OSError:
|
||||
names = None
|
||||
if names is not None and set(names) <= {preview_location.MARKER_NAME}:
|
||||
_remove(preview_location.marker_path(folder))
|
||||
files.remove_empty_folders(folder)
|
||||
_remove(move_plan.journal_path())
|
||||
|
||||
def _move_switch(self, plan):
|
||||
"""Phase 5: one config commit sets the location and records the old
|
||||
stores under ``cleanup``; the runtime follows under the write lock."""
|
||||
from ..config import PromptManagerConfig
|
||||
from ..config_store import get_config_store
|
||||
|
||||
entry = {
|
||||
"from": plan.old.mode,
|
||||
"folder": plan.old.path if plan.from_folder else None,
|
||||
"stores": [s.old_store for s in plan.sources if s.old_store],
|
||||
"drop_keys": self._old_keys_to_drop(plan),
|
||||
"install": plan.install,
|
||||
}
|
||||
# An older entry never removes a store the move just filled (a move
|
||||
# back while old copies still waited).
|
||||
filled = {os.path.normcase(s.new_store) for s in plan.sources}
|
||||
cleanup = []
|
||||
for old_entry in plan.old.cleanup:
|
||||
stores = [
|
||||
store
|
||||
for store in old_entry.get("stores") or []
|
||||
if isinstance(store, str) and os.path.normcase(store) not in filled
|
||||
]
|
||||
folder = old_entry.get("folder")
|
||||
drop = [
|
||||
key
|
||||
for key in old_entry.get("drop_keys") or []
|
||||
if not folder
|
||||
or os.path.normcase(os.path.join(folder, key)) not in filled
|
||||
]
|
||||
cleanup.append({**old_entry, "stores": stores, "drop_keys": drop})
|
||||
cleanup.append(entry)
|
||||
if plan.to_folder:
|
||||
value = {
|
||||
"mode": preview_store.FOLDER_MODE,
|
||||
"path": plan.target["path"],
|
||||
"id": plan.target_id,
|
||||
"install": plan.install,
|
||||
"cleanup": cleanup,
|
||||
}
|
||||
else:
|
||||
value = {
|
||||
"mode": preview_store.SOURCE_MODE,
|
||||
"install": plan.install,
|
||||
"cleanup": cleanup,
|
||||
}
|
||||
|
||||
def mutate(config):
|
||||
section = config.get("thumbnails")
|
||||
if not isinstance(section, dict):
|
||||
section = {}
|
||||
section["location"] = value
|
||||
config["thumbnails"] = section
|
||||
return config
|
||||
|
||||
def publish(_config):
|
||||
PromptManagerConfig.THUMBNAIL_LOCATION = value
|
||||
preview_store.configure(value, check=False)
|
||||
|
||||
get_config_store().commit(mutate, expected_revision=None, publish=publish)
|
||||
# The switch is made: what follows never undoes it.
|
||||
try:
|
||||
preview_store.refresh()
|
||||
_remove(move_plan.journal_path())
|
||||
except OSError as error:
|
||||
self.logger.warning(
|
||||
f"The thumbnail move journal stays until the next start: {error}"
|
||||
)
|
||||
|
||||
def _old_keys_to_drop(self, plan):
|
||||
if not plan.from_folder:
|
||||
return []
|
||||
keys = [s.old_store and os.path.basename(s.old_store) for s in plan.sources]
|
||||
keys = [key for key in keys if key]
|
||||
if plan.to_folder and os.path.normcase(plan.old.path or "") == os.path.normcase(
|
||||
plan.target["path"] or ""
|
||||
):
|
||||
return []
|
||||
return keys + [key for key, _store, _scan in plan.unlisted]
|
||||
|
||||
async def _remove_old_copies(self, job, plan):
|
||||
"""Phase 6: run the cleanup the switch recorded (and earlier ones)."""
|
||||
job.update(status_line="Removing old copies", progress=95)
|
||||
job.publish({"type": "progress", "summary": job.summary()})
|
||||
left = await self._run_in_executor(self._run_cleanup, job)
|
||||
if left:
|
||||
job.scope["cleanup_left"] = True
|
||||
|
||||
def _run_cleanup(self, job=None):
|
||||
"""Remove every recorded old store's previews, temps and empty
|
||||
folders; keep in ``cleanup`` what could not go. Returns how many
|
||||
entries are left (blocking)."""
|
||||
setting = preview_store.setting()
|
||||
remaining = []
|
||||
for entry in setting.cleanup:
|
||||
if not self._cleanup_entry(entry, job):
|
||||
remaining.append(entry)
|
||||
if list(setting.cleanup) != remaining:
|
||||
self._commit_cleanup(remaining)
|
||||
return len(remaining)
|
||||
|
||||
def _cleanup_entry(self, entry, job):
|
||||
"""One recorded removal; True when it is complete."""
|
||||
folder = entry.get("folder")
|
||||
stores = [
|
||||
store for store in entry.get("stores") or [] if isinstance(store, str)
|
||||
]
|
||||
if folder:
|
||||
if not os.path.isdir(folder):
|
||||
return False # the old folder is not reachable: kept for later
|
||||
folder_marker = preview_location.read_marker(folder) or {}
|
||||
keys = folder_marker.get("keys") or {}
|
||||
stores += [
|
||||
os.path.join(folder, key)
|
||||
for key in entry.get("drop_keys") or []
|
||||
if preview_location.valid_key(key) and key in keys
|
||||
]
|
||||
complete = True
|
||||
for store in dict.fromkeys(stores):
|
||||
if not self._remove_store(store, folder, job):
|
||||
complete = False
|
||||
if folder and complete:
|
||||
drop = set(entry.get("drop_keys") or [])
|
||||
|
||||
def forget(marker):
|
||||
for key in drop:
|
||||
marker["keys"].pop(key, None)
|
||||
|
||||
try:
|
||||
with preview_store.write_lock:
|
||||
marker = preview_location.write_marker(folder, forget)
|
||||
except (OSError, ValueError):
|
||||
return False
|
||||
if not marker["keys"] and marker.get("created_by_pm"):
|
||||
names = set(os.listdir(folder))
|
||||
rest = {n for n in names if not files.is_os_metadata(n)}
|
||||
if rest <= {preview_location.MARKER_NAME}:
|
||||
_remove(preview_location.marker_path(folder))
|
||||
files.remove_empty_folders(folder)
|
||||
return complete
|
||||
|
||||
def _remove_store(self, store, folder, job):
|
||||
"""Remove the previews and temps of one old store; True when done."""
|
||||
store = Path(store)
|
||||
if folder:
|
||||
# A key folder directly in the old thumbnail folder, nothing else.
|
||||
if not preview_location.valid_key(store.name) or os.path.normcase(
|
||||
str(store.parent)
|
||||
) != os.path.normcase(str(folder)):
|
||||
return True
|
||||
elif store.name != "thumbnails":
|
||||
return True # a hand-edited entry never names another folder
|
||||
if not store.exists():
|
||||
return True
|
||||
if preview_location.is_link_like(store):
|
||||
return True # never through a link; nothing of ours there
|
||||
complete = True
|
||||
removed = 0
|
||||
for current, dirs, names in os.walk(store):
|
||||
dirs[:] = [
|
||||
d for d in dirs if not preview_location.is_link_like(Path(current) / d)
|
||||
]
|
||||
for name in names:
|
||||
path = Path(current) / name
|
||||
if files.is_temp_name(name) or files._preview_parts(name):
|
||||
if preview_location.is_link_like(path):
|
||||
continue
|
||||
if exit_signal.requested():
|
||||
return False # exiting: the next start finishes it
|
||||
if files.remove_file(path):
|
||||
removed += 1
|
||||
else:
|
||||
complete = False
|
||||
relative = path.relative_to(store).as_posix()
|
||||
self.logger.warning(
|
||||
f"Could not remove the old thumbnail {path}; it is "
|
||||
"removed at the next start"
|
||||
)
|
||||
if job is not None:
|
||||
# D10A: a removal that failed is an error the job lists.
|
||||
job.record(
|
||||
"errors",
|
||||
{
|
||||
"path": relative,
|
||||
"root": store.name,
|
||||
"detail": "In use by another program; removed "
|
||||
"at the next start",
|
||||
},
|
||||
f"In use by another program; removed at the next "
|
||||
f"start: {store.name}/{relative}",
|
||||
)
|
||||
files.remove_empty_folders(store)
|
||||
if job is not None and removed:
|
||||
job.log_line(f"Removed {_number(removed)} old files from {store.name}")
|
||||
return complete
|
||||
|
||||
def _commit_cleanup(self, remaining):
|
||||
from ..config import PromptManagerConfig
|
||||
from ..config_store import get_config_store
|
||||
|
||||
def mutate(config):
|
||||
section = config.get("thumbnails")
|
||||
if isinstance(section, dict) and isinstance(section.get("location"), dict):
|
||||
section["location"]["cleanup"] = remaining
|
||||
return config
|
||||
|
||||
def publish(config):
|
||||
value = (config.get("thumbnails") or {}).get("location")
|
||||
PromptManagerConfig.THUMBNAIL_LOCATION = value
|
||||
preview_store.set_cleanup(remaining)
|
||||
|
||||
get_config_store().commit(mutate, expected_revision=None, publish=publish)
|
||||
|
||||
|
||||
def plan_parent_missing(store):
|
||||
"""A key folder's parent (the thumbnail folder) is gone mid-copy."""
|
||||
return not Path(store).parent.is_dir()
|
||||
@@ -0,0 +1,348 @@
|
||||
"""The file side of Move thumbnails (#320 Part 3): what is copied, how a
|
||||
copy is made and checked, and how old copies are removed.
|
||||
|
||||
No job, HTTP or config code lives here; ``thumbnail_move`` drives it.
|
||||
|
||||
Copy set (B5): each old store is walked with ``os.scandir``, never through a
|
||||
symlink, junction or reparse point (counted as "skipped link"). A regular
|
||||
file is
|
||||
|
||||
- copied when it has the preview grammar (``<name>_thumb<ext>``, any written
|
||||
or legacy extension) and its original still exists in the source;
|
||||
- removed with the old copies, not copied, when it is an orphan (no
|
||||
original), lies under a trash folder, or is a temp sibling of a write
|
||||
(``.<stem>.part-…``) or of a move (``….pm-move-<8 hex>.tmp``);
|
||||
- left in place otherwise ("Left in place"). OS metadata files (``Thumbs.db``,
|
||||
``desktop.ini``, ``.DS_Store``, ``._*``) are removed only when they alone
|
||||
keep a preview folder from being removed.
|
||||
|
||||
A copy goes to ``<name>.pm-move-<8 hex>.tmp`` beside its destination, its
|
||||
size is compared, its mtime is set to the source preview's rounded up to the
|
||||
next 2-second step (FAT/exFAT keep it fresh, a stale preview stays stale),
|
||||
then it is renamed into place. Every handle is closed before a rename or an
|
||||
unlink (Windows).
|
||||
"""
|
||||
|
||||
import errno
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import shutil
|
||||
import stat as stat_module
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from . import preview_location, thumbnail_settings
|
||||
from .thumbnail_encode import _TEMP_NAME
|
||||
|
||||
try:
|
||||
from ...utils.media_folders import in_trash_folder
|
||||
except ImportError: # tests import the package top-level
|
||||
from utils.media_folders import in_trash_folder
|
||||
|
||||
logger = logging.getLogger("prompt_manager.thumbnails")
|
||||
|
||||
MOVE_TEMP = re.compile(r"\.pm-move-[0-9a-f]{8}\.tmp$")
|
||||
MOVE_TEMP_SUFFIX = ".pm-move-{token}.tmp"
|
||||
# The temp name adds this much to a destination path (".pm-move-xxxxxxxx.tmp").
|
||||
TEMP_EXTRA = 21
|
||||
WINDOWS_FILE_LIMIT = 259
|
||||
MTIME_STEP_NS = 2_000_000_000
|
||||
# A file another program holds open (Windows) is retried this often.
|
||||
REMOVE_RETRIES = (0.5, 1.0, 2.0)
|
||||
_WINDOWS_IN_USE = {32, 33} # ERROR_SHARING_VIOLATION, ERROR_LOCK_VIOLATION
|
||||
|
||||
|
||||
def is_os_metadata(name):
|
||||
folded = name.lower()
|
||||
return folded in preview_location.OS_METADATA or folded.startswith("._")
|
||||
|
||||
|
||||
def is_temp_name(name):
|
||||
return bool(_TEMP_NAME.match(name) or MOVE_TEMP.search(name))
|
||||
|
||||
|
||||
def _preview_parts(name):
|
||||
"""``(base, extension)`` of a preview file name, or None."""
|
||||
stem, extension = os.path.splitext(name)
|
||||
suffix = thumbnail_settings.THUMB_SUFFIX
|
||||
if not stem.endswith(suffix) or len(stem) <= len(suffix):
|
||||
return None
|
||||
if extension.lower() not in thumbnail_settings.THUMBNAIL_EXTENSIONS:
|
||||
return None
|
||||
return stem[: -len(suffix)], extension
|
||||
|
||||
|
||||
def _has_original(root, rel_dir, base, extension):
|
||||
"""Whether a media file in ``<root>/<rel_dir>`` owns this preview."""
|
||||
folder = Path(root).joinpath(*rel_dir)
|
||||
try:
|
||||
with os.scandir(folder) as entries:
|
||||
for entry in entries:
|
||||
stem, source_ext = os.path.splitext(entry.name)
|
||||
if stem != base or not source_ext:
|
||||
continue
|
||||
if not thumbnail_settings.owns(extension, source_ext):
|
||||
continue
|
||||
if entry.is_file(follow_symlinks=False):
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
@dataclass
|
||||
class StoreScan:
|
||||
"""One old store's files, sorted into what the move does with them."""
|
||||
|
||||
store: Path
|
||||
copy: list = field(default_factory=list) # (relative parts, size, mtime_ns)
|
||||
orphans: list = field(default_factory=list) # relative parts
|
||||
temps: list = field(default_factory=list)
|
||||
left: list = field(default_factory=list)
|
||||
links: int = 0
|
||||
bytes: int = 0
|
||||
|
||||
@property
|
||||
def removable(self):
|
||||
return [item[0] for item in self.copy] + self.orphans + self.temps
|
||||
|
||||
|
||||
def scan_store(store, root, *, stop=None):
|
||||
"""Sort the files of *store* (the previews of the source *root*).
|
||||
|
||||
A store that does not exist, or is itself a link, scans empty (a link
|
||||
counts once).
|
||||
"""
|
||||
store = Path(store)
|
||||
scan = StoreScan(store)
|
||||
try:
|
||||
if preview_location.is_link_like(store):
|
||||
scan.links += 1
|
||||
return scan
|
||||
if not store.is_dir():
|
||||
return scan
|
||||
except OSError:
|
||||
return scan
|
||||
_walk(scan, store, (), root, stop)
|
||||
return scan
|
||||
|
||||
|
||||
def _walk(scan, folder, rel_dir, root, stop):
|
||||
try:
|
||||
with os.scandir(folder) as iterator:
|
||||
entries = list(iterator)
|
||||
except OSError:
|
||||
return
|
||||
for entry in entries:
|
||||
if stop is not None and stop():
|
||||
return
|
||||
relative = rel_dir + (entry.name,)
|
||||
path = Path(entry.path)
|
||||
try:
|
||||
if preview_location.is_link_like(path):
|
||||
scan.links += 1
|
||||
continue
|
||||
info = entry.stat(follow_symlinks=False)
|
||||
except OSError:
|
||||
continue
|
||||
if stat_module.S_ISDIR(info.st_mode):
|
||||
_walk(scan, path, relative, root, stop)
|
||||
continue
|
||||
if not stat_module.S_ISREG(info.st_mode):
|
||||
scan.left.append(relative)
|
||||
continue
|
||||
if is_temp_name(entry.name):
|
||||
scan.temps.append(relative)
|
||||
continue
|
||||
if is_os_metadata(entry.name):
|
||||
continue # removed only when it alone keeps a folder (rmdir)
|
||||
parts = _preview_parts(entry.name)
|
||||
if parts is None:
|
||||
scan.left.append(relative)
|
||||
continue
|
||||
if in_trash_folder("/".join(relative)) or not _has_original(
|
||||
root, rel_dir, *parts
|
||||
):
|
||||
scan.orphans.append(relative)
|
||||
continue
|
||||
scan.copy.append((relative, info.st_size, info.st_mtime_ns))
|
||||
scan.bytes += info.st_size
|
||||
|
||||
|
||||
# ── copying ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def rounded_mtime_ns(mtime_ns):
|
||||
"""*mtime_ns* rounded up to the next 2-second step (FAT/exFAT)."""
|
||||
return -(-mtime_ns // MTIME_STEP_NS) * MTIME_STEP_NS
|
||||
|
||||
|
||||
def too_long(path, long_paths=None):
|
||||
"""A destination (with its temp name) over Windows' file limit."""
|
||||
if os.name != "nt" and long_paths is not False:
|
||||
return False
|
||||
if long_paths is None:
|
||||
long_paths = preview_location.long_paths_enabled()
|
||||
if long_paths:
|
||||
return False
|
||||
return len(str(path)) + TEMP_EXTRA > WINDOWS_FILE_LIMIT
|
||||
|
||||
|
||||
class CopyFailed(Exception):
|
||||
"""One preview could not be copied (a per-file error, D10A)."""
|
||||
|
||||
|
||||
class FolderGone(Exception):
|
||||
"""The destination cannot take writes any more (abort the move)."""
|
||||
|
||||
|
||||
def _make_folders(base, relative_dir, created):
|
||||
"""Make ``base/relative_dir`` one folder at a time (*base* must exist);
|
||||
every folder made is added to *created*."""
|
||||
current = Path(base)
|
||||
for part in relative_dir:
|
||||
current = current / part
|
||||
try:
|
||||
os.mkdir(current)
|
||||
created.append(str(current))
|
||||
except FileExistsError:
|
||||
pass # already there: made by an earlier copy into this folder
|
||||
except OSError as error:
|
||||
if error.errno in (errno.ENOSPC, errno.EDQUOT, errno.EROFS):
|
||||
raise FolderGone(str(error)) from error
|
||||
raise CopyFailed(
|
||||
"Its folder could not be made; made again on view"
|
||||
) from error
|
||||
if preview_location.is_link_like(current):
|
||||
raise CopyFailed("a link in the destination")
|
||||
|
||||
|
||||
def refuse_links(base, relative_dir):
|
||||
"""Raise CopyFailed when *base* or an existing folder on the way to
|
||||
``base/relative_dir`` is a link (never copy through one)."""
|
||||
current = Path(base)
|
||||
for part in (None, *relative_dir):
|
||||
if part is not None:
|
||||
current = current / part
|
||||
if preview_location.is_link_like(current):
|
||||
raise CopyFailed("a link in the destination")
|
||||
|
||||
|
||||
def copy_one(source, destination, size, mtime_ns, created, *, long_paths=None):
|
||||
"""Copy one preview; returns ``"moved"`` or ``"skipped"`` (already
|
||||
there). Raises CopyFailed (this file) or FolderGone (the destination)."""
|
||||
destination = Path(destination)
|
||||
if too_long(destination, long_paths):
|
||||
raise CopyFailed("Path too long for Windows; the original is shown")
|
||||
wanted = rounded_mtime_ns(mtime_ns)
|
||||
try:
|
||||
existing = os.stat(destination, follow_symlinks=False)
|
||||
except FileNotFoundError:
|
||||
existing = None
|
||||
except OSError as error:
|
||||
raise FolderGone(str(error)) from error
|
||||
if (
|
||||
existing is not None
|
||||
and stat_module.S_ISREG(existing.st_mode)
|
||||
and existing.st_size == size
|
||||
and existing.st_mtime_ns == wanted
|
||||
):
|
||||
return "skipped"
|
||||
temp = destination.with_name(
|
||||
destination.name + MOVE_TEMP_SUFFIX.format(token=secrets.token_hex(4))
|
||||
)
|
||||
try:
|
||||
shutil.copyfile(source, temp)
|
||||
if os.stat(temp).st_size != size:
|
||||
raise CopyFailed("The copy did not match; made again on view")
|
||||
os.utime(temp, ns=(wanted, wanted))
|
||||
os.replace(temp, destination)
|
||||
except CopyFailed:
|
||||
_unlink(temp)
|
||||
raise
|
||||
except FileNotFoundError as error:
|
||||
_unlink(temp)
|
||||
if not Path(source).exists():
|
||||
raise CopyFailed("Could not be read; made again on view") from error
|
||||
raise FolderGone(str(error)) from error
|
||||
except OSError as error:
|
||||
_unlink(temp)
|
||||
if error.errno in (errno.ENOSPC, errno.EDQUOT, errno.EROFS):
|
||||
raise FolderGone(str(error)) from error
|
||||
raise CopyFailed("Could not be copied; made again on view") from error
|
||||
return "moved"
|
||||
|
||||
|
||||
def _unlink(path):
|
||||
"""Remove a copy's temp file after a failed copy; True when it is gone.
|
||||
|
||||
One that cannot go (a Windows lock, permissions) is logged with its path;
|
||||
the next start's temp sweep removes it (``.pm-move-*.tmp``)."""
|
||||
try:
|
||||
os.unlink(path)
|
||||
return True
|
||||
except FileNotFoundError:
|
||||
return True # never written, or already renamed into place
|
||||
except OSError as error:
|
||||
logger.warning(f"Could not remove the move's temp file {path}: {error}")
|
||||
return False
|
||||
|
||||
|
||||
# ── removing ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _in_use(error):
|
||||
return getattr(error, "winerror", None) in _WINDOWS_IN_USE
|
||||
|
||||
|
||||
def remove_file(path, *, sleep=time.sleep):
|
||||
"""Remove one file; a file another program holds open (Windows) is
|
||||
retried 3 times. Returns True when it is gone."""
|
||||
for wait in (0.0,) + REMOVE_RETRIES:
|
||||
if wait:
|
||||
sleep(wait)
|
||||
try:
|
||||
os.unlink(path)
|
||||
return True
|
||||
except FileNotFoundError:
|
||||
return True
|
||||
except OSError as error:
|
||||
if not _in_use(error):
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
def remove_empty_folders(top, *, keep_top=False):
|
||||
"""``rmdir`` the empty folders below *top* (and *top*), bottom-up, never
|
||||
through a link. A folder holding only OS metadata files loses them first.
|
||||
Returns the folders that could not be removed."""
|
||||
top = Path(top)
|
||||
left = []
|
||||
if not top.is_dir() or preview_location.is_link_like(top):
|
||||
return left
|
||||
# Top-down, pruning every link, junction or reparse point, then bottom-up:
|
||||
# never into a folder outside *top* (Windows junctions included).
|
||||
folders = []
|
||||
for folder, dirs, _files in os.walk(top):
|
||||
folder = Path(folder)
|
||||
dirs[:] = [d for d in dirs if not preview_location.is_link_like(folder / d)]
|
||||
folders.append(folder)
|
||||
for folder in reversed(folders):
|
||||
if folder == top and keep_top:
|
||||
continue
|
||||
try:
|
||||
names = os.listdir(folder)
|
||||
except OSError:
|
||||
left.append(folder)
|
||||
continue
|
||||
if names and all(is_os_metadata(name) for name in names):
|
||||
for name in names:
|
||||
remove_file(folder / name)
|
||||
try:
|
||||
os.rmdir(folder)
|
||||
except OSError:
|
||||
left.append(folder)
|
||||
return left
|
||||
@@ -0,0 +1,492 @@
|
||||
"""Move thumbnails, the review (#320 Part 3 phase 1): what a move would copy,
|
||||
remove and leave, checked before anything is written."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import shutil
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from . import preview_location, preview_store
|
||||
from . import thumbnail_move_files as files
|
||||
from .job_routes import busy_response
|
||||
from .jobs import THUMBNAILS_JOB_KIND, JobBusy
|
||||
from .thumbnail_location import default_path
|
||||
|
||||
try:
|
||||
from ...utils import library_location
|
||||
except ImportError: # tests import the package top-level
|
||||
from utils import library_location
|
||||
|
||||
MOVE_TITLE = "Move thumbnails"
|
||||
FINISHING_TITLE = "Move thumbnails (finishing)"
|
||||
PRUNE_TITLE = "Remove thumbnails of removed sources"
|
||||
MOVE_GROUPS = ("moved", "skipped", "left", "errors")
|
||||
PRUNE_GROUPS = ("removed", "left", "errors")
|
||||
PLAN_TTL = 600.0
|
||||
JOURNAL_NAME = "thumbnail-move.json"
|
||||
SPACE_MARGIN = 64 * 2**20
|
||||
SPACE_SHARE = 0.05
|
||||
LISTED = 5
|
||||
SENTENCE = (
|
||||
"Every thumbnail is copied and checked before Prompt Manager switches. "
|
||||
"Scans, deletes and tagging wait until it finishes; closing this page does "
|
||||
"not stop it."
|
||||
)
|
||||
|
||||
|
||||
def _number(value):
|
||||
return f"{value:,}"
|
||||
|
||||
|
||||
def _plural(count, one, many=None):
|
||||
return f"{_number(count)} {one if count == 1 else (many or one + 's')}"
|
||||
|
||||
|
||||
def _size(value):
|
||||
size = float(value)
|
||||
for unit in ("B", "KB", "MB", "GB"):
|
||||
if size < 1024 or unit == "GB":
|
||||
return f"{size:.1f} {unit}" if unit != "B" else f"{int(size)} B"
|
||||
size /= 1024
|
||||
return f"{size:.1f} GB"
|
||||
|
||||
|
||||
def _refusal(code, status=None, **extra):
|
||||
known_status, text = preview_location.MESSAGES.get(code, (400, code))
|
||||
return web.json_response(
|
||||
{"success": False, "code": code, "error": text, **extra},
|
||||
status=status or known_status,
|
||||
)
|
||||
|
||||
|
||||
def journal_path():
|
||||
"""``<data folder>/data/thumbnail-move.json`` (only while a move copies)."""
|
||||
return library_location.cache_dir(JOURNAL_NAME)
|
||||
|
||||
|
||||
def _read_json(path):
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as handle:
|
||||
data = json.load(handle)
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
return data if isinstance(data, dict) else None
|
||||
|
||||
|
||||
def _write_json(path, data):
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
temp = f"{path}.{os.getpid()}.tmp"
|
||||
with open(temp, "w", encoding="utf-8") as handle:
|
||||
json.dump(data, handle, indent=2)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temp, path)
|
||||
|
||||
|
||||
def _remove(path):
|
||||
try:
|
||||
os.unlink(path)
|
||||
except FileNotFoundError:
|
||||
pass # already gone: removing it is the goal
|
||||
|
||||
|
||||
def _is_network(path):
|
||||
text = str(path)
|
||||
if text.startswith(("\\\\", "//")):
|
||||
return True
|
||||
if os.name == "nt":
|
||||
try:
|
||||
import ctypes
|
||||
|
||||
drive = os.path.splitdrive(os.path.abspath(text))[0] + "\\"
|
||||
return ctypes.windll.kernel32.GetDriveTypeW(drive) == 4 # DRIVE_REMOTE
|
||||
except (AttributeError, OSError):
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
def _volume(path):
|
||||
current = Path(os.path.abspath(path))
|
||||
while not current.exists() and current.parent != current:
|
||||
current = current.parent
|
||||
try:
|
||||
return os.stat(current).st_dev
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def _free(path):
|
||||
current = Path(os.path.abspath(path))
|
||||
while not current.exists() and current.parent != current:
|
||||
current = current.parent
|
||||
try:
|
||||
return shutil.disk_usage(current).free
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
# ── the plan ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass
|
||||
class SourcePlan:
|
||||
"""One listed source: its old store, its new store and what moves."""
|
||||
|
||||
root: str
|
||||
label: str
|
||||
key: str | None
|
||||
old_store: str | None
|
||||
new_store: str
|
||||
scan: files.StoreScan | None
|
||||
available: bool = True
|
||||
too_long: list = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MovePlan:
|
||||
"""A reviewed move: from, to, the sources and the removals."""
|
||||
|
||||
token: str
|
||||
created: float
|
||||
revision: str
|
||||
old: preview_location.ParsedLocation
|
||||
target: dict # {"mode", "path"}
|
||||
target_id: str | None
|
||||
target_exists: bool
|
||||
target_default: bool
|
||||
install: str
|
||||
sources: list
|
||||
unlisted: list # (key, store path, scan)
|
||||
needs_bytes: int
|
||||
confirm_required: bool
|
||||
|
||||
@property
|
||||
def to_folder(self):
|
||||
return self.target["mode"] == preview_store.FOLDER_MODE
|
||||
|
||||
@property
|
||||
def from_folder(self):
|
||||
return self.old.mode == preview_store.FOLDER_MODE
|
||||
|
||||
|
||||
def _roots_revision(roots):
|
||||
text = json.dumps(sorted(str(root) for root in roots))
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
|
||||
class ThumbnailMovePlanMixin:
|
||||
"""The move's review and its routes."""
|
||||
|
||||
_move_plans = None
|
||||
|
||||
# ── routes ────────────────────────────────────────────────────────
|
||||
|
||||
def _register_thumbnail_move_routes(self, routes):
|
||||
@routes.post("/prompt_manager/images/thumbnails/plan")
|
||||
async def thumbnail_move_plan_route(request):
|
||||
return await self.plan_thumbnail_move(request)
|
||||
|
||||
@routes.post("/prompt_manager/images/thumbnails/location/recreate")
|
||||
async def thumbnail_location_recreate_route(request):
|
||||
return await self.recreate_thumbnail_folder(request)
|
||||
|
||||
@routes.post("/prompt_manager/images/thumbnails/location/adopt")
|
||||
async def thumbnail_location_adopt_route(request):
|
||||
return await self.adopt_thumbnail_folder(request)
|
||||
|
||||
# ── review ────────────────────────────────────────────────────────
|
||||
|
||||
async def plan_thumbnail_move(self, request):
|
||||
try:
|
||||
body = await request.json()
|
||||
except (ValueError, UnicodeDecodeError):
|
||||
body = None
|
||||
if not isinstance(body, dict) or body.get("mode") != "move":
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"code": "invalid-request",
|
||||
"error": "mode must be move",
|
||||
},
|
||||
status=400,
|
||||
)
|
||||
target = body.get("location")
|
||||
if not isinstance(target, dict) or target.get("mode") not in (
|
||||
preview_store.SOURCE_MODE,
|
||||
preview_store.FOLDER_MODE,
|
||||
):
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"code": "invalid-request",
|
||||
"error": "location must name a mode",
|
||||
},
|
||||
status=400,
|
||||
)
|
||||
blocker = self._jobs.blocker(THUMBNAILS_JOB_KIND, admission="guarded")
|
||||
if blocker is not None:
|
||||
return busy_response(JobBusy(blocker))
|
||||
try:
|
||||
outcome = await self._run_in_executor(self._build_move_plan, target)
|
||||
except Exception:
|
||||
self.logger.exception("The thumbnail move could not be planned")
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"code": None,
|
||||
"error": "The move could not be planned",
|
||||
},
|
||||
status=500,
|
||||
)
|
||||
if isinstance(outcome, str):
|
||||
return _refusal(outcome)
|
||||
plan, payload = outcome
|
||||
if payload.get("insufficient"):
|
||||
return _refusal(
|
||||
"insufficient-space",
|
||||
status=507,
|
||||
error_detail=payload["insufficient"],
|
||||
plan=payload,
|
||||
)
|
||||
self._store_move_plan(plan)
|
||||
return web.json_response({"success": True, **payload})
|
||||
|
||||
def _store_move_plan(self, plan):
|
||||
if self._move_plans is None:
|
||||
self._move_plans = {}
|
||||
now = time.monotonic()
|
||||
self._move_plans = {
|
||||
token: kept
|
||||
for token, kept in self._move_plans.items()
|
||||
if now - kept.created < PLAN_TTL
|
||||
}
|
||||
self._move_plans[plan.token] = plan
|
||||
|
||||
def _build_move_plan(self, target):
|
||||
"""``(plan, payload)`` or a refusal code (blocking)."""
|
||||
old = preview_store.setting()
|
||||
location = preview_store.current()
|
||||
install = old.install or str(uuid.uuid4())
|
||||
target_path = None
|
||||
default = default_path()
|
||||
if target["mode"] == preview_store.FOLDER_MODE:
|
||||
raw = target.get("path")
|
||||
if not isinstance(raw, str):
|
||||
return "location-not-absolute"
|
||||
context = self._thumbnail_location_context()
|
||||
code = preview_location.location_problem(
|
||||
raw, purpose="move", context=context
|
||||
)
|
||||
if code is not None:
|
||||
return code
|
||||
target_path = os.path.abspath(raw)
|
||||
elif old.mode == preview_store.SOURCE_MODE and not old.problem:
|
||||
return "location-same"
|
||||
target_default = bool(default) and os.path.normcase(
|
||||
target_path or ""
|
||||
) == os.path.normcase(os.path.abspath(default))
|
||||
target_exists = bool(target_path) and os.path.isdir(target_path)
|
||||
target_marker = (
|
||||
preview_location.read_marker(target_path) if target_exists else None
|
||||
)
|
||||
roots = [str(root) for root in self._get_all_output_dirs()]
|
||||
labels = self._source_labels(roots)
|
||||
old_keys = {}
|
||||
if old.mode == preview_store.FOLDER_MODE and location.available:
|
||||
marker = preview_location.read_marker(old.path) or {}
|
||||
old_keys = marker.get("keys") or {}
|
||||
new_keys = dict((target_marker or {}).get("keys") or {})
|
||||
taken = {key: entry.get("identity") for key, entry in new_keys.items()}
|
||||
sources = []
|
||||
for root in roots:
|
||||
identity = os.path.normcase(os.path.realpath(root))
|
||||
dev_ino = preview_location.file_identity(root)
|
||||
old_key = None
|
||||
old_store = None
|
||||
available = True
|
||||
if old.mode == preview_store.SOURCE_MODE:
|
||||
old_store = str(preview_store.source_folder(root))
|
||||
elif location.available:
|
||||
old_key = preview_location.find_key(old_keys, root, identity, dev_ino)
|
||||
if old_key:
|
||||
old_store = os.path.join(old.path, old_key)
|
||||
else:
|
||||
available = False
|
||||
key = None
|
||||
if target_path:
|
||||
key = old_key or preview_location.find_key(
|
||||
new_keys, root, identity, dev_ino
|
||||
)
|
||||
if key is None or taken.get(key, identity) != identity:
|
||||
key = preview_location.source_key(root, identity, taken)
|
||||
taken[key] = identity
|
||||
new_store = os.path.join(target_path, key)
|
||||
else:
|
||||
new_store = str(preview_store.source_folder(root))
|
||||
scan = None
|
||||
if old_store:
|
||||
scan = files.scan_store(old_store, root)
|
||||
plan = SourcePlan(
|
||||
root, labels.get(root, root), key, old_store, new_store, scan, available
|
||||
)
|
||||
if scan is not None and target_path:
|
||||
plan.too_long = [
|
||||
item[0]
|
||||
for item in scan.copy
|
||||
if files.too_long(Path(new_store).joinpath(*item[0]))
|
||||
]
|
||||
sources.append(plan)
|
||||
unlisted = self._unlisted_keys(old, location, old_keys, install)
|
||||
needs = sum(source.scan.bytes for source in sources if source.scan is not None)
|
||||
plan = MovePlan(
|
||||
token=secrets.token_urlsafe(16),
|
||||
created=time.monotonic(),
|
||||
revision=_roots_revision(roots),
|
||||
old=old,
|
||||
target={"mode": target["mode"], "path": target_path},
|
||||
target_id=(target_marker or {}).get("id"),
|
||||
target_exists=target_exists,
|
||||
target_default=target_default,
|
||||
install=install,
|
||||
sources=sources,
|
||||
unlisted=unlisted,
|
||||
needs_bytes=needs,
|
||||
confirm_required=bool(target_path) and not target_default,
|
||||
)
|
||||
return plan, self._plan_payload(plan, location)
|
||||
|
||||
def _unlisted_keys(self, old, location, keys, install):
|
||||
"""This install's keys whose source no longer is in any root list
|
||||
(``(key, store, scan)``), in the folder in use."""
|
||||
if old.mode != preview_store.FOLDER_MODE or not location.available:
|
||||
return []
|
||||
listed = {
|
||||
os.path.normcase(os.path.realpath(root))
|
||||
for root in self._all_source_roots()
|
||||
}
|
||||
found = []
|
||||
for key, entry in sorted(keys.items()):
|
||||
if not preview_location.valid_key(key) or not isinstance(entry, dict):
|
||||
continue # a marker is data: never a path out of the folder
|
||||
if entry.get("install") not in (install, None):
|
||||
continue
|
||||
if entry.get("identity") in listed:
|
||||
continue
|
||||
store = os.path.join(old.path, key)
|
||||
found.append(
|
||||
(key, store, files.scan_store(store, entry.get("path") or store))
|
||||
)
|
||||
return found
|
||||
|
||||
def _source_labels(self, roots):
|
||||
from . import sources
|
||||
|
||||
labels = sources._comfyui_labels()
|
||||
return {root: sources.source_label(root, labels) for root in roots}
|
||||
|
||||
def _plan_payload(self, plan, location):
|
||||
sources = plan.sources
|
||||
copies = sum(len(s.scan.copy) for s in sources if s.scan)
|
||||
orphans = sum(len(s.scan.orphans) for s in sources if s.scan)
|
||||
temps = sum(len(s.scan.temps) for s in sources if s.scan)
|
||||
links = sum(s.scan.links for s in sources if s.scan)
|
||||
left = [
|
||||
f"{s.label} · {'/'.join(item)}"
|
||||
for s in sources
|
||||
if s.scan
|
||||
for item in s.scan.left
|
||||
]
|
||||
too_long = [
|
||||
f"{s.label} · {'/'.join(item)}" for s in sources for item in s.too_long
|
||||
]
|
||||
unlisted_count = sum(
|
||||
len(scan.copy) + len(scan.orphans) for _k, _s, scan in plan.unlisted
|
||||
)
|
||||
free = None
|
||||
insufficient = None
|
||||
if plan.needs_bytes:
|
||||
volumes = {}
|
||||
for source in sources:
|
||||
if source.scan is None or not source.scan.bytes:
|
||||
continue
|
||||
volume = _volume(source.new_store)
|
||||
entry = volumes.setdefault(volume, [source.new_store, 0])
|
||||
entry[1] += source.scan.bytes
|
||||
for where, needed in volumes.values():
|
||||
available = _free(where)
|
||||
if available is None:
|
||||
continue
|
||||
free = available if free is None else min(free, available)
|
||||
margin = max(SPACE_MARGIN, int(needed * SPACE_SHARE))
|
||||
if needed + margin > available:
|
||||
insufficient = {
|
||||
"needed": needed + margin,
|
||||
"free": available,
|
||||
"text": (
|
||||
f"There is not enough space for the thumbnails: "
|
||||
f"{_size(available)} free, {_size(needed + margin)} "
|
||||
"needed. Free some space or choose another folder."
|
||||
),
|
||||
}
|
||||
payload = {
|
||||
"mode": "move",
|
||||
"from": {
|
||||
"mode": plan.old.mode,
|
||||
"path": plan.old.path,
|
||||
"available": location.available,
|
||||
},
|
||||
"to": {
|
||||
"mode": plan.target["mode"],
|
||||
"path": plan.target["path"],
|
||||
"exists": plan.target_exists,
|
||||
"will_create": bool(plan.target["path"]) and not plan.target_exists,
|
||||
"default": plan.target_default,
|
||||
},
|
||||
"sources": [
|
||||
{
|
||||
"label": s.label,
|
||||
"key": s.key,
|
||||
"files": len(s.scan.copy) if s.scan else 0,
|
||||
"bytes": s.scan.bytes if s.scan else 0,
|
||||
"too_long": len(s.too_long),
|
||||
"links": s.scan.links if s.scan else 0,
|
||||
"orphans": len(s.scan.orphans) if s.scan else 0,
|
||||
"left_in_place": len(s.scan.left) if s.scan else 0,
|
||||
"available": s.available,
|
||||
}
|
||||
for s in sources
|
||||
],
|
||||
"totals": {
|
||||
"sources": len(sources),
|
||||
"files": copies,
|
||||
"bytes": plan.needs_bytes,
|
||||
"orphans": orphans,
|
||||
"temps": temps,
|
||||
"links": links,
|
||||
"left_in_place": len(left),
|
||||
"too_long": len(too_long),
|
||||
"unlisted_sources": len(plan.unlisted),
|
||||
"unlisted_files": unlisted_count,
|
||||
},
|
||||
"too_long_listed": too_long[:LISTED],
|
||||
"left_listed": left[:LISTED],
|
||||
"unlisted": [
|
||||
{"key": key, "files": len(scan.copy) + len(scan.orphans)}
|
||||
for key, _store, scan in plan.unlisted
|
||||
],
|
||||
"needs_bytes": plan.needs_bytes,
|
||||
"free_bytes": free,
|
||||
"confirm_required": plan.confirm_required,
|
||||
"token": plan.token,
|
||||
"sentence": SENTENCE,
|
||||
}
|
||||
if insufficient:
|
||||
payload["insufficient"] = insufficient
|
||||
return payload
|
||||
@@ -0,0 +1,246 @@
|
||||
"""Move thumbnails after a restart and the unavailable folder's actions
|
||||
(#320 Parts 3 and 4): undo an interrupted copy, finish a removal that
|
||||
waited, Recreate folder and Use this folder."""
|
||||
|
||||
import os
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from . import preview_location, preview_store
|
||||
from . import thumbnail_move_files as files
|
||||
from .job_routes import busy_response
|
||||
from .jobs import THUMBNAILS_JOB_KIND, JobBusy
|
||||
from .thumbnail_move_plan import (
|
||||
FINISHING_TITLE,
|
||||
MOVE_GROUPS,
|
||||
_read_json,
|
||||
_refusal,
|
||||
_remove,
|
||||
)
|
||||
from . import thumbnail_move_plan as move_plan
|
||||
|
||||
|
||||
class ThumbnailMoveRecoveryMixin:
|
||||
"""Startup recovery, the finishing job, Recreate and Use this folder."""
|
||||
|
||||
_move_recover_lock = threading.Lock()
|
||||
# Per API: set once recover_thumbnail_move has run (a move waits for it).
|
||||
_move_recovered = False
|
||||
|
||||
# ── startup ───────────────────────────────────────────────────────
|
||||
|
||||
def recover_thumbnail_move(self):
|
||||
"""Startup (B4): undo an interrupted copy, or forget a journal the
|
||||
switch already made true. Idempotent; the move start waits for it."""
|
||||
with self._move_recover_lock:
|
||||
if self._move_recovered:
|
||||
return
|
||||
try:
|
||||
self._recover_journal()
|
||||
except Exception:
|
||||
self.logger.warning(
|
||||
"The interrupted thumbnail move check failed", exc_info=True
|
||||
)
|
||||
self._move_recovered = True
|
||||
|
||||
def _recover_journal(self):
|
||||
path = move_plan.journal_path()
|
||||
journal = _read_json(path)
|
||||
if journal is None:
|
||||
return
|
||||
setting = preview_store.setting()
|
||||
target = journal.get("to") or {}
|
||||
now = setting.path if setting.mode == preview_store.FOLDER_MODE else None
|
||||
if setting.mode == target.get("mode") and (
|
||||
now is None
|
||||
or os.path.normcase(now) == os.path.normcase(target.get("path") or "")
|
||||
):
|
||||
_remove(path) # the switch happened; only the journal was left
|
||||
return
|
||||
for store in journal.get("stores") or []:
|
||||
if not isinstance(store, str) or not os.path.isdir(store):
|
||||
continue
|
||||
for current, dirs, names in os.walk(store):
|
||||
dirs[:] = [
|
||||
d
|
||||
for d in dirs
|
||||
if not preview_location.is_link_like(Path(current) / d)
|
||||
]
|
||||
for name in names:
|
||||
if files.MOVE_TEMP.search(name):
|
||||
files.remove_file(Path(current) / name)
|
||||
for folder in reversed(journal.get("created_dirs") or []):
|
||||
if not isinstance(folder, str) or not os.path.isdir(folder):
|
||||
continue
|
||||
for current, dirs, names in os.walk(folder):
|
||||
dirs[:] = [
|
||||
d
|
||||
for d in dirs
|
||||
if not preview_location.is_link_like(Path(current) / d)
|
||||
]
|
||||
for name in names:
|
||||
if files._preview_parts(name) or files.is_temp_name(name):
|
||||
files.remove_file(Path(current) / name)
|
||||
files.remove_empty_folders(folder)
|
||||
folder = target.get("path")
|
||||
if journal.get("created_folder") and folder and os.path.isdir(folder):
|
||||
names = set(os.listdir(folder))
|
||||
if names <= {preview_location.MARKER_NAME}:
|
||||
_remove(preview_location.marker_path(folder))
|
||||
files.remove_empty_folders(folder)
|
||||
_remove(path)
|
||||
self.logger.info("An interrupted thumbnail move was undone; nothing changed")
|
||||
|
||||
def _cleanup_reachable(self, entry):
|
||||
folder = entry.get("folder")
|
||||
if folder:
|
||||
return os.path.isdir(folder)
|
||||
return True
|
||||
|
||||
async def finish_thumbnail_move(self, _app=None):
|
||||
"""Startup: recover, then start "Move thumbnails (finishing)" when old
|
||||
copies wait and every one is reachable (on the server's loop)."""
|
||||
await self._run_in_executor(self.recover_thumbnail_move)
|
||||
cleanup = list(preview_store.setting().cleanup)
|
||||
if not cleanup:
|
||||
return None
|
||||
reachable = await self._run_in_executor(
|
||||
lambda: all(self._cleanup_reachable(entry) for entry in cleanup)
|
||||
)
|
||||
if not reachable:
|
||||
return None
|
||||
try:
|
||||
return self._jobs.start(
|
||||
THUMBNAILS_JOB_KIND,
|
||||
FINISHING_TITLE,
|
||||
admission="guarded",
|
||||
cancellable=False,
|
||||
scope={"mode": "move", "phase": "remove"},
|
||||
groups=MOVE_GROUPS,
|
||||
runner=self._run_finishing,
|
||||
)
|
||||
except JobBusy:
|
||||
return None
|
||||
|
||||
async def _run_finishing(self, job):
|
||||
await self._remove_old_copies(job, None)
|
||||
left = len(preview_store.setting().cleanup)
|
||||
self._end_move(
|
||||
job,
|
||||
"finished",
|
||||
(
|
||||
"Old thumbnails removed."
|
||||
if not left
|
||||
else (
|
||||
"Some old thumbnails are still in use; they are removed at "
|
||||
"the next start."
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
# ── Recreate and Use this folder (Part 4) ─────────────────────────
|
||||
|
||||
async def recreate_thumbnail_folder(self, _request):
|
||||
blocker = self._jobs.blocker(THUMBNAILS_JOB_KIND, admission="guarded")
|
||||
if blocker is not None:
|
||||
return busy_response(JobBusy(blocker))
|
||||
code = await self._run_in_executor(self._recreate_folder)
|
||||
if code is not None:
|
||||
status = (
|
||||
409 if code in ("location-has-marker", "location-not-empty") else 400
|
||||
)
|
||||
if code == "location-has-marker":
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"code": code,
|
||||
"error": "The folder is back with its thumbnails. Nothing "
|
||||
"was recreated.",
|
||||
},
|
||||
status=status,
|
||||
)
|
||||
return _refusal(code, status=status)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"location": self._thumbnail_location_payload(with_path=True),
|
||||
}
|
||||
)
|
||||
|
||||
def _recreate_folder(self):
|
||||
setting = preview_store.setting()
|
||||
if setting.mode != preview_store.FOLDER_MODE or not setting.path:
|
||||
return "location-same"
|
||||
folder = setting.path
|
||||
if os.path.isdir(folder):
|
||||
if preview_location.read_marker(folder) is not None:
|
||||
return "location-has-marker"
|
||||
if not preview_location.holds_only_metadata(folder):
|
||||
return "location-not-empty"
|
||||
elif not os.path.isdir(os.path.dirname(folder)):
|
||||
return "location-parent-missing"
|
||||
else:
|
||||
try:
|
||||
os.mkdir(folder)
|
||||
except OSError:
|
||||
return "location-not-writable"
|
||||
try:
|
||||
marker_id = preview_location.init_folder(folder, created_by_pm=True)
|
||||
except OSError:
|
||||
return "location-not-writable"
|
||||
return self._commit_location_id(marker_id)
|
||||
|
||||
async def adopt_thumbnail_folder(self, _request):
|
||||
blocker = self._jobs.blocker(THUMBNAILS_JOB_KIND, admission="guarded")
|
||||
if blocker is not None:
|
||||
return busy_response(JobBusy(blocker))
|
||||
code = await self._run_in_executor(self._adopt_folder)
|
||||
if code is not None:
|
||||
return _refusal(code)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"location": self._thumbnail_location_payload(with_path=True),
|
||||
}
|
||||
)
|
||||
|
||||
def _adopt_folder(self):
|
||||
setting = preview_store.setting()
|
||||
if setting.mode != preview_store.FOLDER_MODE or not setting.path:
|
||||
return "location-same"
|
||||
try:
|
||||
marker = preview_location.read_marker(setting.path)
|
||||
except OSError:
|
||||
return "location-unavailable"
|
||||
if marker is None:
|
||||
return "location-unavailable"
|
||||
return self._commit_location_id(marker["id"])
|
||||
|
||||
def _commit_location_id(self, marker_id):
|
||||
from ..config import PromptManagerConfig
|
||||
from ..config_store import get_config_store
|
||||
|
||||
value = dict(preview_store.setting().to_config())
|
||||
value["id"] = marker_id
|
||||
|
||||
def mutate(config):
|
||||
section = config.get("thumbnails")
|
||||
if not isinstance(section, dict):
|
||||
section = {}
|
||||
section["location"] = value
|
||||
config["thumbnails"] = section
|
||||
return config
|
||||
|
||||
def publish(_config):
|
||||
PromptManagerConfig.THUMBNAIL_LOCATION = value
|
||||
preview_store.configure(value, check=False)
|
||||
|
||||
try:
|
||||
get_config_store().commit(mutate, expected_revision=None, publish=publish)
|
||||
except Exception:
|
||||
self.logger.exception("The thumbnail folder could not be saved")
|
||||
return "location-not-writable"
|
||||
preview_store.refresh()
|
||||
return None
|
||||
@@ -0,0 +1,175 @@
|
||||
"""Remove thumbnails of removed sources (#320 Part 5, D200): a Thumbnail
|
||||
maintenance scope, offered only in folder mode (31A).
|
||||
|
||||
*Removed* means a key of this install in the thumbnail folder's marker whose
|
||||
identity matches no root in any root list. The job removes the
|
||||
preview-grammar files and temps of those key folders, ``rmdir``s the empty
|
||||
folders, drops the keys from the marker and lists anything else as "Left in
|
||||
place". Kind ``thumbnails``, ``mode: "prune"``.
|
||||
"""
|
||||
|
||||
import functools
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from . import preview_location, preview_store
|
||||
from . import thumbnail_move_files as files
|
||||
from .jobs import THUMBNAILS_JOB_KIND
|
||||
from .thumbnail_location import location_refusal
|
||||
from .thumbnail_move_plan import _number, _plural, _size
|
||||
|
||||
PRUNE_TITLE = "Remove thumbnails of removed sources"
|
||||
PRUNE_GROUPS = ("removed", "left", "errors")
|
||||
|
||||
|
||||
def _prune_line(count, files_count, size):
|
||||
if not count:
|
||||
return "No sources were removed: nothing to remove."
|
||||
return (
|
||||
f"{_plural(count, 'source')} no longer listed: "
|
||||
f"{_plural(files_count, 'thumbnail')} ({_size(size)}) are removed."
|
||||
)
|
||||
|
||||
|
||||
class ThumbnailPruneMixin:
|
||||
"""The prune plan and job."""
|
||||
|
||||
def _prune_targets(self):
|
||||
"""``[(key, store, scan)]`` of this install's unlisted keys (blocking)."""
|
||||
setting = preview_store.setting()
|
||||
location = preview_store.current()
|
||||
if setting.mode != preview_store.FOLDER_MODE or not location.available:
|
||||
return []
|
||||
marker = preview_location.read_marker(setting.path) or {}
|
||||
install = setting.install
|
||||
return self._unlisted_keys(setting, location, marker.get("keys") or {}, install)
|
||||
|
||||
def _prune_payload(self, targets):
|
||||
files_count = sum(
|
||||
len(scan.copy) + len(scan.orphans) for _k, _s, scan in targets
|
||||
)
|
||||
size = sum(scan.bytes for _k, _s, scan in targets)
|
||||
return {
|
||||
"success": True,
|
||||
"mode": "prune",
|
||||
"sources": [
|
||||
{
|
||||
"key": key,
|
||||
"files": len(scan.copy) + len(scan.orphans),
|
||||
"bytes": scan.bytes,
|
||||
"left_in_place": len(scan.left),
|
||||
}
|
||||
for key, _store, scan in targets
|
||||
],
|
||||
"targets": files_count,
|
||||
"bytes": size,
|
||||
"line": _prune_line(len(targets), files_count, size),
|
||||
"folder": preview_store.setting().path,
|
||||
}
|
||||
|
||||
async def get_prune_plan(self):
|
||||
refused = location_refusal()
|
||||
if refused is not None:
|
||||
return refused
|
||||
if preview_store.current().mode != preview_store.FOLDER_MODE:
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"code": "invalid-request",
|
||||
"error": "Thumbnails are kept inside each source: no removed "
|
||||
"source has thumbnails here.",
|
||||
},
|
||||
status=400,
|
||||
)
|
||||
targets = await self._run_in_executor(self._prune_targets)
|
||||
return web.json_response(
|
||||
self._prune_payload(targets), headers={"Cache-Control": "no-store"}
|
||||
)
|
||||
|
||||
def start_thumbnail_prune(self):
|
||||
"""Start the prune job (sync, on the loop); raises JobBusy."""
|
||||
return self._jobs.start(
|
||||
THUMBNAILS_JOB_KIND,
|
||||
PRUNE_TITLE,
|
||||
admission="guarded",
|
||||
cancellable=True,
|
||||
scope={"mode": "prune"},
|
||||
groups=PRUNE_GROUPS,
|
||||
runner=self._run_thumbnail_prune,
|
||||
)
|
||||
|
||||
async def _run_thumbnail_prune(self, job):
|
||||
targets = await self._run_in_executor(self._prune_targets)
|
||||
job.update(total=len(targets), processed=0)
|
||||
setting = preview_store.setting()
|
||||
for done, (key, store, scan) in enumerate(targets, 1):
|
||||
if job.cancelled:
|
||||
break
|
||||
removed, failed = await self._run_in_executor(
|
||||
functools.partial(self._prune_store, store)
|
||||
)
|
||||
job.record_counted(
|
||||
"removed", removed, [{"path": key, "detail": "Thumbnails removed"}]
|
||||
)
|
||||
if failed:
|
||||
job.record_counted("errors", failed)
|
||||
if scan.left:
|
||||
job.record_counted(
|
||||
"left",
|
||||
len(scan.left),
|
||||
[{"path": f"{key}/{'/'.join(item)}"} for item in scan.left],
|
||||
)
|
||||
await self._run_in_executor(
|
||||
functools.partial(self._forget_key, setting.path, key, store)
|
||||
)
|
||||
job.update(
|
||||
processed=done,
|
||||
progress=int(done * 100 / len(targets)),
|
||||
status_line=f"Removing thumbnails of {_number(done)} of "
|
||||
f"{_number(len(targets))} removed sources",
|
||||
)
|
||||
job.publish({"type": "progress", "summary": job.summary()})
|
||||
line = f"{_number(job.counters.get('removed', 0))} thumbnails removed"
|
||||
if job.counters.get("errors"):
|
||||
line += f" · {_plural(job.counters['errors'], 'error')}"
|
||||
job.update(status_line=line, progress=100)
|
||||
job.log_line(line)
|
||||
job.finish("cancelled" if job.cancelled else "finished")
|
||||
|
||||
@staticmethod
|
||||
def _prune_store(store):
|
||||
"""Remove the previews and temps of one key folder; ``(removed, failed)``."""
|
||||
removed = failed = 0
|
||||
store = Path(store)
|
||||
if not store.is_dir() or preview_location.is_link_like(store):
|
||||
return 0, 0
|
||||
for current, dirs, names in os.walk(store):
|
||||
dirs[:] = [
|
||||
d for d in dirs if not preview_location.is_link_like(Path(current) / d)
|
||||
]
|
||||
for name in names:
|
||||
path = Path(current) / name
|
||||
if not (files.is_temp_name(name) or files._preview_parts(name)):
|
||||
continue
|
||||
if preview_location.is_link_like(path):
|
||||
continue
|
||||
if files.remove_file(path):
|
||||
removed += 1
|
||||
else:
|
||||
failed += 1
|
||||
files.remove_empty_folders(store)
|
||||
return removed, failed
|
||||
|
||||
@staticmethod
|
||||
def _forget_key(folder, key, store):
|
||||
"""Drop *key* from the marker once its folder is gone."""
|
||||
if os.path.exists(store):
|
||||
return
|
||||
|
||||
def forget(marker):
|
||||
marker["keys"].pop(key, None)
|
||||
|
||||
with preview_store.write_lock:
|
||||
preview_location.write_marker(folder, forget)
|
||||
@@ -114,9 +114,16 @@ def validate_edge(value):
|
||||
# ── names ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def preview_stem(rel_path):
|
||||
"""``<rel without extension>_thumb`` (POSIX): a preview's name inside the
|
||||
store of its source (``preview_store``)."""
|
||||
return f"{Path(rel_path).with_suffix('').as_posix()}{THUMB_SUFFIX}"
|
||||
|
||||
|
||||
def thumbnail_stem_rel(rel_path):
|
||||
"""``thumbnails/<rel without extension>_thumb`` (POSIX)."""
|
||||
return f"thumbnails/{Path(rel_path).with_suffix('').as_posix()}{THUMB_SUFFIX}"
|
||||
"""``thumbnails/<rel without extension>_thumb`` (POSIX): URL text only.
|
||||
Where the preview is on disk is ``preview_store``'s answer."""
|
||||
return f"thumbnails/{preview_stem(rel_path)}"
|
||||
|
||||
|
||||
def thumbnail_rel(rel_path, extension=None):
|
||||
@@ -207,9 +214,20 @@ def owned_in_folder(folder, stem, source_extension, preferred=None):
|
||||
|
||||
|
||||
def _owned(root, rel_path, preferred):
|
||||
from . import preview_store
|
||||
|
||||
rel_path = Path(rel_path)
|
||||
stem = Path(root) / thumbnail_stem_rel(rel_path)
|
||||
return owned_in_folder(stem.parent, stem.name, rel_path.suffix, preferred)
|
||||
store = preview_store.store_dir(root)
|
||||
if store is None:
|
||||
return iter(())
|
||||
stem = store / preview_stem(rel_path)
|
||||
found = owned_in_folder(stem.parent, stem.name, rel_path.suffix, preferred)
|
||||
if preview_store.current().mode == preview_store.SOURCE_MODE:
|
||||
return found
|
||||
# Folder mode: timed, so a stalled share marks the folder unreachable,
|
||||
# and only previews no link takes out of the key folder (deletes use it).
|
||||
with preview_store.store_io():
|
||||
return iter([path for path in found if preview_store.may_write(path)])
|
||||
|
||||
|
||||
def owned_thumbnails(root, rel_path, preferred=None):
|
||||
|
||||
@@ -59,6 +59,9 @@ REWRITE_FAILED = (
|
||||
)
|
||||
RECLAIM_FAILED = "Reclaim space failed; the database is unchanged."
|
||||
ROW_KEPT = "could not be rewritten; it keeps its inline text and reads as before"
|
||||
# A protected value's row (#586), in the review's "not text; kept as is".
|
||||
VALUE_KEPT = "stored value not text; kept as is"
|
||||
KEPT_COLUMNS = {"workflow_data": "workflow", "prompt_metadata": "prompt graph"}
|
||||
|
||||
|
||||
NOTHING_LEFT = "Every image record already stores its workflow once"
|
||||
@@ -177,7 +180,19 @@ class WorkflowStorageRoutesMixin:
|
||||
protected += batch["protected"]
|
||||
errors += len(batch["errors"])
|
||||
job.record_counted("changed", batch["rows"])
|
||||
job.record_counted("protected", batch["protected"])
|
||||
job.record_counted(
|
||||
"protected",
|
||||
batch["protected"],
|
||||
[
|
||||
{
|
||||
"path": "",
|
||||
"detail": f"Image record {image_id}: "
|
||||
f"{KEPT_COLUMNS.get(column, column)}, {VALUE_KEPT}",
|
||||
}
|
||||
for image_id, column in batch["kept"]
|
||||
],
|
||||
complete=True,
|
||||
)
|
||||
for image_id in batch["errors"]:
|
||||
job.record(
|
||||
"errors",
|
||||
|
||||
+13
-56
@@ -78,27 +78,6 @@ def _canonical_path(path: str) -> str:
|
||||
return os.path.normcase(os.path.realpath(path))
|
||||
|
||||
|
||||
def write_private_json(path: str, data: Any) -> None:
|
||||
"""Write ``data`` as JSON to ``path`` readable by the current user only.
|
||||
|
||||
config.json can hold an API key (integrations.lora_manager), so it is
|
||||
created with mode 0600 and, on POSIX, an existing more open file is
|
||||
tightened too. Windows has no POSIX modes; there the chmod is a best
|
||||
effort that only affects the read-only bit. Parent directories are
|
||||
created as needed. Raises OSError when the file cannot be written.
|
||||
"""
|
||||
import json
|
||||
|
||||
parent_dir = os.path.dirname(path)
|
||||
if parent_dir:
|
||||
os.makedirs(parent_dir, exist_ok=True)
|
||||
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2)
|
||||
if os.name == "posix":
|
||||
os.chmod(path, 0o600)
|
||||
|
||||
|
||||
def _lexical_path(path: str) -> str:
|
||||
"""``path`` normalised for comparison without touching the filesystem.
|
||||
|
||||
@@ -717,12 +696,6 @@ class AutotagConfig:
|
||||
def suggestion_limit(cls) -> int:
|
||||
return cls.SUGGESTION_LIMIT
|
||||
|
||||
@staticmethod
|
||||
def valid_suggestion_limit(value: Any) -> bool:
|
||||
return (
|
||||
type(value) is int and 1 <= value <= MAX_SUGGESTION_LIMIT
|
||||
) # bool, float and str are rejected
|
||||
|
||||
@classmethod
|
||||
def all_instructions(cls) -> List[Dict[str, str]]:
|
||||
"""Built-in first, then the saved entries (copies)."""
|
||||
@@ -940,6 +913,8 @@ class PromptManagerConfig:
|
||||
THUMBNAIL_QUALITY = 80
|
||||
THUMBNAIL_EDGE = 512
|
||||
THUMBNAIL_EDGE_RANGE = (128, 2048)
|
||||
# #320 thumbnail location (config.json ``thumbnails.location``), as read.
|
||||
THUMBNAIL_LOCATION = None
|
||||
|
||||
@classmethod
|
||||
def max_worker_threads(cls) -> int:
|
||||
@@ -1001,6 +976,12 @@ class PromptManagerConfig:
|
||||
"format": cls.THUMBNAIL_FORMAT,
|
||||
"quality": cls.THUMBNAIL_QUALITY,
|
||||
"edge": cls.THUMBNAIL_EDGE,
|
||||
# #320: as read; only the Move thumbnails job writes it.
|
||||
**(
|
||||
{"location": cls.THUMBNAIL_LOCATION}
|
||||
if cls.THUMBNAIL_LOCATION is not None
|
||||
else {}
|
||||
),
|
||||
},
|
||||
"gallery": GalleryConfig.get_config(),
|
||||
"integrations": IntegrationConfig.get_config(),
|
||||
@@ -1055,35 +1036,6 @@ class PromptManagerConfig:
|
||||
else:
|
||||
config_logger.info(f"Config file not found: {config_path}, using defaults")
|
||||
|
||||
@classmethod
|
||||
def save_to_file(cls, config_path: Optional[str] = None):
|
||||
"""Save the current configuration to a JSON file.
|
||||
|
||||
Serializes the complete configuration (including gallery settings) to
|
||||
a JSON file. Creates the directory structure if it doesn't exist.
|
||||
|
||||
Args:
|
||||
config_path (str): Path where the JSON configuration file should be saved.
|
||||
Parent directories will be created if they don't exist.
|
||||
Defaults to :meth:`get_config_path`.
|
||||
|
||||
Raises:
|
||||
The method handles all exceptions internally and logs errors rather
|
||||
than propagating them.
|
||||
|
||||
Example:
|
||||
PromptManagerConfig.save_to_file('backup_config.json')
|
||||
PromptManagerConfig.save_to_file('/etc/comfyui/prompt_manager.json')
|
||||
"""
|
||||
if config_path is None:
|
||||
config_path = cls.get_config_path()
|
||||
|
||||
try:
|
||||
write_private_json(config_path, cls.get_config())
|
||||
config_logger.info(f"Saved configuration to {config_path}")
|
||||
except Exception as e:
|
||||
config_logger.error(f"Error saving config to {config_path}: {e}")
|
||||
|
||||
@classmethod
|
||||
def update_config(cls, new_config: Dict[str, Any]):
|
||||
"""Update configuration attributes from a dictionary.
|
||||
@@ -1175,6 +1127,11 @@ class PromptManagerConfig:
|
||||
low, high = cls.THUMBNAIL_EDGE_RANGE
|
||||
if type(edge) is int and low <= edge <= high:
|
||||
cls.THUMBNAIL_EDGE = edge
|
||||
# #320: kept as read (``preview_location.parse_location`` checks it;
|
||||
# a bad value becomes an unavailable ``invalid`` location, never a
|
||||
# silent fall-back to the sources).
|
||||
if "location" in section:
|
||||
cls.THUMBNAIL_LOCATION = section["location"]
|
||||
|
||||
|
||||
# Load configuration on import
|
||||
|
||||
+3
-2
@@ -1,10 +1,11 @@
|
||||
[project]
|
||||
name = "promptmanager"
|
||||
description = "A powerful ComfyUI custom node that extends the standard text encoder with persistent prompt storage, advanced search capabilities, and an automatic image gallery system using SQLite."
|
||||
version = "3.4.1"
|
||||
version = "3.6.0"
|
||||
license = {file = "LICENSE"}
|
||||
requires-python = ">=3.12"
|
||||
dependencies = ["watchdog>=2.1.0", "Pillow>=8.0.0"]
|
||||
dependencies = ["watchdog>=2.1.0", "Pillow>=10.1.0"]
|
||||
classifiers = ["Operating System :: OS Independent"]
|
||||
|
||||
[build-system]
|
||||
requires = ["setuptools>=84.2.1"]
|
||||
|
||||
@@ -1,18 +1,18 @@
|
||||
# Runtime dependencies
|
||||
# Runtime dependencies (Pillow, aiohttp and numpy floors match ComfyUI requirements.txt)
|
||||
watchdog>=6.0.0
|
||||
Pillow>=12.3.0
|
||||
aiohttp>=3.14.5
|
||||
Pillow>=10.1.0
|
||||
aiohttp>=3.11.8
|
||||
|
||||
# Tests and coverage
|
||||
coverage>=7.16.2
|
||||
numpy>=2.2.6 # WD14 engine tests
|
||||
coverage>=7.17.0
|
||||
numpy>=1.25.0 # WD14 engine tests
|
||||
|
||||
# Code quality
|
||||
black==26.10.1
|
||||
flake8==7.4.1
|
||||
flake8==7.5.0
|
||||
|
||||
# Security scanning
|
||||
bandit[toml]>=1.9.4
|
||||
|
||||
# Pre-commit hooks
|
||||
pre-commit>=4.6.2
|
||||
pre-commit>=4.7.0
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
# Runtime dependencies for PromptManager.
|
||||
# aiohttp, numpy and torch come from ComfyUI itself and are not listed here.
|
||||
watchdog>=2.1.0 # Output directory monitoring
|
||||
Pillow>=8.0.0 # Image metadata extraction and thumbnails
|
||||
Pillow>=10.1.0 # Image metadata extraction and thumbnails
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 1.5 MiB |
@@ -0,0 +1,47 @@
|
||||
// #615: each old page says it will be removed and links to where its job lives now. The page keeps working, and the
|
||||
// notice takes its own room: nothing is pushed off a page whose layout fills the window.
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
|
||||
const PAGES = Object.freeze([
|
||||
{ path: '/prompt_manager/admin', section: 'library', heading: 'Prompt library' },
|
||||
{ path: '/prompt_manager/gallery', section: 'images', heading: 'Images' },
|
||||
{ path: '/prompt_manager/gallery.html', section: 'inspect', heading: 'Inspect a file' },
|
||||
]);
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
await expect(await request.post('/__test__/reset', {})).toBeOK();
|
||||
await page.addInitScript(() => {
|
||||
localStorage.setItem('pm_update_notice_dismissed', '3.2.0');
|
||||
localStorage.setItem('gallerySettings', JSON.stringify({ checkThumbnailsAtStartup: false }));
|
||||
});
|
||||
});
|
||||
|
||||
test.afterEach(async ({ page, browserName }) => {
|
||||
await saveCoverage(page, browserName);
|
||||
});
|
||||
|
||||
for (const { path, section, heading } of PAGES) {
|
||||
test(`${path} says it will be removed and links to ${section}`, async ({ page }) => {
|
||||
await page.goto(path);
|
||||
const notice = page.locator('#legacy-removal-notice');
|
||||
await expect(notice).toBeVisible();
|
||||
await expect(notice).toContainText('This page will be removed in a future release.');
|
||||
await notice.getByRole('link', { name: 'Open the workspace' }).click();
|
||||
await expect(page).toHaveURL(new RegExp(`/prompt_manager/workspace#/${section}$`));
|
||||
await expect(page.locator('#workspace-main h1')).toHaveText(heading);
|
||||
});
|
||||
}
|
||||
|
||||
test('the notice leaves the metadata page filling the window', async ({ page }) => {
|
||||
await page.goto('/prompt_manager/gallery.html');
|
||||
await expect(page.locator('#legacy-removal-notice')).toBeVisible();
|
||||
const { bottom, height, scroll } = await page.locator('#image-area').evaluate(area => ({
|
||||
bottom: area.getBoundingClientRect().bottom,
|
||||
height: window.innerHeight,
|
||||
scroll: document.documentElement.scrollHeight,
|
||||
}));
|
||||
expect(bottom).toBeLessThanOrEqual(height);
|
||||
expect(scroll).toBeLessThanOrEqual(height);
|
||||
});
|
||||
@@ -21,8 +21,9 @@ export function mapLibraryConcept(reference) {
|
||||
// Save view is the concept's own (saved views, #249 D16 retired).
|
||||
const toolbar={...library.toolbar};
|
||||
toolbar.items=toolbar.items.map(item=>{
|
||||
// D14: the Library appends its Oldest first after the concept's sorts; D153 (#447) drops "Highest rated".
|
||||
if(item.id==='sort')return {...item,options:[...item.options.filter(option=>option!=='Highest rated'),'Oldest first']};
|
||||
// D14: the Library appends its Oldest first after the concept's sorts; D153 (#447) drops "Highest rated"; D205
|
||||
// (#620) appends the old Dashboard's three sorts.
|
||||
if(item.id==='sort')return {...item,options:[...item.options.filter(option=>option!=='Highest rated'),'Oldest first','Recently used','Most used','Name Z–A']};
|
||||
// D36: the category list is the Library's real list, sorted alphabetically; the concept lists prompt order.
|
||||
if(item.id==='category-filter')return {...item,options:[item.options[0],...item.options.slice(1).sort(nocase)]};
|
||||
return item;
|
||||
|
||||
@@ -22,6 +22,9 @@ from PIL.PngImagePlugin import PngInfo
|
||||
from tests.browser.folder_trash import close_trash, replace_folder
|
||||
from tests.browser.malloc_tuning import pin_mmap_threshold
|
||||
from tests.browser.request_hold import register_request_holds
|
||||
from tests.browser.thumbnail_location_fixture import (
|
||||
register_thumbnail_location_commands,
|
||||
)
|
||||
from tests.browser import slow_reset_watch
|
||||
from tests.browser.fast_reset import SEED_ON_LIVE_ENV, ScratchSeeder
|
||||
from tests.browser.workspace_activity_fixture import register_activity_commands
|
||||
@@ -249,6 +252,7 @@ def create_app(root: Path) -> web.Application:
|
||||
library_editor_commands = register_library_editor_commands(routes, api, root)
|
||||
register_vocabulary_commands(routes, api, root)
|
||||
restore_sources = register_source_change_commands(routes, api)
|
||||
reset_thumbnail_location = register_thumbnail_location_commands(routes, api, root)
|
||||
register_tagging_commands(routes, api, root, model, on_reset=restore_sources)
|
||||
register_review_commands(routes, api, root)
|
||||
register_collections_commands(routes, api, root)
|
||||
@@ -460,6 +464,8 @@ def create_app(root: Path) -> web.Application:
|
||||
restore_sources()
|
||||
# No preview queued by the last test may land in the new fixture.
|
||||
await api._run_in_executor(api.close_auto_thumbnails)
|
||||
# Previews inside each source again (#320).
|
||||
await api._run_in_executor(reset_thumbnail_location)
|
||||
library_editor_commands.reset()
|
||||
await api._run_in_executor(reset_logs)
|
||||
data = await request.json() if request.can_read_body else {}
|
||||
@@ -604,6 +610,7 @@ def create_app(root: Path) -> web.Application:
|
||||
history_commands.reset()
|
||||
library_editor_commands.reset()
|
||||
reveal_stub.reset()
|
||||
await api._run_in_executor(reset_thumbnail_location)
|
||||
history_fixture = None
|
||||
manifest = await api._run_in_executor(
|
||||
seeder.run,
|
||||
|
||||
@@ -167,8 +167,6 @@ const IMAGES_COUNT_URL = '/prompt_manager/images/output?mode=workspace&limit=1';
|
||||
|
||||
/** The concept's shell and Overview with the recorded deviations applied, given the server's real overview. */
|
||||
function mapOverviewConcept(reference, overview, images) {
|
||||
// D165: the discreet legacy-pages line follows the settings layout.
|
||||
reference = { ...reference, tree: [...reference.tree, 'p.legacy-pages.small'] };
|
||||
const relabel = { 'Database integrity': 'Not checked', 'Positive prompts': number(overview.library.positive_prompts),
|
||||
'Negative variants': number(overview.library.negative_variants), Images: number(images) };
|
||||
const rows = {
|
||||
@@ -265,7 +263,7 @@ const UNMAPPED_PATHS = [
|
||||
'columns.surfaces.0.rows.length', 'columns.surfaces.0.tree.4', 'columns.surfaces.0.tree.length',
|
||||
'columns.surfaces.1.rows.0.description', 'columns.surfaces.1.rows.1.title', 'columns.surfaces.1.rows.2.description',
|
||||
'heading.actions.0', 'heading.actions.length', 'metrics.items.3.value', 'nav.label',
|
||||
'save.button.cursor', 'save.button.opacity', 'save.status.text', 'tree.3', 'tree.length',
|
||||
'save.button.cursor', 'save.button.opacity', 'save.status.text',
|
||||
].sort();
|
||||
|
||||
test('system comparison detects mutations', async ({ page, request }) => {
|
||||
@@ -656,9 +654,10 @@ function mapSourcesConcept(reference, { settings, sources }) {
|
||||
// D113: the range ends at the server's core count (the concept's fixed 16) and shows the real worker count.
|
||||
return [{ ...row, control: { ...row.control, max: String(settings.cpu_count), value: String(settings.worker_threads) } }];
|
||||
}),
|
||||
// D158: the inserted row after the first setting row; D98: "Refresh files" closes the surface.
|
||||
// D158: the inserted row after the first setting row; D196 (#320): the Thumbnail location block after
|
||||
// "Thumbnail generation"; D98: "Refresh files" closes the surface.
|
||||
surface: { ...reference.surface, tree: [...reference.surface.tree.slice(0, rowAt + 1), 'div.setting-row',
|
||||
...reference.surface.tree.slice(rowAt + 1), 'div.operation'] },
|
||||
...reference.surface.tree.slice(rowAt + 1), 'div.setting-block', 'div.operation'] },
|
||||
operations: [...reference.operations, { ...reference.operations[0], title: REFRESH_FILES.title,
|
||||
description: REFRESH_FILES.description, actions: [{ ...reference.operations[0].actions[0], text: REFRESH_FILES.label }] }],
|
||||
};
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
"""The thumbnail location on the test server (#320), undone by every reset.
|
||||
|
||||
Folder mode is reachable from config.json and tests only until the Move
|
||||
thumbnails job lands, so page tests set it here, inside the server's root:
|
||||
|
||||
- ``POST /__test__/thumbnail-location`` ``{"mode": "folder"}`` (or
|
||||
``"source"``): previews live in ``<root>/thumbs/<key>`` from now on (a new
|
||||
folder with Prompt Manager's marker).
|
||||
- ``GET /__test__/thumbnail-files``: the preview files in the thumbnail
|
||||
folder and whether any source grew a ``thumbnails/`` folder.
|
||||
- ``POST /__test__/thumbnail-folder/remove``: the thumbnail folder goes, as
|
||||
when its drive is unplugged; the server notices it on its next check.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Callable
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from py.api import PromptManagerAPI
|
||||
|
||||
FOLDER_NAME = "thumbs"
|
||||
|
||||
|
||||
def register_thumbnail_location_commands(
|
||||
routes: web.RouteTableDef, api: PromptManagerAPI, root: Path
|
||||
) -> Callable[[], None]:
|
||||
"""Register the commands; returns the reset every reset runs."""
|
||||
from py.api import preview_location, preview_store
|
||||
from py.api import thumbnail_location, thumbnail_move_plan
|
||||
from py.config import PromptManagerConfig
|
||||
from py.config_store import get_config_store
|
||||
|
||||
folder = root / FOLDER_NAME
|
||||
default = root / "default-thumbs"
|
||||
custom = root / "custom-thumbs" # a folder a page types in
|
||||
journal = root / "data" / "thumbnail-move.json"
|
||||
# The move's journal and default folder live in this server's root, never
|
||||
# in the checkout's data folder.
|
||||
thumbnail_move_plan.journal_path = lambda: str(journal)
|
||||
thumbnail_move_plan.default_path = lambda: str(default)
|
||||
thumbnail_location.default_path = lambda: str(default)
|
||||
api.recover_thumbnail_move()
|
||||
|
||||
def forget_config(config: dict) -> dict:
|
||||
section = config.get("thumbnails")
|
||||
if isinstance(section, dict):
|
||||
section.pop("location", None)
|
||||
return config
|
||||
|
||||
real_free = thumbnail_move_plan._free
|
||||
|
||||
def reset() -> None:
|
||||
thumbnail_move_plan._free = real_free
|
||||
preview_store.configure(None)
|
||||
PromptManagerConfig.THUMBNAIL_LOCATION = None
|
||||
store = get_config_store()
|
||||
config, _revision = store.read()
|
||||
if (
|
||||
isinstance(config.get("thumbnails"), dict)
|
||||
and "location" in config["thumbnails"]
|
||||
):
|
||||
store.commit(forget_config, expected_revision=None)
|
||||
for path in (folder, default, custom):
|
||||
shutil.rmtree(path, ignore_errors=True)
|
||||
journal.unlink(missing_ok=True)
|
||||
|
||||
def use(mode: str) -> dict:
|
||||
reset()
|
||||
if mode == "source":
|
||||
return {"mode": "source"}
|
||||
folder.mkdir()
|
||||
marker_id = preview_location.init_folder(str(folder), created_by_pm=True)
|
||||
location = preview_store.configure(
|
||||
{
|
||||
"mode": "folder",
|
||||
"path": str(folder),
|
||||
"id": marker_id,
|
||||
"install": "browser-tests",
|
||||
}
|
||||
)
|
||||
return {"mode": location.mode, "available": location.available}
|
||||
|
||||
@routes.post("/__test__/thumbnail-location")
|
||||
async def set_location(request: web.Request) -> web.Response:
|
||||
data = await request.json()
|
||||
mode = data.get("mode") if isinstance(data, dict) else None
|
||||
if mode not in ("source", "folder"):
|
||||
raise web.HTTPBadRequest()
|
||||
return web.json_response(await api._run_in_executor(use, mode))
|
||||
|
||||
def files() -> dict:
|
||||
setting = preview_store.setting()
|
||||
where = Path(setting.path) if setting.path else folder
|
||||
previews = sorted(
|
||||
path.relative_to(where).as_posix()
|
||||
for path in where.rglob("*_thumb.*")
|
||||
if path.is_file()
|
||||
)
|
||||
sources = [str(path) for path in api._get_all_output_dirs()]
|
||||
return {
|
||||
"folder": previews,
|
||||
"folder_exists": folder.exists() or default.exists() or custom.exists(),
|
||||
"custom_path": str(custom),
|
||||
"source_thumbnails": any(
|
||||
(Path(source) / "thumbnails").exists() for source in sources
|
||||
),
|
||||
}
|
||||
|
||||
@routes.get("/__test__/thumbnail-files")
|
||||
async def thumbnail_files(_request: web.Request) -> web.Response:
|
||||
return web.json_response(await api._run_in_executor(files))
|
||||
|
||||
def remove() -> bool:
|
||||
setting = preview_store.setting()
|
||||
shutil.rmtree(setting.path or folder, ignore_errors=True)
|
||||
return preview_store.refresh()
|
||||
|
||||
@routes.post("/__test__/thumbnail-folder/remove")
|
||||
async def remove_folder(_request: web.Request) -> web.Response:
|
||||
available = await api._run_in_executor(remove)
|
||||
return web.json_response({"available": available})
|
||||
|
||||
def foreign() -> bool:
|
||||
"""Another install recreated the folder: its marker has another id."""
|
||||
setting = preview_store.setting()
|
||||
|
||||
def other(marker: dict) -> None:
|
||||
marker["id"] = "another-install"
|
||||
|
||||
preview_location.write_marker(setting.path, other)
|
||||
return preview_store.refresh()
|
||||
|
||||
@routes.post("/__test__/thumbnail-folder/foreign")
|
||||
async def foreign_folder(_request: web.Request) -> web.Response:
|
||||
return web.json_response({"available": await api._run_in_executor(foreign)})
|
||||
|
||||
def unlisted() -> None:
|
||||
"""A source this install no longer lists left its thumbnails here."""
|
||||
setting = preview_store.setting()
|
||||
|
||||
def add(marker: dict) -> None:
|
||||
marker["keys"]["gone-0123abcd"] = {
|
||||
"path": "/gone",
|
||||
"identity": "/gone",
|
||||
"install": setting.install,
|
||||
}
|
||||
|
||||
preview_location.write_marker(setting.path, add)
|
||||
gone = Path(setting.path) / "gone-0123abcd"
|
||||
gone.mkdir()
|
||||
(gone / "x_thumb.webp").write_bytes(b"preview")
|
||||
preview_store.refresh()
|
||||
|
||||
@routes.post("/__test__/thumbnail-folder/unlisted")
|
||||
async def unlisted_source(_request: web.Request) -> web.Response:
|
||||
await api._run_in_executor(unlisted)
|
||||
return web.json_response({"success": True})
|
||||
|
||||
@routes.post("/__test__/thumbnail-space/full")
|
||||
async def space_full(_request: web.Request) -> web.Response:
|
||||
"""Every target disk reports 1 byte free until the next reset."""
|
||||
thumbnail_move_plan._free = lambda _path: 1
|
||||
return web.json_response({"success": True})
|
||||
|
||||
return reset
|
||||
@@ -261,3 +261,20 @@ test('prompt deletion requires confirmation and can retry after a network failur
|
||||
await expect(await request.get(url)).toBeOK();
|
||||
}
|
||||
});
|
||||
|
||||
// #320: with previews in one folder, the legacy gallery still shows them, made
|
||||
// and served from that folder (the fixture's own previews inside the sources
|
||||
// are never served then).
|
||||
test('the legacy gallery shows thumbnails kept in one folder', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/thumbnail-location', { data: { mode: 'folder' } })).toBeOK();
|
||||
const served = page.waitForResponse(response =>
|
||||
response.url().includes('/prompt_manager/images/serve/thumbnails/') && response.status() === 200);
|
||||
await page.goto('/prompt_manager/gallery');
|
||||
await expect(page.locator('#galleryGrid .image-item')).toHaveCount(2);
|
||||
const response = await served; // made in the folder before it was served
|
||||
const name = decodeURIComponent(new URL(response.url()).pathname.split('/serve/thumbnails/')[1]);
|
||||
const files = await (await request.get('/__test__/thumbnail-files')).json();
|
||||
const stem = name.replace(/\.[^.]+$/, '');
|
||||
expect(files.folder.some(file => /^[a-z0-9_-]+-[0-9a-f]{8}\//.test(file)
|
||||
&& file.slice(file.indexOf('/') + 1).replace(/\.[^.]+$/, '') === stem)).toBe(true);
|
||||
});
|
||||
|
||||
@@ -191,7 +191,7 @@ test('library toolbar matches the concept and tags live in Refine this view',asy
|
||||
expect(await toolbar.evaluate(node=>[...node.children].map(child=>child.tagName.toLowerCase()+(child.id?`#${child.id}`:'')+(child.className?`.${child.className.split(' ').filter(name=>!name.startsWith('ca-')).join('.')}`:'')))).toEqual(['label.search-field','select#category-filter','select#sort','button','button']);
|
||||
await expect(toolbar.locator('#search')).toHaveAttribute('placeholder','Search prompts, tags, filenames…');
|
||||
// D153 (#447): no "Highest rated".
|
||||
await expect(toolbar.locator('#sort option')).toHaveText(['Recently added','Name A–Z','Oldest first']);
|
||||
await expect(toolbar.locator('#sort option')).toHaveText(['Recently added','Name A–Z','Oldest first','Recently used','Most used','Name Z–A']);
|
||||
await expect(toolbar.getByRole('button')).toHaveText(['Save view','Filters']);
|
||||
await expect(page.getByRole('button',{name:'Refresh library',exact:true})).toHaveCount(0);
|
||||
await expect(page.getByRole('textbox',{name:'Tags',exact:true})).toHaveCount(0);
|
||||
|
||||
@@ -205,7 +205,7 @@ test('a view stored under older rules still lists and opens with its usable valu
|
||||
return (await response.json()).id;
|
||||
};
|
||||
const imagesId = await raw({ page: 'images', name: 'Old cats', state: JSON.stringify({ q: 'cat', sort: 'shuffle', layout: 'grid' }) });
|
||||
await raw({ page: 'library', name: '', state: JSON.stringify({ q: 'dog', sort: 'name-desc' }) });
|
||||
await raw({ page: 'library', name: '', state: JSON.stringify({ q: 'dog', sort: 'shuffle' }) });
|
||||
expect(await listed(request)).toEqual([
|
||||
{ id: imagesId, page: 'images', name: 'Old cats', state: { ...IMAGE_DEFAULTS, q: 'cat' } },
|
||||
expect.objectContaining({ page: 'library', name: 'Saved view', state: { q: 'dog', category: '', sort: 'newest', tags: [], tag_mode: 'all', folder: '' } }),
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { seedConcept } from './concept-inventory.mjs';
|
||||
|
||||
// #320 Thumbnail location (D196–D201, the approved 320 mock) on the concept-shaped real fixture: the row in System ›
|
||||
// Sources & scans reviews and runs Move thumbnails both ways, the review refuses a folder in place, and an unplugged
|
||||
// thumbnail folder shows the notice whose Recreate folder brings it back. The hooks are thumbnail_location_fixture.py's.
|
||||
const workspace = '/prompt_manager/workspace';
|
||||
const NOTICE_KEY = 'pm_update_notice_dismissed';
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
await seedConcept(request);
|
||||
await expect(await request.post('/__test__/system/reset')).toBeOK();
|
||||
await expect(await request.post('/__test__/thumbnail-location', { data: { mode: 'source' } })).toBeOK();
|
||||
page.locationErrors = [];
|
||||
page.on('pageerror', error => page.locationErrors.push(error.message));
|
||||
await page.addInitScript(key => { if (location.protocol.startsWith('http')) localStorage.setItem(key, '3.2.0'); }, NOTICE_KEY);
|
||||
});
|
||||
|
||||
test.afterEach(async ({ page, request, browserName }) => {
|
||||
await saveCoverage(page, browserName);
|
||||
await request.post('/__test__/thumbnail-location', { data: { mode: 'source' } });
|
||||
expect(page.locationErrors).toEqual([]);
|
||||
});
|
||||
|
||||
const block = page => page.locator('#location-block');
|
||||
const dialog = page => page.locator('dialog#modal[open]');
|
||||
|
||||
async function files(request) {
|
||||
const response = await request.get('/__test__/thumbnail-files');
|
||||
await expect(response).toBeOK();
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async function openSources(page) {
|
||||
await page.goto(`${workspace}#/system/sources`);
|
||||
await expect(block(page).locator('#location-status')).not.toHaveText('');
|
||||
}
|
||||
|
||||
/** Review the row's choice and start the move; resolves once its job view says it finished. */
|
||||
async function moveFromRow(page) {
|
||||
const planned = page.waitForResponse(response => response.url().endsWith('/images/thumbnails/plan')
|
||||
&& response.request().method() === 'POST');
|
||||
await block(page).getByRole('button', { name: 'Review move…' }).click();
|
||||
expect((await planned).status()).toBe(200);
|
||||
await expect(dialog(page).locator('h2')).toHaveText('Move thumbnails');
|
||||
await dialog(page).getByRole('button', { name: 'Move thumbnails' }).click();
|
||||
await expect(page.locator('#workspace-main h1')).toHaveText('Move thumbnails');
|
||||
await expect(page.locator('#workspace-main')).toContainText('Thumbnails moved to');
|
||||
}
|
||||
|
||||
test('the row moves thumbnails into one folder and back', async ({ page, request }) => {
|
||||
await openSources(page);
|
||||
const row = block(page);
|
||||
await expect(row.getByRole('radio', { name: /Inside each source/ })).toBeChecked();
|
||||
await expect(row.locator('#location-status')).toHaveText('Now: inside each source.');
|
||||
await expect(row.getByRole('button', { name: 'Review move…' })).toHaveAttribute('aria-disabled', 'true');
|
||||
await expect(row.locator('#location-path')).toBeDisabled();
|
||||
|
||||
await row.getByRole('radio', { name: /One folder/ }).check();
|
||||
await expect(row.locator('#location-path')).toBeEnabled();
|
||||
await expect(row.locator('#location-path')).toHaveValue(/default-thumbs$/);
|
||||
await expect(row.locator('#location-status')).toHaveText(/^Now: inside each source\. The suggested folder is next to your library/);
|
||||
await expect(row.getByRole('button', { name: 'Review move…' })).toHaveAttribute('aria-disabled', 'false');
|
||||
await moveFromRow(page);
|
||||
await expect(page.locator('#job-phase')).toHaveText(/^Thumbnails are now read from the new location/);
|
||||
const moved = await files(request);
|
||||
expect(moved.folder.length).toBeGreaterThan(0);
|
||||
expect(moved.folder.every(name => /^[a-z0-9_-]+-[0-9a-f]{8}\//.test(name))).toBe(true);
|
||||
expect(moved.source_thumbnails).toBe(false);
|
||||
|
||||
// Images shows thumbnails served from the folder.
|
||||
const served = page.waitForResponse(response => response.url().includes('/images/serve/thumbnails/')
|
||||
&& response.status() === 200);
|
||||
await page.goto(`${workspace}#/images`);
|
||||
await served;
|
||||
|
||||
await openSources(page);
|
||||
await expect(block(page).locator('#location-status')).toHaveText(/^Now: one folder, .*default-thumbs\. Using the default folder/);
|
||||
await block(page).getByRole('radio', { name: /Inside each source/ }).check();
|
||||
await moveFromRow(page);
|
||||
const back = await files(request);
|
||||
expect(back.source_thumbnails).toBe(true);
|
||||
expect(back.folder_exists).toBe(false);
|
||||
});
|
||||
|
||||
test('a folder the server refuses is explained under the field', async ({ page }) => {
|
||||
await openSources(page);
|
||||
const row = block(page);
|
||||
await row.getByRole('radio', { name: /One folder/ }).check();
|
||||
await row.locator('#location-path').fill('thumbnails');
|
||||
await row.getByRole('button', { name: 'Review move…' }).click();
|
||||
await expect(row.locator('#location-problem')).toHaveText(
|
||||
'Enter a full folder path, such as D:\\Thumbnails or /mnt/fast/thumbnails.');
|
||||
await expect(row.locator('#location-path')).toHaveAttribute('aria-invalid', 'true');
|
||||
await expect(row.locator('#location-path')).toHaveAttribute('aria-describedby', 'location-problem');
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
await row.locator('#location-path').fill('/somewhere/else');
|
||||
await expect(row.locator('#location-problem')).toHaveText('');
|
||||
});
|
||||
|
||||
test('an unplugged folder shows the notice and Recreate folder brings it back', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/thumbnail-location', { data: { mode: 'folder' } })).toBeOK();
|
||||
const removed = await request.post('/__test__/thumbnail-folder/remove');
|
||||
expect((await removed.json()).available).toBe(false);
|
||||
await openSources(page);
|
||||
const notice = page.locator('#unavailable-notice');
|
||||
await expect(notice.locator('h2')).toHaveText('The thumbnail folder is unavailable');
|
||||
await expect(notice).toContainText('is missing. Originals are shown and no thumbnails are written until it is back.');
|
||||
await expect(block(page).locator('#location-status')).toHaveText(/· unavailable\. Originals are shown\.$/);
|
||||
await notice.getByRole('button', { name: 'Recreate folder' }).click();
|
||||
await expect(dialog(page).locator('h2')).toHaveText('Recreate the thumbnail folder?');
|
||||
const recreated = page.waitForResponse(response => response.url().endsWith('/location/recreate'));
|
||||
await dialog(page).getByRole('button', { name: 'Recreate folder' }).click();
|
||||
expect((await recreated).status()).toBe(200);
|
||||
await expect(notice).toHaveCount(0);
|
||||
await expect(block(page).locator('#location-status')).toHaveText(/^Now: one folder, .*thumbs\.$/);
|
||||
});
|
||||
|
||||
test('a folder of your own needs the confirmation before it moves', async ({ page, request }) => {
|
||||
const custom = (await files(request)).custom_path;
|
||||
await openSources(page);
|
||||
const row = block(page);
|
||||
await row.getByRole('radio', { name: /One folder/ }).check();
|
||||
await row.locator('#location-path').fill(custom);
|
||||
await row.getByRole('button', { name: 'Review move…' }).click();
|
||||
const review = dialog(page);
|
||||
await expect(review.locator('dt')).toContainText(['From', 'To', 'Thumbnails', 'Free space', 'Removed after the move']);
|
||||
await expect(review.locator('dd').nth(1)).toHaveText(`${custom} (new folder)`);
|
||||
const move = review.getByRole('button', { name: 'Move thumbnails' });
|
||||
await expect(move).toHaveAttribute('aria-disabled', 'true');
|
||||
await review.getByRole('checkbox', { name: /may create its own folders/ }).check();
|
||||
await expect(move).toHaveAttribute('aria-disabled', 'false');
|
||||
await move.click();
|
||||
await expect(page.locator('#workspace-main')).toContainText('Thumbnails moved to');
|
||||
expect((await files(request)).folder.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('another folder’s marker offers Use this folder', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/thumbnail-location', { data: { mode: 'folder' } })).toBeOK();
|
||||
await expect(await request.post('/__test__/thumbnail-folder/foreign')).toBeOK();
|
||||
await openSources(page);
|
||||
const notice = page.locator('#unavailable-notice');
|
||||
await expect(notice).toContainText('holds another thumbnail folder');
|
||||
await notice.getByRole('button', { name: 'Use this folder' }).click();
|
||||
await expect(notice).toHaveCount(0);
|
||||
await expect(block(page).locator('#location-status')).toHaveText(/^Now: one folder, .*thumbs\.$/);
|
||||
});
|
||||
|
||||
test('Overview shows the notice and Change location opens the row', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/thumbnail-location', { data: { mode: 'folder' } })).toBeOK();
|
||||
const removed = await request.post('/__test__/thumbnail-folder/remove');
|
||||
expect((await removed.json()).available).toBe(false);
|
||||
await page.goto(`${workspace}#/system/overview`);
|
||||
const notice = page.locator('#unavailable-notice');
|
||||
await expect(notice.locator('h2')).toHaveText('The thumbnail folder is unavailable');
|
||||
await notice.getByRole('button', { name: 'Change location' }).click();
|
||||
await expect(page).toHaveURL(/#\/system\/sources$/);
|
||||
const sourcesNotice = page.locator('#unavailable-notice');
|
||||
await sourcesNotice.getByRole('button', { name: 'Change location' }).click();
|
||||
await expect(block(page).getByRole('radio', { name: /One folder/ })).toBeChecked();
|
||||
await expect(block(page).locator('#location-path')).toBeFocused();
|
||||
});
|
||||
|
||||
test('Overview’s Use this folder takes the folder and the notice goes', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/thumbnail-location', { data: { mode: 'folder' } })).toBeOK();
|
||||
await expect(await request.post('/__test__/thumbnail-folder/foreign')).toBeOK();
|
||||
await page.goto(`${workspace}#/system/overview`);
|
||||
const notice = page.locator('#unavailable-notice');
|
||||
await expect(notice).toContainText('holds another thumbnail folder');
|
||||
const adopted = page.waitForResponse(response => response.url().endsWith('/location/adopt'));
|
||||
await notice.getByRole('button', { name: 'Use this folder' }).click();
|
||||
expect((await adopted).status()).toBe(200);
|
||||
await expect(notice).toHaveCount(0);
|
||||
});
|
||||
|
||||
test('Thumbnail maintenance removes the thumbnails of removed sources in folder mode', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/thumbnail-location', { data: { mode: 'folder' } })).toBeOK();
|
||||
await expect(await request.post('/__test__/thumbnail-folder/unlisted')).toBeOK();
|
||||
await openSources(page);
|
||||
const generation = page.locator('#workspace-main .operation').filter({ hasText: 'Thumbnail generation' });
|
||||
await generation.getByRole('button', { name: 'Review options' }).click();
|
||||
const scope = dialog(page).locator('#thumb-mode');
|
||||
await expect(scope.locator('option')).toHaveText(['Missing thumbnails only', 'Rebuild all (explicit opt-in)',
|
||||
'Convert outdated (removes older formats)', 'Remove thumbnails of removed sources']);
|
||||
await scope.selectOption('prune');
|
||||
await expect(dialog(page).locator('[role=status]')).toHaveText(
|
||||
/^1 source no longer listed: 1 thumbnail \(.+\) are removed\.$/);
|
||||
await expect(dialog(page).locator('dl.meta-grid dt')).toHaveText(['gone-0123abcd']);
|
||||
await expect(dialog(page)).toContainText('If a source comes back, its thumbnails are made again as you browse.');
|
||||
await scope.selectOption('missing');
|
||||
await expect(dialog(page).locator('dl.meta-grid')).toBeHidden();
|
||||
await scope.selectOption('prune');
|
||||
await dialog(page).getByRole('button', { name: 'Start' }).click();
|
||||
await expect(page.locator('#workspace-main h1')).toHaveText('Remove thumbnails of removed sources');
|
||||
await expect(page.locator('#workspace-main')).toContainText('1 thumbnails removed');
|
||||
});
|
||||
|
||||
test('a disk without the space shows the review but cannot move', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/thumbnail-space/full')).toBeOK();
|
||||
await openSources(page);
|
||||
const row = block(page);
|
||||
await row.getByRole('radio', { name: /One folder/ }).check();
|
||||
const planned = page.waitForResponse(response => response.url().endsWith('/images/thumbnails/plan'));
|
||||
await row.getByRole('button', { name: 'Review move…' }).click();
|
||||
expect((await planned).status()).toBe(507);
|
||||
const review = dialog(page);
|
||||
await expect(review.locator('.review-refusal')).toHaveText(/^There is not enough space for the thumbnails: 1 B free/);
|
||||
await expect(review.getByRole('button', { name: 'Move thumbnails' })).toHaveAttribute('aria-disabled', 'true');
|
||||
await review.getByRole('button', { name: 'Cancel' }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
});
|
||||
@@ -135,34 +135,11 @@ test('/prompt_manager/web lands on the workspace Library and Back does not bounc
|
||||
await expect(main.getByRole('heading', { level: 1, name: 'Overview', exact: true })).toBeVisible();
|
||||
});
|
||||
|
||||
// D165 (#525, maintainer ruling): a discreet line under System is the workspace's only visible way to the legacy pages.
|
||||
test('System Overview links the legacy pages discreetly, and each one loads (#525)', async ({ page, context }) => {
|
||||
// #620: the old pages are on their way out (notice since 3.5.1), so the workspace no longer links them.
|
||||
test('System no longer links the old pages (#620)', async ({ page }) => {
|
||||
await page.goto(`${workspace}#/system/overview`);
|
||||
const line = page.locator('#workspace-main .legacy-pages');
|
||||
await expect(line).toBeVisible();
|
||||
await expect(line).toContainText('Legacy pages');
|
||||
const links = line.getByRole('link');
|
||||
await expect(links).toHaveCount(4);
|
||||
for (const [name, url, ready] of [
|
||||
['Dashboard', '/prompt_manager/admin', '#addPromptBtn'],
|
||||
['Tags', '/prompt_manager/admin#/tags', '#addPromptBtn'],
|
||||
['Gallery', '/prompt_manager/gallery', 'body'],
|
||||
['Metadata viewer', '/prompt_manager/gallery.html', 'body'],
|
||||
]) {
|
||||
const opened = context.waitForEvent('page');
|
||||
await line.getByRole('link', { name, exact: true }).click();
|
||||
const tab = await opened;
|
||||
await tab.waitForLoadState('domcontentloaded');
|
||||
const target = new URL(tab.url());
|
||||
expect(target.pathname + target.hash).toBe(url);
|
||||
await expect(tab.locator(ready).first()).toBeAttached();
|
||||
await tab.close();
|
||||
}
|
||||
// Only on System's Overview section, and gone with the page.
|
||||
await page.locator('#workspace-main .section-nav').getByRole('button', { name: 'Appearance', exact: true }).click();
|
||||
await expect(line).toBeHidden();
|
||||
await page.goto(`${workspace}#/library`);
|
||||
await expect(page.locator('#workspace-main .legacy-pages')).toHaveCount(0);
|
||||
await expect(page.locator('#workspace-main h1')).toHaveText('System');
|
||||
await expect(page.locator('#workspace-main a[href*="/admin"], #workspace-main a[href*="/gallery"]')).toHaveCount(0);
|
||||
});
|
||||
|
||||
const bootFailure = page => page.locator('#workspace-boot-failure');
|
||||
@@ -175,7 +152,7 @@ test('a healthy workspace never shows the load failure message (#525)', async ({
|
||||
});
|
||||
|
||||
for (const [label, pattern] of [['the entry module', '**/js/workspace/index.mjs*'], ['a dependency', '**/js/workspace/state.mjs*']]) {
|
||||
test(`when ${label} fails to load the page still offers the legacy pages (#525)`, async ({ page }) => {
|
||||
test(`when ${label} fails to load the page offers a reload and ComfyUI (#525, #620)`, async ({ page }) => {
|
||||
const pageErrors = [];
|
||||
page.on('pageerror', error => pageErrors.push(error.message));
|
||||
await page.route(pattern, route => route.abort());
|
||||
@@ -183,19 +160,17 @@ for (const [label, pattern] of [['the entry module', '**/js/workspace/index.mjs*
|
||||
const failure = bootFailure(page);
|
||||
await expect(failure).toBeVisible();
|
||||
await expect(failure).toHaveAttribute('role', 'alert');
|
||||
const dashboard = failure.getByRole('link', { name: 'Dashboard', exact: true });
|
||||
await expect(dashboard).toBeVisible();
|
||||
await expect(failure.getByRole('link')).toHaveCount(4);
|
||||
await dashboard.click();
|
||||
await expect(page).toHaveURL(/\/prompt_manager\/admin$/);
|
||||
await expect(failure.getByRole('link', { name: 'Reload the page', exact: true })).toHaveAttribute('href', 'workspace');
|
||||
await expect(failure.getByRole('link', { name: 'ComfyUI', exact: true })).toHaveAttribute('href', '../');
|
||||
await expect(failure.getByRole('link')).toHaveCount(2);
|
||||
expect(pageErrors).toEqual([]);
|
||||
});
|
||||
}
|
||||
|
||||
test('when a module throws while starting the page still offers the legacy pages (#525)', async ({ page }) => {
|
||||
test('when a module throws while starting the page still offers a reload (#525, #620)', async ({ page }) => {
|
||||
await page.route('**/js/workspace/state.mjs*', route => route.fulfill({ contentType: 'text/javascript', body: "throw new Error('boot failure');" }));
|
||||
await page.goto(`${workspace}#/library`);
|
||||
await expect(bootFailure(page).getByRole('link', { name: 'Dashboard', exact: true })).toBeVisible();
|
||||
await expect(bootFailure(page).getByRole('link', { name: 'Reload the page', exact: true })).toBeVisible();
|
||||
// The broken module's own error is this test's premise, not a failure of the page.
|
||||
page.workspaceErrors.length = 0;
|
||||
});
|
||||
@@ -569,7 +544,7 @@ test('System is native: the concept shell, section navigation, sections and the
|
||||
// D92: no "Sample configuration" chip.
|
||||
await expect(main.locator('.heading .chip')).toHaveCount(0);
|
||||
expect(await main.evaluate(node => [...node.children].map(child => `${child.localName}.${child.className}`)))
|
||||
.toEqual(['div.heading', 'div.section-nav', 'div.settings-layout', 'p.small legacy-pages']);
|
||||
.toEqual(['div.heading', 'div.section-nav', 'div.settings-layout']);
|
||||
const nav = main.getByRole('group', { name: 'System sections', exact: true });
|
||||
await expect(nav.getByRole('button')).toHaveText(SYSTEM_SECTION_LABELS);
|
||||
await expect(nav.getByRole('button', { name: 'Overview', exact: true })).toHaveAttribute('aria-pressed', 'true');
|
||||
|
||||
+11
-1
@@ -126,6 +126,9 @@
|
||||
"POST /prompt_manager/images/removed/return",
|
||||
"POST /prompt_manager/images/rescope",
|
||||
"POST /prompt_manager/images/reveal",
|
||||
"POST /prompt_manager/images/thumbnails/location/adopt",
|
||||
"POST /prompt_manager/images/thumbnails/location/recreate",
|
||||
"POST /prompt_manager/images/thumbnails/plan",
|
||||
"POST /prompt_manager/jobs/autotag",
|
||||
"POST /prompt_manager/jobs/duplicates",
|
||||
"POST /prompt_manager/jobs/image-tags",
|
||||
@@ -217,6 +220,13 @@
|
||||
"thumbnail_encoders",
|
||||
"thumbnail_encoders.jpeg",
|
||||
"thumbnail_encoders.jxl",
|
||||
"thumbnail_encoders.webp"
|
||||
"thumbnail_encoders.webp",
|
||||
"thumbnail_location",
|
||||
"thumbnail_location.available",
|
||||
"thumbnail_location.cleanup_pending",
|
||||
"thumbnail_location.default_path",
|
||||
"thumbnail_location.mode",
|
||||
"thumbnail_location.path",
|
||||
"thumbnail_location.problem"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -371,6 +371,16 @@ test("ApiError keeps the job that applied a used plan (plan-used, System S7)", (
|
||||
}
|
||||
});
|
||||
|
||||
test("ApiError keeps a move refused for space with its review (#320)", () => {
|
||||
const plan = { totals: { files: 3 }, insufficient: { text: "Not enough space" } };
|
||||
assert.equal(new ApiError(507, "no space", "insufficient-space", null, { plan }).plan, plan);
|
||||
for (const bad of [null, "x", [1], undefined]) {
|
||||
assert.equal(new ApiError(507, "no space", "insufficient-space", null, { plan: bad }).plan, null);
|
||||
}
|
||||
assert.equal(new ApiError(409, "busy", "busy", null, { plan }).plan, null);
|
||||
assert.equal(new ApiError(507, "no space", "insufficient-space").plan, null);
|
||||
});
|
||||
|
||||
test("ApiError keeps a validated running job on busy and a validated source on invalid-source", () => {
|
||||
const body = { running: { id: "a".repeat(32), title: "Scan for metadata" }, source: "output/gone" };
|
||||
const busy = new ApiError(409, "busy", "busy", null, body);
|
||||
|
||||
@@ -255,7 +255,8 @@ test('All is offered only with two or more eligible nodes and is never selected
|
||||
test('Apply to All previews every node first, then sends one after another in node id order',async()=>{
|
||||
const {createSendSession,ALL_NODES}=await source;const s=await allSession(createSendSession);
|
||||
s.session.selectTarget(ALL_NODES);await s.session.apply();
|
||||
assert.deepEqual(order(s.calls),['preview 2','preview 9','preview 10','apply 2','preview 9','apply 9','preview 10','apply 10']);
|
||||
// Each node takes a fresh ticket, the first too: the host keeps only its latest 100 (#593).
|
||||
assert.deepEqual(order(s.calls),['preview 2','preview 9','preview 10','preview 2','apply 2','preview 9','apply 9','preview 10','apply 10']);
|
||||
assert.equal(s.session.state.phase,'success');assert.equal(s.session.state.message,'Sent to 3 of 3 nodes in Flow - ComfyUI.');
|
||||
await s.session.apply();assert.equal(s.calls.filter(call=>call[0]==='apply').length,3);s.session.dispose();
|
||||
});
|
||||
@@ -363,3 +364,48 @@ test('a changed prepend or append after the review refuses that node',async()=>{
|
||||
assert.deepEqual(s.calls.filter(call=>call[0]==='apply').map(call=>call[1]),['2','10']);
|
||||
assert.match(s.session.state.message,/^Sent to 2 of 3 nodes in Flow - ComfyUI\. #9 Node 9: Its prompt changed since the preview\./);s.session.dispose();
|
||||
});
|
||||
|
||||
// #593: Send to All through the real Toolkit bridge and the real adapter, as the review bot reproduced it.
|
||||
function realWindow(count) {
|
||||
const listeners=new Set();
|
||||
const channelFactory=()=>{const own=new Set();return{
|
||||
addEventListener(type,listener){if(type==='message'){own.add(listener);listeners.add(listener);}},removeEventListener(type,listener){listeners.delete(listener);},
|
||||
postMessage(data){for(const listener of [...listeners])if(!own.has(listener))queueMicrotask(()=>listener({data:structuredClone(data),origin:'https://comfy.example'}));},close(){},
|
||||
};};
|
||||
// Like LiteGraph, a widget write moves the version of the graph it is in.
|
||||
const graph=()=>{const made={_version:1,setDirtyCanvas(){made._version++;}};made._nodes=Array.from({length:count},(_,index)=>{
|
||||
const widgets=['text','prepend_text','append_text'].map(name=>({name,value:name==='text'?'old':'',inputEl:{value:name==='text'?'old':''}}));
|
||||
return {id:index+1,type:'PromptManager',title:`Node ${index+1}`,widgets,widgets_values:widgets.map(widget=>widget.value),inputs:[],setDirtyCanvas(){}};
|
||||
});return made;};
|
||||
return {channelFactory,graph};
|
||||
}
|
||||
async function realSend(count,{afterFirstApply}={}) {
|
||||
const {createSendSession,ALL_NODES}=await source;
|
||||
const {createBridgeHost,createBridgeClient}=await import('../../web/js/shared/toolkit/prompt-bridge.js');
|
||||
const {createPromptAdapter}=await import('../../web/js/prompt-bridge-adapter.js');
|
||||
const win=realWindow(count);const app={graph:win.graph(),canvas:{}};
|
||||
const adapter=createPromptAdapter(app,{runtime:{}});let applied=0;
|
||||
const watched={...adapter,apply(snapshot){const result=adapter.apply(snapshot);if(++applied===1)afterFirstApply?.(app,win);return result;}};
|
||||
const options={namespace:'/prompt_manager/',origin:'https://comfy.example',channelFactory:win.channelFactory};
|
||||
const host=createBridgeHost({...options,label:'Flow - ComfyUI',adapter:watched});
|
||||
const client=createBridgeClient(options);
|
||||
const session=createSendSession({client,roles:{positive:'new'},onChange:()=>{}});
|
||||
try {
|
||||
await session.discover();session.selectTarget(ALL_NODES);await session.apply();
|
||||
return {state:session.state,app};
|
||||
} finally { session.dispose();host.dispose(); }
|
||||
}
|
||||
test('All refuses the later nodes when another workflow is loaded after the first send (#593)',async()=>{
|
||||
let first;
|
||||
const {state,app}=await realSend(3,{afterFirstApply:(target,win)=>{first=target.graph;target.graph=win.graph();target.graph._version=first._version;}});
|
||||
// The new workflow has the same ids, text and composition, and even the same graph version: only node identity differs.
|
||||
assert.equal(first._nodes[0].widgets[0].value,'new');
|
||||
assert.deepEqual(app.graph._nodes.map(node=>node.widgets[0].value),['old','old','old']);
|
||||
assert.equal(state.phase,'error');
|
||||
assert.match(state.message,/^Sent to 1 of 3 nodes in Flow - ComfyUI\. #2 Node 2: The workflow changed since this window was found\./);
|
||||
});
|
||||
test('All sends to every node of a window with more than 100 of them (#593)',async()=>{
|
||||
const {state,app}=await realSend(101);
|
||||
assert.equal(state.message,'Sent to 101 of 101 nodes in Flow - ComfyUI.');
|
||||
assert.ok(app.graph._nodes.every(node=>node.widgets[0].value==='new'));
|
||||
});
|
||||
|
||||
@@ -143,7 +143,7 @@ test('the shelf no longer sorts by rating (D153)',async()=>{
|
||||
const {createLibraryStream}=await source;const s=createLibraryStream({api:{get:async()=>page([])}});
|
||||
await assert.rejects(s.reset({sort:'rating'}),/Invalid library sort/);
|
||||
assert.throws(()=>createLibraryStream({api:{get:async()=>page([])},restore:{filters:{sort:'rating'},state:{}}}),/Invalid library sort/);
|
||||
for(const sort of ['newest','oldest','name']) assert.equal(await s.reset({sort}),0);s.dispose();
|
||||
for(const sort of ['newest','oldest','name','name-desc','most-used','last-used']) assert.equal(await s.reset({sort}),0);s.dispose();
|
||||
});
|
||||
test('Refine › Folder nests each folder under its exact parent, by path segment (#444 live B1)',async()=>{
|
||||
const {libraryFolderOptions}=await source;assert.equal(typeof libraryFolderOptions,'function');
|
||||
|
||||
@@ -32,7 +32,7 @@ test('a saved state always holds every page key and keeps only values its page a
|
||||
{ ...full, q: '', folder: '', tags: [] });
|
||||
assert.deepEqual(normalizeViewState('images', { ...full, category: 'bad\u0007', tags: [' '] }), { ...full, category: '', tags: [] });
|
||||
// Library has no source or media, and no Name Z–A; its folder (#444) is the Library's own.
|
||||
assert.deepEqual(normalizeViewState('library', { ...full, sort: 'name-desc' }), { q: 'castle', category: 'Portraits', sort: 'newest', tags: ['moody', 'fog'], tag_mode: 'all', folder: 'a/b' });
|
||||
assert.deepEqual(normalizeViewState('library', { ...full, sort: 'rating' }), { q: 'castle', category: 'Portraits', sort: 'newest', tags: ['moody', 'fog'], tag_mode: 'all', folder: 'a/b' });
|
||||
assert.deepEqual(normalizeViewState('library', { sort: 'oldest' }), { ...LIBRARY_DEFAULTS, sort: 'oldest' });
|
||||
// The returned tags are a copy.
|
||||
const tags = ['one'];
|
||||
@@ -244,5 +244,7 @@ test('a view saved under the retired Highest rated sort opens on Recently added
|
||||
assert.equal(imagesFilters({ sort: 'rating' }).sort, 'newest');
|
||||
assert.equal(libraryFilters({ sort: 'rating' }).sort, 'newest');
|
||||
assert.equal(normalizeViewState('images', { sort: 'name-desc' }).sort, 'name-desc');
|
||||
assert.equal(normalizeViewState('library', { sort: 'oldest' }).sort, 'oldest');
|
||||
for (const sort of ['oldest', 'name-desc', 'most-used', 'last-used']) {
|
||||
assert.equal(normalizeViewState('library', { sort }).sort, sort);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -93,13 +93,6 @@ test('every Overview card opens its own native page, never a legacy one (#525)',
|
||||
}
|
||||
});
|
||||
|
||||
test('the legacy pages System links are the four that stay at their URLs (#525 D165)', async () => {
|
||||
const { LEGACY_PAGES } = await source;
|
||||
assert.deepEqual(LEGACY_PAGES.map(page => page.href), ['/prompt_manager/admin', '/prompt_manager/admin#/tags',
|
||||
'/prompt_manager/gallery', '/prompt_manager/gallery.html']);
|
||||
for (const page of LEGACY_PAGES) assert.ok(page.label);
|
||||
});
|
||||
|
||||
test('a job of the Activity page is its own deep link', async () => {
|
||||
const { resolveRoute, routeTitle, isHandoff } = await source;
|
||||
const id = '0123456789abcdef'.repeat(2);
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
// Thumbnail location (#320, D196–D201): the row's state and status line, the move review's rows, the unavailable
|
||||
// notice's text, the Activity views of a move and a prune, the prune scope's plan line and the Move library row.
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
|
||||
const locationModule = import('../../web/js/workspace/system-thumbnail-location.mjs');
|
||||
const kindsModule = import('../../web/js/workspace/activity-kinds.mjs');
|
||||
const reviewModule = import('../../web/js/workspace/thumbnail-review.mjs');
|
||||
const libraryModule = import('../../web/js/workspace/library-location.mjs');
|
||||
|
||||
const SOURCE = Object.freeze({ mode: 'source', path: null, default_path: '/data/thumbnails', available: true,
|
||||
problem: null, cleanup_pending: 0 });
|
||||
const FOLDER = Object.freeze({ mode: 'folder', path: 'E:\\Thumbnails', default_path: '/data/thumbnails',
|
||||
available: true, problem: null, cleanup_pending: 0 });
|
||||
|
||||
const PLAN = Object.freeze({
|
||||
from: { mode: 'source', path: null, available: true },
|
||||
to: { mode: 'folder', path: 'E:\\Thumbnails', exists: true, will_create: false, default: false },
|
||||
totals: { sources: 3, files: 8512, bytes: 1.2 * 1024 ** 3, orphans: 41, temps: 2, links: 1, left_in_place: 1,
|
||||
too_long: 32, unlisted_sources: 0, unlisted_files: 0 },
|
||||
too_long_listed: ['Archive 2025 · a_thumb.webp', 'Archive 2025 · b_thumb.webp'],
|
||||
left_listed: ['Archive 2025 · notes.txt'],
|
||||
unlisted: [],
|
||||
free_bytes: 1.6 * 1024 ** 4,
|
||||
confirm_required: true,
|
||||
token: 't',
|
||||
});
|
||||
|
||||
test('the status line says where previews are now', async () => {
|
||||
const { statusText } = await locationModule;
|
||||
assert.equal(statusText(SOURCE), 'Now: inside each source.');
|
||||
assert.equal(statusText(FOLDER), 'Now: one folder, E:\\Thumbnails.');
|
||||
assert.equal(statusText({ ...FOLDER, path: '/data/thumbnails' }),
|
||||
'Now: one folder, /data/thumbnails. Using the default folder next to your library.');
|
||||
assert.equal(statusText({ ...FOLDER, available: false, problem: 'missing' }),
|
||||
'Now: one folder, E:\\Thumbnails · unavailable. Originals are shown.');
|
||||
assert.match(statusText({ ...SOURCE, cleanup_pending: 1 }), /Old thumbnails in 1 place wait to be removed/);
|
||||
assert.equal(statusText(undefined), '');
|
||||
assert.match(statusText(SOURCE, { mode: 'folder', path: '/data/thumbnails' }),
|
||||
/^Now: inside each source\. The suggested folder is next to your library/);
|
||||
assert.equal(statusText(SOURCE, { mode: 'folder', path: '/other' }), 'Now: inside each source.');
|
||||
});
|
||||
|
||||
test('Review move… is offered only for another location', async () => {
|
||||
const { choiceChanged } = await locationModule;
|
||||
assert.equal(choiceChanged(SOURCE, { mode: 'source' }), false);
|
||||
assert.equal(choiceChanged(SOURCE, { mode: 'folder', path: ' ' }), false);
|
||||
assert.equal(choiceChanged(SOURCE, { mode: 'folder', path: '/data/thumbnails' }), true);
|
||||
assert.equal(choiceChanged(FOLDER, { mode: 'folder', path: 'E:\\Thumbnails' }), false);
|
||||
assert.equal(choiceChanged(FOLDER, { mode: 'folder', path: 'F:\\Thumbs' }), true);
|
||||
assert.equal(choiceChanged(FOLDER, { mode: 'source' }), true);
|
||||
assert.equal(choiceChanged({ ...SOURCE, problem: 'invalid' }, { mode: 'source' }), true);
|
||||
assert.equal(choiceChanged(undefined, { mode: 'source' }), false);
|
||||
});
|
||||
|
||||
test('the review lists what moves, what is removed and what stays', async () => {
|
||||
const { reviewRows } = await locationModule;
|
||||
const rows = Object.fromEntries(reviewRows(PLAN));
|
||||
assert.equal(rows.From, 'Inside each source (3 sources)');
|
||||
assert.equal(rows.To, 'E:\\Thumbnails (empty folder)');
|
||||
assert.equal(rows.Thumbnails, '8,512 in 3 sources · 1.2 GB');
|
||||
assert.equal(rows['Free space'], '1638.4 GB free');
|
||||
assert.match(rows['Too long for Windows'][0], /^32 thumbnails cannot be stored there/);
|
||||
assert.deepEqual(rows['Too long for Windows'][1], [...PLAN.too_long_listed, 'and 30 more']);
|
||||
assert.deepEqual(rows['Removed after the move'][1],
|
||||
['41 thumbnails whose original is gone', '2 unfinished temporary files']);
|
||||
assert.equal(rows['Left in place'][0], '1 file that is not a thumbnail:');
|
||||
assert.equal(rows['Links skipped'], '1 link inside a thumbnails folder is not followed');
|
||||
});
|
||||
|
||||
test('the review of a move back names the folder and the removed sources', async () => {
|
||||
const { reviewRows } = await locationModule;
|
||||
const back = reviewRows({ ...PLAN, from: { mode: 'folder', path: '/data/thumbnails' },
|
||||
to: { mode: 'source', path: null }, unlisted: [{ key: 'old-1c22e0a4', files: 240 }],
|
||||
totals: { sources: 1, files: 1, bytes: 10 }, free_bytes: undefined });
|
||||
const rows = Object.fromEntries(back);
|
||||
assert.equal(rows.From, '/data/thumbnails');
|
||||
assert.equal(rows.To, 'Inside each source (1 source)');
|
||||
assert.equal(rows['Free space'], undefined);
|
||||
assert.match(rows['Removed after the move'][0], /The folder itself is removed once empty/);
|
||||
assert.deepEqual(rows['Removed after the move'][1],
|
||||
['1 source no longer listed: 240 thumbnails are removed; they are made again if it returns']);
|
||||
// Nothing else removed: one full stop, never "created it.." (live test of #592).
|
||||
const plain = Object.fromEntries(reviewRows({ ...PLAN, from: { mode: 'folder', path: '/data/thumbnails' },
|
||||
to: { mode: 'source', path: null }, unlisted: [], totals: { sources: 1, files: 1, bytes: 10 } }));
|
||||
assert.match(plain['Removed after the move'][0], /Prompt Manager created it\.$/);
|
||||
const fresh = Object.fromEntries(reviewRows({ ...PLAN, to: { mode: 'folder', path: '/d', default: true, will_create: true } }));
|
||||
assert.equal(fresh.To, '/d (new, next to your library)');
|
||||
});
|
||||
|
||||
test('the notice says why the folder is unavailable', async () => {
|
||||
const { noticeText } = await locationModule;
|
||||
assert.match(noticeText({ ...FOLDER, problem: 'missing' }), /^E:\\Thumbnails is missing\. Originals are shown/);
|
||||
assert.match(noticeText({ ...FOLDER, problem: 'unreachable' }), /can’t be reached/);
|
||||
assert.match(noticeText({ ...FOLDER, problem: 'marker-mismatch' }), /holds another thumbnail folder/);
|
||||
assert.match(noticeText({ ...FOLDER, problem: 'invalid' }), /config\.json can’t be used/);
|
||||
assert.match(noticeText({ ...FOLDER, problem: null }), /is being checked/);
|
||||
});
|
||||
|
||||
test('sizes are written as the server writes them', async () => {
|
||||
const { sizeText } = await locationModule;
|
||||
assert.equal(sizeText(512), '512 B');
|
||||
assert.equal(sizeText(85.2 * 1024 * 1024), '85.2 MB');
|
||||
});
|
||||
|
||||
test('a move and a prune show their own tiles in Activity', async () => {
|
||||
const { viewOf, movePhase, finishToast } = await kindsModule;
|
||||
const move = { kind: 'thumbnails', state: 'running', cancellable: true, scope: { mode: 'move', from: 'source' },
|
||||
counters: { moved: 3, skipped: 0, left: 1, errors: 2 }, status_line: '' };
|
||||
assert.deepEqual(viewOf(move).tiles.map(([group, label]) => label), ['Moved', 'Already there', 'Left in place', 'Errors']);
|
||||
assert.equal(viewOf(move).cancelLabel, 'Cancel move');
|
||||
assert.match(movePhase(move), /^Step 1 of 3: copying\. Thumbnails are still read from inside each source/);
|
||||
assert.match(movePhase({ ...move, cancellable: false }), /can no longer be cancelled/);
|
||||
assert.match(movePhase({ ...move, scope: { mode: 'move', phase: 'remove' } }), /Removing the old copies/);
|
||||
assert.match(movePhase({ ...move, state: 'finished' }), /1 file that is not a thumbnail was left in place/);
|
||||
assert.match(movePhase({ ...move, state: 'cancelled' }), /^Nothing changed/);
|
||||
assert.match(movePhase({ ...move, state: 'finished', counters: { moved: 3, left: 2 } }),
|
||||
/2 files that are not thumbnails were left in place\.$/);
|
||||
assert.equal(movePhase({ ...move, state: 'finished', counters: undefined }), 'Thumbnails are now read from the new location.');
|
||||
assert.match(movePhase({ ...move, scope: { mode: 'move', from: 'folder' } }), /read from the old folder until the switch/);
|
||||
const done = { ...move, state: 'finished', status_line: 'Thumbnails moved to E:\\T: 3 moved · old copies removed.' };
|
||||
assert.equal(finishToast(done), done.status_line);
|
||||
const prune = { kind: 'thumbnails', state: 'running', cancellable: true, scope: { mode: 'prune' },
|
||||
counters: { removed: 1, left: 0, errors: 0 } };
|
||||
assert.deepEqual(viewOf(prune).tiles.map(([, label]) => label), ['Removed', 'Left in place', 'Errors']);
|
||||
assert.equal(viewOf(move).line(move), '');
|
||||
assert.equal(viewOf(done).line(done), done.status_line);
|
||||
assert.equal(viewOf(move).kept(move), '3 moved');
|
||||
assert.equal(finishToast({ ...move, state: 'cancelled', status_line: '' }), viewOf(move).finishToast({ ...move,
|
||||
state: 'cancelled', status_line: '' }));
|
||||
const pruned = { ...prune, state: 'finished', status_line: '1 thumbnails removed' };
|
||||
assert.equal(viewOf(prune).line(prune), '');
|
||||
assert.equal(viewOf(pruned).line(pruned), '1 thumbnails removed');
|
||||
assert.equal(viewOf(pruned).line({ ...pruned, status_line: null }), '');
|
||||
assert.equal(viewOf(done).line({ ...done, status_line: null }), '');
|
||||
assert.equal(finishToast(pruned), '1 thumbnails removed');
|
||||
assert.equal(viewOf(prune).kept(prune), '1 removed');
|
||||
assert.match(finishToast({ ...prune, state: 'finished', status_line: '', title: 'Remove thumbnails of removed sources' }),
|
||||
/Remove thumbnails of removed sources/);
|
||||
const generate = { kind: 'thumbnails', state: 'running', cancellable: true, scope: { mode: 'missing' },
|
||||
counters: { generated: 0, skipped: 0, errors: 0 } };
|
||||
assert.deepEqual(viewOf(generate).tiles.map(([, label]) => label), ['Generated', 'Skipped', 'Errors']);
|
||||
});
|
||||
|
||||
test('the prune scope reads its plan line from the server', async () => {
|
||||
const { planText, PLAN_FAILED, PRUNE_OPTION } = await reviewModule;
|
||||
assert.equal(PRUNE_OPTION[1], 'Remove thumbnails of removed sources');
|
||||
assert.equal(planText({ mode: 'prune', line: '2 sources no longer listed: 1,240 thumbnails (85.2 MB) are removed.' }),
|
||||
'2 sources no longer listed: 1,240 thumbnails (85.2 MB) are removed.');
|
||||
assert.equal(planText({ mode: 'prune' }), PLAN_FAILED);
|
||||
});
|
||||
|
||||
test('the prune review names the folder and each removed source', async () => {
|
||||
const { pruneText, pruneEntries } = await reviewModule;
|
||||
assert.match(pruneText({ folder: 'E:\\Thumbs' }), /^Only Prompt Manager’s thumbnails in their folders inside E:\\Thumbs are /);
|
||||
assert.match(pruneText({}), /inside the thumbnail folder are removed\. If a source comes back/);
|
||||
assert.deepEqual(pruneEntries({ sources: [{ key: 'gone-0123abcd', files: 1240, bytes: 85.2 * 1024 * 1024 },
|
||||
{ key: 'one-0123abcd', files: 1, bytes: 512 }] }),
|
||||
[['gone-0123abcd', '1,240 thumbnails · 85.2 MB'], ['one-0123abcd', '1 thumbnail · 512 B']]);
|
||||
assert.deepEqual(pruneEntries(null), []);
|
||||
assert.deepEqual(pruneEntries({ sources: 'x' }), []);
|
||||
});
|
||||
|
||||
test('Move library says where thumbnails kept in one folder stay', async () => {
|
||||
const { reviewGrid } = await libraryModule;
|
||||
const base = { path: 'p', target: 't', size_bytes: 0, config_moves: true };
|
||||
assert.equal(reviewGrid(base).some(([term]) => term === 'Thumbnails'), false);
|
||||
const rows = Object.fromEntries(reviewGrid({ ...base, thumbnails_folder: '/data/thumbnails' }));
|
||||
assert.equal(rows.Thumbnails, 'Stay in /data/thumbnails. Move them in System › Sources & scans.');
|
||||
});
|
||||
@@ -732,6 +732,32 @@ class TestAutotagErrorBodiesHideServerPaths(AutotagAPITestCase):
|
||||
|
||||
class TestUnloadAutotag(AutotagAPITestCase):
|
||||
|
||||
async def test_unload_runs_off_the_loop_and_holds_the_gate_until_done(self):
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
self.addCleanup(release.set)
|
||||
threads = []
|
||||
|
||||
def unload():
|
||||
threads.append(threading.get_ident())
|
||||
started.set()
|
||||
release.wait(5)
|
||||
self.service._loaded_type = None
|
||||
|
||||
self.service._loaded_type = "gguf"
|
||||
self.service.unload_model = unload
|
||||
first = asyncio.ensure_future(
|
||||
self.client.request("POST", "/prompt_manager/autotag/unload")
|
||||
)
|
||||
self.assertTrue(await asyncio.to_thread(started.wait, 5))
|
||||
self.assertNotEqual(threads, [threading.get_ident()])
|
||||
second = await self.client.request("POST", "/prompt_manager/autotag/unload")
|
||||
self.assertEqual(second.status, 409)
|
||||
release.set()
|
||||
data = await (await first).json()
|
||||
self.assertEqual(data["message"], "GGUF model unloaded successfully")
|
||||
self.assertIsNone(model_gate.holder())
|
||||
|
||||
async def test_nothing_loaded(self):
|
||||
resp = await self.client.request("POST", "/prompt_manager/autotag/unload")
|
||||
data = await resp.json()
|
||||
|
||||
+3
-54
@@ -343,7 +343,7 @@ class TestReadJsonBody(AioHTTPTestCase):
|
||||
|
||||
|
||||
class TestLongJobLock(unittest.TestCase):
|
||||
"""_acquire_long_job: one job per family at a time, released on exit."""
|
||||
"""The refusal a busy long job answers with."""
|
||||
|
||||
def setUp(self):
|
||||
database = MagicMock()
|
||||
@@ -353,33 +353,6 @@ class TestLongJobLock(unittest.TestCase):
|
||||
def _run(self, coro):
|
||||
return asyncio.run(coro)
|
||||
|
||||
def test_second_acquire_is_refused_until_first_releases(self):
|
||||
async def scenario():
|
||||
first = self.api._acquire_long_job("duplicates")
|
||||
self.assertIsNotNone(first)
|
||||
async with first:
|
||||
self.assertIsNone(self.api._acquire_long_job("duplicates"))
|
||||
# Other families are independent
|
||||
other = self.api._acquire_long_job("thumbnails")
|
||||
self.assertIsNotNone(other)
|
||||
async with other:
|
||||
pass
|
||||
again = self.api._acquire_long_job("duplicates")
|
||||
self.assertIsNotNone(again)
|
||||
async with again:
|
||||
pass
|
||||
|
||||
self._run(scenario())
|
||||
|
||||
def test_lock_is_released_when_the_job_raises(self):
|
||||
async def scenario():
|
||||
with self.assertRaises(RuntimeError):
|
||||
async with self.api._acquire_long_job("duplicates"):
|
||||
raise RuntimeError("boom")
|
||||
self.assertIsNotNone(self.api._acquire_long_job("duplicates"))
|
||||
|
||||
self._run(scenario())
|
||||
|
||||
def test_busy_response_is_409_with_envelope(self):
|
||||
resp = self.api._long_job_busy_response("Duplicate scan")
|
||||
self.assertEqual(resp.status, 409)
|
||||
@@ -796,6 +769,7 @@ class TestRevalidatedCachingBehindMiddlewares(CoreRoutesTestCase):
|
||||
self.assertEqual(resp.status, 200, path)
|
||||
self.assertEqual(resp.headers.get("Content-Encoding"), "gzip", path)
|
||||
self.assertEqual(resp.headers["Cache-Control"], "no-cache", path)
|
||||
await resp.read() # aiohttp before 3.11.11 hangs on an unread body
|
||||
again = await self.client.request(
|
||||
"GET",
|
||||
path,
|
||||
@@ -806,6 +780,7 @@ class TestRevalidatedCachingBehindMiddlewares(CoreRoutesTestCase):
|
||||
)
|
||||
self.assertEqual(again.status, 304, path)
|
||||
self.assertEqual(again.headers["Cache-Control"], "no-cache", path)
|
||||
await again.read()
|
||||
|
||||
|
||||
# ── instance helpers ──────────────────────────────────────────────────
|
||||
@@ -951,32 +926,6 @@ class TestMetadataParsing(unittest.TestCase):
|
||||
f.write(b"nope")
|
||||
self.assertEqual(self.api._extract_comfyui_metadata(garbage), {})
|
||||
|
||||
def test_parse_a1111_parameters(self):
|
||||
meta = {"parameters": "a cat\nNegative prompt: dog\nSteps: 20"}
|
||||
parsed = self.api._parse_comfyui_prompt(meta)
|
||||
self.assertEqual(parsed["positive_prompt"], "a cat")
|
||||
self.assertEqual(parsed["negative_prompt"], "dog")
|
||||
self.assertEqual(parsed["parameters"]["parameters"], meta["parameters"])
|
||||
|
||||
def test_parse_comfyui_fields(self):
|
||||
meta = {
|
||||
"prompt": json.dumps({"1": {"class_type": "X", "inputs": {}}}),
|
||||
"workflow": "not json",
|
||||
"steps": "20",
|
||||
"cfg": "not-json-either",
|
||||
}
|
||||
parsed = self.api._parse_comfyui_prompt(meta)
|
||||
self.assertIsInstance(parsed["prompt"], dict)
|
||||
self.assertEqual(parsed["workflow"], "not json")
|
||||
self.assertEqual(parsed["parameters"]["steps"], 20)
|
||||
self.assertEqual(parsed["parameters"]["cfg"], "not-json-either")
|
||||
|
||||
parsed = self.api._parse_comfyui_prompt(
|
||||
{"prompt": "plain text", "workflow": json.dumps({"nodes": []})}
|
||||
)
|
||||
self.assertEqual(parsed["prompt"], "plain text")
|
||||
self.assertEqual(parsed["workflow"], {"nodes": []})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -452,6 +452,40 @@ class TestReview(DuplicateFilesTestCase):
|
||||
)
|
||||
|
||||
|
||||
class TestReviewProtectedRows(DuplicateFilesTestCase):
|
||||
async def test_cleanup_lists_each_copy_the_review_kept(self):
|
||||
if not TestReview._can_symlink(self.tmpdir): # pragma: no cover
|
||||
self.skipTest("symlinks are not available")
|
||||
self.trio()
|
||||
job, group = await self.found_group()
|
||||
# b.png becomes a link after the find: the review protects it (#586).
|
||||
(self.output_dir / "b.png").unlink()
|
||||
os.symlink(str(self.output_dir / "a.png"), str(self.output_dir / "b.png"))
|
||||
plan = await self.review(job.id, [self.keep_of(group, "a.png")])
|
||||
self.assertEqual(plan["protected"]["count"], 1)
|
||||
body = await self.apply(plan["token"])
|
||||
cleanup = self.api._jobs.get(body["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS)
|
||||
self.assertEqual(cleanup.state, "finished", cleanup.log)
|
||||
rows = {row["path"]: row for row in cleanup.outcomes["kept"]}
|
||||
self.assertEqual(
|
||||
rows["b.png"],
|
||||
{
|
||||
"path": "b.png",
|
||||
"root": rows["a.png"]["root"],
|
||||
"root_index": 0,
|
||||
"detail": "Kept: a link, not a regular file",
|
||||
},
|
||||
)
|
||||
self.assertEqual(rows["a.png"]["detail"], "Kept copy")
|
||||
self.assertEqual(cleanup.counters, {"removed": 1, "kept": 2, "errors": 0})
|
||||
report = cleanup.report()["outcomes"]["kept"]
|
||||
self.assertEqual(
|
||||
sorted(row["path"] for row in report["rows"]), ["a.png", "b.png"]
|
||||
)
|
||||
self.assertFalse(report["truncated"])
|
||||
|
||||
|
||||
class TestCleanupJob(DuplicateFilesTestCase):
|
||||
async def test_cleanup_is_a_records_job_with_a_safety_backup(self):
|
||||
self.trio()
|
||||
|
||||
@@ -42,6 +42,37 @@ class TestLibraryAPI(ImageAPITestCase):
|
||||
self.assertEqual(len(media["images"]), 1)
|
||||
self.assertEqual(media["images"][0]["path"], "same.png")
|
||||
|
||||
async def test_used_and_reverse_name_sorts_page_through_every_prompt(self):
|
||||
ids = {text: self._save_prompt(text) for text in ("alpha", "beta", "gamma")}
|
||||
with self.api.db.model.get_connection() as conn:
|
||||
for text, runs, used in (
|
||||
("alpha", 1, "2026-01-01T00:00:00+00:00"),
|
||||
("beta", 9, "2026-02-01T00:00:00+00:00"),
|
||||
("gamma", 0, None),
|
||||
):
|
||||
conn.execute(
|
||||
"UPDATE prompts SET run_count=?, last_used_at=? WHERE id=?",
|
||||
(runs, used, ids[text]),
|
||||
)
|
||||
conn.commit()
|
||||
for sort, expected in (
|
||||
("most-used", ["beta", "alpha", "gamma"]),
|
||||
("last-used", ["beta", "alpha", "gamma"]),
|
||||
("name-desc", ["gamma", "beta", "alpha"]),
|
||||
):
|
||||
with self.subTest(sort=sort):
|
||||
seen, cursor = [], None
|
||||
for _ in expected:
|
||||
params = {"sort": sort, "limit": 1}
|
||||
if cursor:
|
||||
params["cursor"] = cursor
|
||||
response, page = await self.get(**params)
|
||||
self.assertEqual(response.status, 200)
|
||||
seen.extend(item["text"] for item in page["items"])
|
||||
cursor = page["next_cursor"]
|
||||
self.assertEqual(seen, expected)
|
||||
self.assertIsNone(cursor)
|
||||
|
||||
async def test_scoped_cursor_tampering_filters_and_continuation(self):
|
||||
for text in ("one", "two", "three"):
|
||||
self._save_prompt(text)
|
||||
|
||||
@@ -91,6 +91,8 @@ class TestLocationRoute(LibraryAPITestCase):
|
||||
"size_bytes": os.path.getsize(self.api.db.model.db_path),
|
||||
"target_source": "default",
|
||||
"config_moves": True,
|
||||
# #320 D201: previews kept inside each source (no folder).
|
||||
"thumbnails_folder": None,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@@ -266,6 +266,30 @@ class TestApply(LogCleanupTestCase):
|
||||
report = json.dumps(job.report())
|
||||
self.assertNotIn(self.tmpdir, report)
|
||||
|
||||
async def test_apply_lists_each_kept_file(self):
|
||||
linked = self.seed()
|
||||
plan = await self.preview()
|
||||
_summary, job = await self.run_apply(plan)
|
||||
expected = [
|
||||
("prompt_manager.log", "Kept: the active log"),
|
||||
("prompt_manager.log.9", "Kept: another file"),
|
||||
("prompt_manager.log.bak", "Kept: another file"),
|
||||
]
|
||||
if linked:
|
||||
expected.append(("prompt_manager.log.4", "Kept: a link"))
|
||||
rows = job.outcomes["protected"]
|
||||
self.assertEqual(
|
||||
sorted((row["path"], row["detail"]) for row in rows), sorted(expected)
|
||||
)
|
||||
self.assertEqual(
|
||||
{(row["root"], row["root_index"]) for row in rows}, {("", None)}
|
||||
)
|
||||
report = job.report()["outcomes"]["protected"]
|
||||
self.assertEqual(
|
||||
(report["count"], len(report["rows"]), report["truncated"]),
|
||||
(plan["protected"]["count"], len(expected), False),
|
||||
)
|
||||
|
||||
async def test_apply_rows_name_each_file(self):
|
||||
self.seed()
|
||||
plan = await self.preview()
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
"""Protected maintenance items are listed in Activity, not only counted (#586).
|
||||
|
||||
Each record a maintenance job keeps gets one ``protected`` row with its
|
||||
reason, in the wording of that operation's review, and the job's
|
||||
downloadable report lists every one of them (never cut at 200 rows). A kept
|
||||
image link names its file, so Activity can still open and locate it.
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
_mock_server = MagicMock()
|
||||
_mock_server.PromptServer.instance.routes = MagicMock()
|
||||
sys.modules.setdefault("server", _mock_server)
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from py.api.jobs import MAX_OUTCOME_ROWS # noqa: E402
|
||||
from py.config import GalleryConfig, PromptManagerConfig # noqa: E402
|
||||
|
||||
try:
|
||||
from tests.test_api_maintenance import ApplyTestCase # noqa: E402
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from test_api_maintenance import ApplyTestCase # noqa: E402
|
||||
|
||||
|
||||
def _record(detail):
|
||||
return {"path": "", "root": "", "root_index": None, "detail": detail}
|
||||
|
||||
|
||||
class TestProtectedRows(ApplyTestCase):
|
||||
def duplicate(self, text, created_at="2999-01-01"):
|
||||
with self.api.db.model.get_connection() as conn:
|
||||
return conn.execute(
|
||||
"INSERT INTO prompts(text, created_at) VALUES(?, ?)",
|
||||
(text.upper(), created_at),
|
||||
).lastrowid
|
||||
|
||||
def assert_complete(self, job, count):
|
||||
self.assertEqual(job.counters["protected"], count)
|
||||
report = job.report()["outcomes"]["protected"]
|
||||
self.assertEqual(
|
||||
(report["count"], len(report["rows"]), report["truncated"]),
|
||||
(count, count, False),
|
||||
)
|
||||
|
||||
async def test_duplicate_prompts_lists_each_kept_copy(self):
|
||||
paired = self.save("A dog")
|
||||
self.api.db.save_prompt_pair("A dog", "")
|
||||
paired_copy = self.duplicate("A dog")
|
||||
titled = self.save("A bird")
|
||||
titled_copy = self.duplicate("A bird")
|
||||
with self.api.db.model.get_connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO prompt_library_state(prompt_id,title) VALUES(?,?)",
|
||||
(titled_copy, "Bird recipe"),
|
||||
)
|
||||
plan, _summary, job = await self.run_apply("duplicate_prompts")
|
||||
self.assertEqual(job.state, "finished", job.log)
|
||||
self.assertEqual(
|
||||
sorted(row["detail"] for row in job.outcomes["protected"]),
|
||||
sorted(
|
||||
[
|
||||
f"Prompt {paired}: kept, belongs to a saved pair",
|
||||
f"Prompt {paired_copy}: kept, belongs to a saved pair",
|
||||
f"Prompt {titled}: kept, a copy has its own Library title",
|
||||
f"Prompt {titled_copy}: kept, a copy has its own Library title",
|
||||
]
|
||||
),
|
||||
)
|
||||
self.assertEqual(plan["protected"]["count"], 4)
|
||||
self.assert_complete(job, 4)
|
||||
|
||||
async def test_unused_components_lists_each_kept_prompt_with_its_reasons(self):
|
||||
self.save("Orphan")
|
||||
tagged = self.save("Kept", tags=["__protected__"])
|
||||
self.api.db.save_prompt_pair("Paired", "")
|
||||
paired = self.api.db.search_prompts(text="Paired")[0]["id"]
|
||||
with self.api.db.model.get_connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO prompt_tags(prompt_id, tag_id) "
|
||||
"SELECT ?, id FROM tags WHERE name='__protected__'",
|
||||
(paired,),
|
||||
)
|
||||
plan, _summary, job = await self.run_apply("unused_components")
|
||||
self.assertEqual(job.state, "finished", job.log)
|
||||
self.assertEqual(
|
||||
sorted(job.outcomes["protected"], key=lambda row: row["detail"]),
|
||||
sorted(
|
||||
[
|
||||
_record(f"Prompt {tagged}: kept, tagged __protected__"),
|
||||
_record(
|
||||
f"Prompt {paired}: kept, in a saved pair; "
|
||||
"tagged __protected__"
|
||||
),
|
||||
],
|
||||
key=lambda row: row["detail"],
|
||||
),
|
||||
)
|
||||
self.assertEqual(plan["protected"]["count"], 2)
|
||||
self.assert_complete(job, 2)
|
||||
|
||||
async def test_report_lists_every_kept_prompt(self):
|
||||
PromptManagerConfig.SAFETY_BACKUP = False
|
||||
count = MAX_OUTCOME_ROWS + 5
|
||||
for n in range(count):
|
||||
self.save(f"Kept {n:03d}", tags=["__protected__"])
|
||||
_plan, _summary, job = await self.run_apply("unused_components")
|
||||
self.assertEqual(len(job.outcomes["protected"]), MAX_OUTCOME_ROWS)
|
||||
self.assert_complete(job, count)
|
||||
|
||||
async def test_missing_links_lists_each_kept_link_with_its_file(self):
|
||||
unmounted = Path(self.tmpdir) / "unmounted"
|
||||
unmounted.mkdir()
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir), str(unmounted)]
|
||||
prompt = self.save("Links")
|
||||
self.image(prompt, "gone.png", exists=False)
|
||||
self.image(prompt, "sub/away.png", folder=unmounted, exists=False)
|
||||
self.image(prompt, "x.png", folder=Path(self.tmpdir) / "elsewhere")
|
||||
shutil.rmtree(unmounted)
|
||||
plan, _summary, job = await self.run_apply("missing_links")
|
||||
self.assertEqual(job.state, "finished", job.log)
|
||||
self.assertEqual(plan["protected"]["count"], 2)
|
||||
rows = sorted(job.outcomes["protected"], key=lambda row: row["path"])
|
||||
self.assertEqual(
|
||||
[(row["path"], row["detail"]) for row in rows],
|
||||
[
|
||||
("sub/away.png", "Kept: link under an unavailable source"),
|
||||
("x.png", "Kept: link outside the image sources"),
|
||||
],
|
||||
)
|
||||
# The gone source keeps the path under its label; outside every
|
||||
# source the row names the file only.
|
||||
self.assertEqual((rows[0]["root"], rows[0]["root_index"]), ("unmounted", None))
|
||||
self.assertEqual((rows[1]["root"], rows[1]["root_index"]), ("", None))
|
||||
self.assertNotIn(self.tmpdir, str(job.report()))
|
||||
self.assert_complete(job, 2)
|
||||
@@ -684,6 +684,28 @@ class TestTagMutationRoutes(RouteCoverageCase):
|
||||
|
||||
class TestSaveRoute(RouteCoverageCase):
|
||||
|
||||
async def test_a_malformed_body_is_refused_and_saves_nothing(self):
|
||||
for body in (
|
||||
[],
|
||||
"text",
|
||||
{"text": 123},
|
||||
{"text": "fixture", "category": 123},
|
||||
{"text": "fixture", "notes": 123},
|
||||
{"text": "fixture", "tags": 123},
|
||||
{"text": "fixture", "rating": "high"},
|
||||
):
|
||||
with self.subTest(body=body):
|
||||
status, data = await self._json(
|
||||
"POST", "/prompt_manager/save", json=body
|
||||
)
|
||||
self.assertEqual(status, 400)
|
||||
self.assertFalse(data["success"])
|
||||
status, data = await self._json(
|
||||
"POST", "/prompt_manager/save", data=b"{not json"
|
||||
)
|
||||
self.assertEqual(status, 400)
|
||||
self.assertEqual(self.api.db.search_prompts(text="fixture"), [])
|
||||
|
||||
async def test_invalid_rating_is_400(self):
|
||||
status, data = await self._json(
|
||||
"POST", "/prompt_manager/save", json={"text": "ok", "rating": 99}
|
||||
|
||||
@@ -0,0 +1,467 @@
|
||||
"""The thumbnail location setting (#320 PR 2): parsing, validation, source
|
||||
keys, the marker, the settings payloads and reverse containment.
|
||||
|
||||
Spelling rules are tested on both path flavors (``ntpath`` and
|
||||
``posixpath``) as text; filesystem rules run on the native one.
|
||||
"""
|
||||
|
||||
import json
|
||||
import ntpath
|
||||
import os
|
||||
import posixpath
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
sys.modules.setdefault("server", MagicMock())
|
||||
|
||||
from py.api import preview_location as pl # noqa: E402
|
||||
from py.api import preview_store # noqa: E402
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
class TestParse(unittest.TestCase):
|
||||
def test_missing_or_source_is_inside_each_source(self):
|
||||
for value in (None, {"mode": "source"}, {}):
|
||||
with self.subTest(value=value):
|
||||
parsed = pl.parse_location(value)
|
||||
self.assertEqual((parsed.mode, parsed.problem), ("source", None))
|
||||
|
||||
def test_a_folder_location_is_kept_as_spelled(self):
|
||||
parsed = pl.parse_location(
|
||||
{"mode": "folder", "path": "D:\\Thumbs", "id": "x", "install": "i"},
|
||||
flavor=ntpath,
|
||||
)
|
||||
self.assertEqual(
|
||||
(parsed.mode, parsed.path, parsed.id, parsed.problem),
|
||||
("folder", "D:\\Thumbs", "x", None),
|
||||
)
|
||||
|
||||
def test_invalid_values_never_fall_back_to_the_sources(self):
|
||||
for value in (
|
||||
"folder",
|
||||
{"mode": "elsewhere"},
|
||||
{"mode": "folder", "path": "relative", "id": "x"},
|
||||
{"mode": "folder", "path": "/abs"},
|
||||
{"mode": "folder", "path": "/abs", "id": ""},
|
||||
{"mode": "folder", "path": 7, "id": "x"},
|
||||
):
|
||||
with self.subTest(value=value):
|
||||
parsed = pl.parse_location(value, flavor=posixpath)
|
||||
self.assertEqual((parsed.mode, parsed.problem), ("folder", "invalid"))
|
||||
|
||||
|
||||
class TestForm(unittest.TestCase):
|
||||
def test_windows_spellings(self):
|
||||
cases = {
|
||||
"D:\\Thumbs": None,
|
||||
"\\\\server\\share\\thumbs": None,
|
||||
"C:thumbs": "location-not-absolute",
|
||||
"\\thumbs": "location-not-absolute",
|
||||
"thumbs": "location-not-absolute",
|
||||
"": "location-not-absolute",
|
||||
" D:\\Thumbs": "location-not-absolute",
|
||||
"\\\\?\\D:\\Thumbs": "location-device-path",
|
||||
"\\\\.\\D:\\Thumbs": "location-device-path",
|
||||
"D:\\CON\\x": "location-invalid-name",
|
||||
"D:\\com1.txt": "location-invalid-name",
|
||||
"D:\\thumbs.": "location-invalid-name",
|
||||
"D:\\thumbs ": "location-invalid-name",
|
||||
"D:\\a|b": "location-invalid-name",
|
||||
"D:\\a?b": "location-invalid-name",
|
||||
"D:\\a\x01b": "location-invalid-name",
|
||||
}
|
||||
for path, code in cases.items():
|
||||
with self.subTest(path=path):
|
||||
self.assertEqual(pl.form_problem(path, flavor=ntpath), code)
|
||||
|
||||
def test_posix_spellings(self):
|
||||
cases = {
|
||||
"/mnt/fast/thumbs": None,
|
||||
"mnt/thumbs": "location-not-absolute",
|
||||
"/mnt/nul": "location-invalid-name",
|
||||
"/mnt/a:b": "location-invalid-name",
|
||||
"//?/x": "location-device-path",
|
||||
}
|
||||
for path, code in cases.items():
|
||||
with self.subTest(path=path):
|
||||
self.assertEqual(pl.form_problem(path, flavor=posixpath), code)
|
||||
|
||||
def test_roots_and_home_are_too_broad(self):
|
||||
self.assertTrue(pl.is_too_broad("C:\\", home="C:\\Users\\a", flavor=ntpath))
|
||||
self.assertTrue(
|
||||
pl.is_too_broad("\\\\server\\share", home="C:\\Users\\a", flavor=ntpath)
|
||||
)
|
||||
self.assertTrue(
|
||||
pl.is_too_broad("C:\\users\\A", home="C:\\Users\\a", flavor=ntpath)
|
||||
)
|
||||
self.assertTrue(pl.is_too_broad("/", home="/home/a", flavor=posixpath))
|
||||
self.assertTrue(pl.is_too_broad("/home/a/", home="/home/a", flavor=posixpath))
|
||||
self.assertFalse(
|
||||
pl.is_too_broad("/home/a/thumbs", home="/home/a", flavor=posixpath)
|
||||
)
|
||||
|
||||
def test_long_windows_paths_are_refused_without_long_paths(self):
|
||||
short = "D:\\" + "a" * 100
|
||||
long = "D:\\" + "a" * 200
|
||||
self.assertFalse(pl.too_long_for_windows(short, long_paths=False))
|
||||
self.assertTrue(pl.too_long_for_windows(long, long_paths=False))
|
||||
self.assertFalse(pl.too_long_for_windows(long, long_paths=True))
|
||||
|
||||
|
||||
class TestSourceKeys(unittest.TestCase):
|
||||
def test_slug(self):
|
||||
self.assertEqual(pl.slug("ComfyUI Output"), "comfyui-output")
|
||||
self.assertEqual(pl.slug("Ünïcödé_Fôlder"), "unicode_folder")
|
||||
self.assertEqual(pl.slug(""), "source")
|
||||
self.assertEqual(pl.slug("!!!"), "source")
|
||||
self.assertEqual(len(pl.slug("x" * 80)), 24)
|
||||
|
||||
def test_keys_are_a_slug_plus_a_hash(self):
|
||||
key = pl.source_key("D:\\AI\\Output", "d:\\ai\\output", {})
|
||||
self.assertRegex(key, r"^output-[0-9a-f]{8}$")
|
||||
self.assertRegex(pl.source_key("D:\\", "d:\\", {}), r"^source-[0-9a-f]{8}$")
|
||||
self.assertRegex(
|
||||
pl.source_key("\\\\nas\\share\\Out", "\\\\nas\\share\\out", {}),
|
||||
r"^out-[0-9a-f]{8}$",
|
||||
)
|
||||
|
||||
def test_a_taken_key_grows_its_hash(self):
|
||||
first = pl.source_key("/a/out", "/a/out", {})
|
||||
taken = {first: "/someone/else"}
|
||||
second = pl.source_key("/a/out", "/a/out", taken)
|
||||
self.assertRegex(second, r"^out-[0-9a-f]{12}$")
|
||||
taken[second] = "/another"
|
||||
self.assertRegex(
|
||||
pl.source_key("/a/out", "/a/out", taken), r"^out-[0-9a-f]{16}$"
|
||||
)
|
||||
self.assertEqual(pl.source_key("/a/out", "/a/out", {first: "/a/out"}), first)
|
||||
|
||||
def test_lookup_by_spelling_identity_then_file_id(self):
|
||||
keys = {
|
||||
"out-0123abcd": {
|
||||
"path": "D:\\Out",
|
||||
"identity": "d:\\out",
|
||||
"dev_ino": [1, 2],
|
||||
},
|
||||
"lib-0123abcd": {"path": "E:\\Lib", "identity": "e:\\lib", "dev_ino": None},
|
||||
}
|
||||
self.assertEqual(pl.find_key(keys, "D:\\Out", "x", None), "out-0123abcd")
|
||||
self.assertEqual(pl.find_key(keys, "Z:\\Out", "d:\\out", None), "out-0123abcd")
|
||||
# A letter spelling and its UNC spelling: the same file id.
|
||||
self.assertEqual(
|
||||
pl.find_key(keys, "\\\\nas\\out", "\\\\nas\\out", (1, 2)), "out-0123abcd"
|
||||
)
|
||||
self.assertIsNone(pl.find_key(keys, "F:\\New", "f:\\new", (9, 9)))
|
||||
|
||||
|
||||
class TestMarker(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self._tmp.cleanup)
|
||||
self.folder = self._tmp.name
|
||||
|
||||
def test_init_creates_once_and_adopts_after(self):
|
||||
first = pl.init_folder(self.folder, created_by_pm=True)
|
||||
self.assertEqual(pl.init_folder(self.folder), first)
|
||||
marker = pl.read_marker(self.folder)
|
||||
self.assertEqual(marker["id"], first)
|
||||
self.assertTrue(marker["created_by_pm"])
|
||||
self.assertEqual(marker["version"], 1)
|
||||
|
||||
def test_writes_merge_what_another_writer_added(self):
|
||||
pl.init_folder(self.folder)
|
||||
pl.write_marker(
|
||||
self.folder, lambda m: m["keys"].update({"a-0123abcd": {"install": "1"}})
|
||||
)
|
||||
pl.write_marker(
|
||||
self.folder, lambda m: m["keys"].update({"b-0123abcd": {"install": "2"}})
|
||||
)
|
||||
self.assertEqual(
|
||||
sorted(pl.read_marker(self.folder)["keys"]), ["a-0123abcd", "b-0123abcd"]
|
||||
)
|
||||
# No temp file is left beside it.
|
||||
self.assertEqual(os.listdir(self.folder), [pl.MARKER_NAME])
|
||||
|
||||
def test_two_processes_adding_keys_at_once_keep_every_key(self):
|
||||
"""Installs sharing one folder (bot review, P2): the read-modify-write
|
||||
of the marker is locked across processes, so no writer drops another's
|
||||
key. Each child widens the window by pausing inside ``mutate``."""
|
||||
import subprocess
|
||||
|
||||
pl.init_folder(self.folder)
|
||||
child = (
|
||||
"import sys, time\n"
|
||||
"sys.path.insert(0, sys.argv[1])\n"
|
||||
"from py.api import preview_location as pl\n"
|
||||
"folder, tag, start = sys.argv[2], sys.argv[3], float(sys.argv[4])\n"
|
||||
"while time.time() < start: time.sleep(0.001)\n"
|
||||
"for n in range(8):\n"
|
||||
" def add(m, n=n):\n"
|
||||
" time.sleep(0.01)\n"
|
||||
" m['keys'][f'{tag}-{n:08x}'] = {'install': tag}\n"
|
||||
" pl.write_marker(folder, add)\n"
|
||||
)
|
||||
import time
|
||||
|
||||
start = str(time.time() + 1.0)
|
||||
procs = [
|
||||
subprocess.Popen( # nosec B603 - this interpreter, fixed arguments
|
||||
[sys.executable, "-c", child, str(REPO), self.folder, tag, start]
|
||||
)
|
||||
for tag in ("a", "b", "c")
|
||||
]
|
||||
for proc in procs:
|
||||
self.assertEqual(proc.wait(timeout=60), 0)
|
||||
keys = pl.read_marker(self.folder)["keys"]
|
||||
self.assertEqual(len(keys), 24, sorted(keys))
|
||||
self.assertEqual(os.listdir(self.folder), [pl.MARKER_NAME])
|
||||
|
||||
def test_a_lock_left_by_a_crashed_writer_is_taken_over_when_stale(self):
|
||||
pl.init_folder(self.folder)
|
||||
lock = os.path.join(self.folder, pl.MARKER_LOCK_NAME)
|
||||
with open(lock, "w", encoding="utf-8") as handle:
|
||||
handle.write("crashed")
|
||||
old = pl.time.time() - pl.MARKER_LOCK_STALE - 5
|
||||
os.utime(lock, (old, old))
|
||||
pl.write_marker(self.folder, lambda m: m["keys"].update({"a-0123abcd": {}}))
|
||||
self.assertIn("a-0123abcd", pl.read_marker(self.folder)["keys"])
|
||||
self.assertFalse(os.path.exists(lock))
|
||||
|
||||
def test_a_held_lock_times_out_without_writing(self):
|
||||
pl.init_folder(self.folder)
|
||||
lock = os.path.join(self.folder, pl.MARKER_LOCK_NAME)
|
||||
with open(lock, "w", encoding="utf-8") as handle:
|
||||
handle.write("held")
|
||||
with patch.object(pl, "MARKER_LOCK_TIMEOUT", 0.05):
|
||||
with self.assertRaises(TimeoutError):
|
||||
pl.write_marker(self.folder, lambda m: m["keys"].update({"x": {}}))
|
||||
self.assertTrue(os.path.exists(lock)) # never taken from its holder
|
||||
self.assertEqual(pl.read_marker(self.folder)["keys"], {})
|
||||
|
||||
def test_a_folder_without_the_marker_is_missing(self):
|
||||
self.assertEqual(pl.folder_state(self.folder, "x")[:2], (False, "missing"))
|
||||
missing = os.path.join(self.folder, "gone")
|
||||
self.assertEqual(pl.folder_state(missing, "x")[:2], (False, "missing"))
|
||||
marker_id = pl.init_folder(self.folder)
|
||||
self.assertEqual(pl.folder_state(self.folder, marker_id)[:2], (True, None))
|
||||
self.assertEqual(
|
||||
pl.folder_state(self.folder, "other")[:2], (False, "marker-mismatch")
|
||||
)
|
||||
|
||||
@unittest.skipUnless(hasattr(os, "symlink"), "needs symlinks")
|
||||
def test_a_linked_folder_is_not_available(self):
|
||||
real = os.path.join(self.folder, "real")
|
||||
os.mkdir(real)
|
||||
marker_id = pl.init_folder(real)
|
||||
link = os.path.join(self.folder, "link")
|
||||
try:
|
||||
os.symlink(real, link, target_is_directory=True)
|
||||
except OSError:
|
||||
self.skipTest("symlinks not permitted")
|
||||
self.assertEqual(pl.folder_state(link, marker_id)[:2], (False, "invalid"))
|
||||
|
||||
|
||||
class TestValidation(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self._tmp.cleanup)
|
||||
self.base = os.path.realpath(self._tmp.name)
|
||||
self.source = os.path.join(self.base, "output")
|
||||
self.node = os.path.join(self.base, "node")
|
||||
os.mkdir(self.source)
|
||||
os.mkdir(self.node)
|
||||
self.context = pl.Context(
|
||||
roots=(self.source,),
|
||||
node_folder=self.node,
|
||||
default_path=os.path.join(self.node, "data", "thumbnails"),
|
||||
home=os.path.join(self.base, "home"),
|
||||
environ={},
|
||||
long_paths=True,
|
||||
folder_paths=None,
|
||||
)
|
||||
|
||||
def problem(self, path, purpose="move", **changes):
|
||||
context = self.context
|
||||
if changes:
|
||||
context = pl.Context(**{**context.__dict__, **changes})
|
||||
return pl.location_problem(path, purpose=purpose, context=context)
|
||||
|
||||
def test_an_empty_new_folder_beside_the_sources_is_accepted(self):
|
||||
self.assertIsNone(self.problem(os.path.join(self.base, "thumbs")))
|
||||
os.mkdir(os.path.join(self.base, "empty"))
|
||||
Path(self.base, "empty", "Thumbs.db").write_bytes(b"")
|
||||
self.assertIsNone(self.problem(os.path.join(self.base, "empty")))
|
||||
|
||||
def test_codes(self):
|
||||
cases = {
|
||||
"relative": "location-not-absolute",
|
||||
os.path.join(self.source, "thumbs"): "location-served",
|
||||
os.path.join(self.node, "thumbs"): "location-node-folder",
|
||||
self.base: "location-contains-source",
|
||||
os.path.join(self.base, "no", "parent"): "location-parent-missing",
|
||||
}
|
||||
for path, code in cases.items():
|
||||
with self.subTest(path=path):
|
||||
self.assertEqual(self.problem(path), code)
|
||||
|
||||
def test_the_default_may_sit_in_the_node_folder_and_make_its_parents(self):
|
||||
self.assertIsNone(self.problem(self.context.default_path))
|
||||
|
||||
def test_the_folder_in_use_is_the_same(self):
|
||||
folder = os.path.join(self.base, "thumbs")
|
||||
os.mkdir(folder)
|
||||
self.assertEqual(self.problem(folder, current_path=folder), "location-same")
|
||||
|
||||
def test_a_folder_inside_the_folder_in_use_is_refused(self):
|
||||
# Codex P2 on #592: the old key folders are removed after the move,
|
||||
# and with them a target nested inside one of them.
|
||||
folder = os.path.join(self.base, "thumbs")
|
||||
os.mkdir(folder)
|
||||
pl.init_folder(folder)
|
||||
for nested in (
|
||||
os.path.join(folder, "output-0123abcd", "nested"),
|
||||
os.path.join(folder, "beside"),
|
||||
):
|
||||
with self.subTest(nested=nested):
|
||||
self.assertEqual(
|
||||
self.problem(nested, current_path=folder),
|
||||
"location-inside-current",
|
||||
)
|
||||
|
||||
def test_a_folder_with_other_files_is_not_empty_unless_it_is_ours(self):
|
||||
folder = os.path.join(self.base, "full")
|
||||
os.mkdir(folder)
|
||||
Path(folder, "photo.jpg").write_bytes(b"x")
|
||||
self.assertEqual(self.problem(folder), "location-not-empty")
|
||||
pl.init_folder(folder)
|
||||
self.assertIsNone(self.problem(folder))
|
||||
|
||||
@unittest.skipIf(os.name == "nt" or os.geteuid() == 0, "needs POSIX modes")
|
||||
def test_a_folder_that_refuses_writes(self):
|
||||
folder = os.path.join(self.base, "ro")
|
||||
os.mkdir(folder)
|
||||
os.chmod(folder, 0o500)
|
||||
self.addCleanup(os.chmod, folder, 0o700)
|
||||
self.assertEqual(self.problem(folder), "location-not-writable")
|
||||
|
||||
def test_a_stat_that_does_not_answer_is_unavailable(self):
|
||||
release = threading.Event()
|
||||
self.addCleanup(release.set)
|
||||
|
||||
def stuck(*_args):
|
||||
release.wait(10)
|
||||
|
||||
with patch.object(pl, "_filesystem_problem", stuck):
|
||||
code = pl.location_problem(
|
||||
os.path.join(self.base, "thumbs"),
|
||||
purpose="move",
|
||||
context=self.context,
|
||||
timeout=0.1,
|
||||
)
|
||||
self.assertEqual(code, "location-unavailable")
|
||||
|
||||
def test_a_long_windows_path(self):
|
||||
path = os.path.join(self.base, "t" * 220)
|
||||
self.assertEqual(self.problem(path, long_paths=False), "location-path-too-long")
|
||||
|
||||
def test_every_code_has_a_message_without_a_path(self):
|
||||
for code, (status, text) in pl.MESSAGES.items():
|
||||
with self.subTest(code=code):
|
||||
self.assertIn(status, (400, 409, 507))
|
||||
self.assertTrue(text.endswith("."))
|
||||
|
||||
|
||||
try:
|
||||
from tests.test_api_images import ImageAPITestCase
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from test_api_images import ImageAPITestCase
|
||||
|
||||
|
||||
class TestLocationRoutes(ImageAPITestCase):
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
self.addCleanup(preview_store.configure, None)
|
||||
|
||||
def use_folder(self):
|
||||
folder = self.comfy_dir / "thumbs"
|
||||
folder.mkdir()
|
||||
marker_id = pl.init_folder(str(folder), created_by_pm=True)
|
||||
preview_store.configure(
|
||||
{"mode": "folder", "path": str(folder), "id": marker_id}
|
||||
)
|
||||
return folder
|
||||
|
||||
async def test_settings_get_names_the_location_with_its_path(self):
|
||||
folder = self.use_folder()
|
||||
resp = await self.client.get("/prompt_manager/settings")
|
||||
location = (await resp.json())["thumbnail_location"]
|
||||
self.assertEqual(location["mode"], "folder")
|
||||
self.assertEqual(location["path"], str(folder))
|
||||
self.assertTrue(location["available"])
|
||||
self.assertIsNone(location["problem"])
|
||||
self.assertIn("default_path", location)
|
||||
|
||||
async def test_settings_post_never_changes_the_location(self):
|
||||
resp = await self.client.post(
|
||||
"/prompt_manager/settings",
|
||||
json={"thumbnail_location": {"mode": "folder", "path": "/x"}},
|
||||
)
|
||||
self.assertEqual(resp.status, 409)
|
||||
body = await resp.json()
|
||||
self.assertEqual(body["code"], "location-needs-move")
|
||||
self.assertNotIn("/x", body["error"])
|
||||
|
||||
async def test_status_has_the_location_but_no_path(self):
|
||||
folder = self.use_folder()
|
||||
resp = await self.client.get("/prompt_manager/images/thumbnails/status")
|
||||
body = await resp.text()
|
||||
self.assertNotIn(str(folder), body)
|
||||
self.assertEqual(
|
||||
json.loads(body)["location"],
|
||||
{"mode": "folder", "available": True, "problem": None},
|
||||
)
|
||||
|
||||
async def test_a_source_inside_or_around_the_folder_is_refused(self):
|
||||
from py.api import sources
|
||||
|
||||
folder = self.use_folder()
|
||||
inside = folder / "sub"
|
||||
inside.mkdir()
|
||||
with patch(
|
||||
"py.config.GalleryConfig.gallery_root_problem",
|
||||
classmethod(lambda cls, path: (None, "")),
|
||||
):
|
||||
for path in (inside, folder, self.comfy_dir):
|
||||
with self.subTest(path=path):
|
||||
code, reason = sources.source_problem(str(path))
|
||||
self.assertEqual(code, "holds-thumbnails")
|
||||
self.assertEqual(reason, pl.HOLDS_THUMBNAILS_REASON)
|
||||
self.assertEqual(sources.source_problem(str(self.output_dir))[0], None)
|
||||
|
||||
async def test_a_library_move_into_the_folder_is_refused(self):
|
||||
from py.api import library_location as routes
|
||||
|
||||
folder = self.use_folder()
|
||||
self.assertTrue(routes._inside_thumbnail_folder(str(folder / "lib")))
|
||||
self.assertFalse(routes._inside_thumbnail_folder(str(self.output_dir)))
|
||||
|
||||
def test_startup_makes_a_location_inside_a_source_invalid(self):
|
||||
inside = self.output_dir / "thumbs"
|
||||
inside.mkdir()
|
||||
marker_id = pl.init_folder(str(inside))
|
||||
from py.config import PromptManagerConfig
|
||||
|
||||
value = {"mode": "folder", "path": str(inside), "id": marker_id}
|
||||
with patch.object(PromptManagerConfig, "THUMBNAIL_LOCATION", value):
|
||||
location = self.api.start_preview_location()
|
||||
self.assertEqual((location.available, location.problem), (False, "invalid"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -363,11 +363,11 @@ class TestWorkflowStorageOutcomes(WorkflowStorageTestCase):
|
||||
prompt_id = self.old_library(count=4)
|
||||
conn = self.api.db.model.get_connection()
|
||||
with conn:
|
||||
conn.execute(
|
||||
image_id = conn.execute(
|
||||
"INSERT INTO generated_images (prompt_id, image_path, filename, "
|
||||
"file_path, workflow_data) VALUES (?, '/x/b', 'b', '/x/b', x'00')",
|
||||
(prompt_id,),
|
||||
)
|
||||
).lastrowid
|
||||
store_text(conn, "a document no record references")
|
||||
_plan, _summary, job = await self.start()
|
||||
await self.finish(job)
|
||||
@@ -377,6 +377,13 @@ class TestWorkflowStorageOutcomes(WorkflowStorageTestCase):
|
||||
"Rewrote 4 image records; 1 stored value not text, kept as is", log
|
||||
)
|
||||
self.assertIn("Removed 1 unused workflow document", log)
|
||||
# #586: the kept value has its row, in the review's wording.
|
||||
self.assertEqual(
|
||||
[row["detail"] for row in job.outcomes["protected"]],
|
||||
[f"Image record {image_id}: workflow, stored value not text; kept as is"],
|
||||
)
|
||||
report = job.report()["outcomes"]["protected"]
|
||||
self.assertEqual((report["count"], report["truncated"]), (1, False))
|
||||
|
||||
async def reclaim_with(self, **patches):
|
||||
self.old_library(count=4)
|
||||
|
||||
@@ -197,7 +197,6 @@ class JobCase(unittest.TestCase):
|
||||
node.logger = logging.getLogger("test.cached_runs")
|
||||
node.prompt_tracker = mock.Mock()
|
||||
self.trackers.append(node.prompt_tracker)
|
||||
node.comfyui_integration = mock.Mock()
|
||||
node._inject_lora_trigger_words = lambda text: text
|
||||
node.pending_first_use = self.pending
|
||||
node.pending_pair_first_use = self.pair_pending
|
||||
|
||||
@@ -1,153 +0,0 @@
|
||||
"""Tests for utils.comfyui_integration."""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import types
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from utils.comfyui_integration import ComfyUIMetadataIntegration
|
||||
|
||||
|
||||
class TestSaveImageNotPatched(unittest.TestCase):
|
||||
"""Regression for #75: SaveImage must save each node's own prompt inputs.
|
||||
|
||||
The old SaveImage patch wrote the last-run PromptManager prompt into every
|
||||
PromptManager node, so negative prompts were saved as copies of the positive.
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
def save_images(
|
||||
self_node,
|
||||
images,
|
||||
filename_prefix="ComfyUI",
|
||||
prompt=None,
|
||||
extra_pnginfo=None,
|
||||
):
|
||||
return {"prompt": prompt}
|
||||
|
||||
self.original_save_images = save_images
|
||||
fake_nodes = types.ModuleType("nodes")
|
||||
fake_nodes.SaveImage = type("SaveImage", (), {"save_images": save_images})
|
||||
self._saved_nodes = sys.modules.get("nodes")
|
||||
sys.modules["nodes"] = fake_nodes
|
||||
self.fake_nodes = fake_nodes
|
||||
ComfyUIMetadataIntegration._instance = None
|
||||
|
||||
def tearDown(self):
|
||||
ComfyUIMetadataIntegration._instance = None
|
||||
if self._saved_nodes is None:
|
||||
sys.modules.pop("nodes", None)
|
||||
else:
|
||||
sys.modules["nodes"] = self._saved_nodes
|
||||
|
||||
def test_save_images_is_left_untouched(self):
|
||||
ComfyUIMetadataIntegration()
|
||||
self.assertIs(self.fake_nodes.SaveImage.save_images, self.original_save_images)
|
||||
|
||||
def test_registered_prompt_does_not_rewrite_saved_metadata(self):
|
||||
integration = ComfyUIMetadataIntegration()
|
||||
integration.register_prompt("pm_1", "positive final text", {})
|
||||
prompt = {
|
||||
"39": {"class_type": "PromptManager", "inputs": {"text": "a cat"}},
|
||||
"40": {"class_type": "PromptManager", "inputs": {"text": "blurry"}},
|
||||
}
|
||||
|
||||
result = self.fake_nodes.SaveImage().save_images([], prompt=prompt)
|
||||
|
||||
self.assertEqual(result["prompt"]["39"]["inputs"]["text"], "a cat")
|
||||
self.assertEqual(result["prompt"]["40"]["inputs"]["text"], "blurry")
|
||||
|
||||
def test_prompt_registry_still_available_to_nodes(self):
|
||||
integration = ComfyUIMetadataIntegration()
|
||||
integration.register_prompt("pm_1", "hello", {})
|
||||
self.assertEqual(integration.get_current_prompt_text("pm_1"), "hello")
|
||||
|
||||
|
||||
import threading # noqa: E402
|
||||
import unittest.mock as mock # noqa: E402
|
||||
|
||||
from utils import comfyui_integration # noqa: E402
|
||||
from utils.comfyui_integration import get_comfyui_integration # noqa: E402
|
||||
|
||||
|
||||
class IntegrationTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
ComfyUIMetadataIntegration._instance = None
|
||||
comfyui_integration._integration_instance = None
|
||||
# Patch the module's own ``time`` name rather than ``time.time`` globally:
|
||||
# background threads from other tests (image monitor, watchdog) would
|
||||
# otherwise tick this clock and make fresh registrations look stale.
|
||||
self.clock = mock.patch.object(
|
||||
comfyui_integration, "time", types.SimpleNamespace(time=self._now)
|
||||
)
|
||||
self.clock.start()
|
||||
self.addCleanup(self.clock.stop)
|
||||
self._ticks = 1000.0
|
||||
self.integration = get_comfyui_integration()
|
||||
|
||||
def tearDown(self):
|
||||
ComfyUIMetadataIntegration._instance = None
|
||||
comfyui_integration._integration_instance = None
|
||||
|
||||
def _now(self):
|
||||
self._ticks += 1.0
|
||||
return self._ticks
|
||||
|
||||
|
||||
class TestSingletonAndRegistry(IntegrationTestCase):
|
||||
def test_singleton_keeps_registrations_across_constructions(self):
|
||||
self.assertIs(get_comfyui_integration(), self.integration)
|
||||
self.assertIs(ComfyUIMetadataIntegration(), self.integration)
|
||||
self.integration.register_prompt("n1", "hello", {"category": "x"})
|
||||
ComfyUIMetadataIntegration() # __init__ runs again: must not reset state
|
||||
self.assertEqual(self.integration.get_current_prompt_text("n1"), "hello")
|
||||
|
||||
def test_latest_prompt_of_this_thread_wins_without_a_node_id(self):
|
||||
self.integration.register_prompt("n1", "first", {})
|
||||
self.integration.register_prompt("n2", "second", {})
|
||||
self.assertEqual(self.integration.get_current_prompt_text(), "second")
|
||||
self.assertEqual(self.integration.get_current_prompt_text("missing"), "second")
|
||||
self.assertEqual(self.integration.get_current_prompt_text("n1"), "first")
|
||||
|
||||
def test_nothing_registered_returns_none(self):
|
||||
self.assertIsNone(self.integration.get_current_prompt_text())
|
||||
self.assertIsNone(self.integration.get_current_prompt_text("n1"))
|
||||
|
||||
|
||||
class TestCrossThreadLookup(IntegrationTestCase):
|
||||
def _register_in_thread(self, text):
|
||||
worker = threading.Thread(
|
||||
target=self.integration.register_prompt, args=("w1", text, {})
|
||||
)
|
||||
worker.start()
|
||||
worker.join()
|
||||
|
||||
def test_recent_prompt_from_another_thread_is_used(self):
|
||||
self._register_in_thread("from worker")
|
||||
self.assertEqual(self.integration.get_current_prompt_text(), "from worker")
|
||||
|
||||
def test_stale_prompt_from_another_thread_is_ignored(self):
|
||||
self._register_in_thread("from worker")
|
||||
self._ticks += 400.0
|
||||
self.assertIsNone(self.integration.get_current_prompt_text())
|
||||
|
||||
def test_global_entry_of_this_thread_is_found_without_thread_local(self):
|
||||
self.integration.register_prompt("n1", "mine", {})
|
||||
self.integration._thread_local = threading.local()
|
||||
self.assertEqual(self.integration.get_current_prompt_text(), "mine")
|
||||
|
||||
def test_cleanup_removes_old_registrations_only(self):
|
||||
self._register_in_thread("old")
|
||||
self._ticks += 700.0
|
||||
self.integration.register_prompt("n2", "new", {})
|
||||
self.integration.cleanup_old_prompts(max_age_seconds=600)
|
||||
texts = sorted(v["text"] for v in self.integration._current_prompts.values())
|
||||
self.assertEqual(texts, ["new"])
|
||||
self.integration.cleanup_old_prompts(max_age_seconds=600) # nothing to do
|
||||
self.assertEqual(len(self.integration._current_prompts), 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+4
-63
@@ -213,7 +213,8 @@ class TestPromptManagerConfig(unittest.TestCase):
|
||||
try:
|
||||
# Modify a value
|
||||
PromptManagerConfig.MAX_SEARCH_RESULTS = 42
|
||||
PromptManagerConfig.save_to_file(tmp.name)
|
||||
with open(tmp.name, "w", encoding="utf-8") as handle:
|
||||
json.dump(PromptManagerConfig.get_config(), handle)
|
||||
|
||||
# Reset and reload
|
||||
PromptManagerConfig.MAX_SEARCH_RESULTS = 100
|
||||
@@ -240,55 +241,6 @@ class TestPromptManagerConfig(unittest.TestCase):
|
||||
finally:
|
||||
os.unlink(tmp.name)
|
||||
|
||||
def test_save_creates_directory(self):
|
||||
tmp_dir = tempfile.mkdtemp()
|
||||
nested_path = os.path.join(tmp_dir, "subdir", "config.json")
|
||||
try:
|
||||
PromptManagerConfig.save_to_file(nested_path)
|
||||
self.assertTrue(os.path.exists(nested_path))
|
||||
with open(nested_path) as f:
|
||||
data = json.load(f)
|
||||
self.assertIn("database", data)
|
||||
finally:
|
||||
import shutil
|
||||
|
||||
shutil.rmtree(tmp_dir)
|
||||
|
||||
def test_saved_file_is_private_to_the_user(self):
|
||||
import stat
|
||||
|
||||
tmp_dir = tempfile.mkdtemp()
|
||||
try:
|
||||
path = os.path.join(tmp_dir, "config.json")
|
||||
PromptManagerConfig.save_to_file(path)
|
||||
mode = stat.S_IMODE(os.stat(path).st_mode)
|
||||
if os.name == "posix":
|
||||
self.assertEqual(mode, 0o600)
|
||||
with open(path) as f:
|
||||
self.assertIn("database", json.load(f))
|
||||
# Overwriting an existing, more open file tightens it as well
|
||||
os.chmod(path, 0o644)
|
||||
PromptManagerConfig.save_to_file(path)
|
||||
if os.name == "posix":
|
||||
self.assertEqual(stat.S_IMODE(os.stat(path).st_mode), 0o600)
|
||||
finally:
|
||||
import shutil
|
||||
|
||||
shutil.rmtree(tmp_dir, ignore_errors=True)
|
||||
|
||||
def test_saved_file_is_valid_json(self):
|
||||
tmp = tempfile.NamedTemporaryFile(
|
||||
delete=False, suffix=".json", dir=tempfile.gettempdir()
|
||||
)
|
||||
tmp.close()
|
||||
try:
|
||||
PromptManagerConfig.save_to_file(tmp.name)
|
||||
with open(tmp.name) as f:
|
||||
data = json.load(f)
|
||||
self.assertIsInstance(data, dict)
|
||||
finally:
|
||||
os.unlink(tmp.name)
|
||||
|
||||
|
||||
class TestIntegrationConfig(unittest.TestCase):
|
||||
"""Test IntegrationConfig for LoRA Manager settings."""
|
||||
@@ -404,18 +356,6 @@ class TestConfigPathOverride(unittest.TestCase):
|
||||
os.environ[self.ENV] = custom
|
||||
self.assertEqual(PromptManagerConfig.get_config_path(), custom)
|
||||
|
||||
def test_save_without_path_uses_env_override(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
custom = os.path.join(tmpdir, "nested", "custom.json")
|
||||
os.environ[self.ENV] = custom
|
||||
PromptManagerConfig.RESULT_TIMEOUT = 42
|
||||
|
||||
PromptManagerConfig.save_to_file()
|
||||
|
||||
with open(custom) as f:
|
||||
saved = json.load(f)
|
||||
self.assertEqual(saved["web_ui"]["result_timeout"], 42)
|
||||
|
||||
def test_load_without_path_uses_env_override(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
custom = os.path.join(tmpdir, "custom.json")
|
||||
@@ -503,7 +443,8 @@ class TestAutotagConfig(unittest.TestCase):
|
||||
PromptManagerConfig.update_config(
|
||||
{"autotag": {"defaults": {"suggestion_limit": 25}}}
|
||||
)
|
||||
PromptManagerConfig.save_to_file(path)
|
||||
with open(path, "w", encoding="utf-8") as handle:
|
||||
json.dump(PromptManagerConfig.get_config(), handle)
|
||||
self.config_class.SUGGESTION_LIMIT = 10
|
||||
PromptManagerConfig.load_from_file(path)
|
||||
self.assertEqual(self.config_class.suggestion_limit(), 25)
|
||||
|
||||
@@ -15,12 +15,11 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
@patch("prompt_manager_base.get_image_monitor")
|
||||
@patch("prompt_manager_base.get_prompt_tracker")
|
||||
@patch("prompt_manager_base.get_comfyui_integration")
|
||||
@patch("prompt_manager_base.PromptDatabase")
|
||||
class TestPromptManagerTextEmptyInput(unittest.TestCase):
|
||||
"""Test PromptManagerText with empty/blank text inputs (issue #123)."""
|
||||
|
||||
def _make_node(self, mock_db_class, mock_integration, mock_tracker, mock_monitor):
|
||||
def _make_node(self, mock_db_class, mock_tracker, mock_monitor):
|
||||
mock_db = Mock()
|
||||
mock_db_class.return_value = mock_db
|
||||
mock_db.save_prompt.return_value = 1
|
||||
@@ -30,73 +29,64 @@ class TestPromptManagerTextEmptyInput(unittest.TestCase):
|
||||
|
||||
return PromptManagerText()
|
||||
|
||||
def test_empty_string(self, mock_db, mock_integ, mock_tracker, mock_monitor):
|
||||
def test_empty_string(self, mock_db, mock_tracker, mock_monitor):
|
||||
"""Empty string should not raise UnboundLocalError."""
|
||||
node = self._make_node(mock_db, mock_integ, mock_tracker, mock_monitor)
|
||||
node = self._make_node(mock_db, mock_tracker, mock_monitor)
|
||||
result = node.process_text(text="")
|
||||
self.assertIsInstance(result, tuple)
|
||||
self.assertEqual(result[0], "")
|
||||
|
||||
def test_whitespace_only(self, mock_db, mock_integ, mock_tracker, mock_monitor):
|
||||
def test_whitespace_only(self, mock_db, mock_tracker, mock_monitor):
|
||||
"""Whitespace-only string should not raise UnboundLocalError."""
|
||||
node = self._make_node(mock_db, mock_integ, mock_tracker, mock_monitor)
|
||||
node = self._make_node(mock_db, mock_tracker, mock_monitor)
|
||||
result = node.process_text(text=" ")
|
||||
self.assertIsInstance(result, tuple)
|
||||
|
||||
def test_none_coerced_empty(self, mock_db, mock_integ, mock_tracker, mock_monitor):
|
||||
def test_none_coerced_empty(self, mock_db, mock_tracker, mock_monitor):
|
||||
"""Falsy text value should not raise UnboundLocalError."""
|
||||
node = self._make_node(mock_db, mock_integ, mock_tracker, mock_monitor)
|
||||
node = self._make_node(mock_db, mock_tracker, mock_monitor)
|
||||
# ComfyUI may pass empty string for unconnected inputs
|
||||
result = node.process_text(text="", category="", tags="")
|
||||
self.assertIsInstance(result, tuple)
|
||||
|
||||
def test_empty_text_with_prepend_append(
|
||||
self, mock_db, mock_integ, mock_tracker, mock_monitor
|
||||
):
|
||||
def test_empty_text_with_prepend_append(self, mock_db, mock_tracker, mock_monitor):
|
||||
"""Empty main text with prepend/append should produce single space, not
|
||||
double."""
|
||||
node = self._make_node(mock_db, mock_integ, mock_tracker, mock_monitor)
|
||||
node = self._make_node(mock_db, mock_tracker, mock_monitor)
|
||||
result = node.process_text(text="", prepend_text="before", append_text="after")
|
||||
self.assertIsInstance(result, tuple)
|
||||
self.assertEqual(result[0], "before after")
|
||||
self.assertNotIn(" ", result[0])
|
||||
|
||||
def test_valid_text_still_works(
|
||||
self, mock_db, mock_integ, mock_tracker, mock_monitor
|
||||
):
|
||||
def test_valid_text_still_works(self, mock_db, mock_tracker, mock_monitor):
|
||||
"""Normal text input should continue to work correctly."""
|
||||
node = self._make_node(mock_db, mock_integ, mock_tracker, mock_monitor)
|
||||
node = self._make_node(mock_db, mock_tracker, mock_monitor)
|
||||
result = node.process_text(text="a beautiful sunset")
|
||||
self.assertIsInstance(result, tuple)
|
||||
self.assertEqual(result[0], "a beautiful sunset")
|
||||
|
||||
def test_prepend_text_append_all_present(
|
||||
self, mock_db, mock_integ, mock_tracker, mock_monitor
|
||||
):
|
||||
def test_prepend_text_append_all_present(self, mock_db, mock_tracker, mock_monitor):
|
||||
"""All three parts should combine with single spaces."""
|
||||
node = self._make_node(mock_db, mock_integ, mock_tracker, mock_monitor)
|
||||
node = self._make_node(mock_db, mock_tracker, mock_monitor)
|
||||
result = node.process_text(
|
||||
text="main prompt", prepend_text="before", append_text="after"
|
||||
)
|
||||
self.assertEqual(result[0], "before main prompt after")
|
||||
|
||||
def test_only_prepend(self, mock_db, mock_integ, mock_tracker, mock_monitor):
|
||||
def test_only_prepend(self, mock_db, mock_tracker, mock_monitor):
|
||||
"""Prepend with empty text should output just prepend."""
|
||||
node = self._make_node(mock_db, mock_integ, mock_tracker, mock_monitor)
|
||||
node = self._make_node(mock_db, mock_tracker, mock_monitor)
|
||||
result = node.process_text(text="", prepend_text="before")
|
||||
self.assertEqual(result[0], "before")
|
||||
|
||||
|
||||
@patch("prompt_manager_base.get_image_monitor")
|
||||
@patch("prompt_manager_base.get_prompt_tracker")
|
||||
@patch("prompt_manager_base.get_comfyui_integration")
|
||||
@patch("prompt_manager_base.PromptDatabase")
|
||||
class TestPromptManagerEmptyInput(unittest.TestCase):
|
||||
"""Test PromptManager (CLIP variant) with empty/blank text inputs (issue #123)."""
|
||||
|
||||
def _make_node_and_clip(
|
||||
self, mock_db_class, mock_integration, mock_tracker, mock_monitor
|
||||
):
|
||||
def _make_node_and_clip(self, mock_db_class, mock_tracker, mock_monitor):
|
||||
mock_db = Mock()
|
||||
mock_db_class.return_value = mock_db
|
||||
mock_db.save_prompt.return_value = 1
|
||||
@@ -110,30 +100,22 @@ class TestPromptManagerEmptyInput(unittest.TestCase):
|
||||
|
||||
return PromptManager(), mock_clip
|
||||
|
||||
def test_empty_string(self, mock_db, mock_integ, mock_tracker, mock_monitor):
|
||||
def test_empty_string(self, mock_db, mock_tracker, mock_monitor):
|
||||
"""Empty string should not raise UnboundLocalError."""
|
||||
node, clip = self._make_node_and_clip(
|
||||
mock_db, mock_integ, mock_tracker, mock_monitor
|
||||
)
|
||||
node, clip = self._make_node_and_clip(mock_db, mock_tracker, mock_monitor)
|
||||
result = node.encode_prompt(clip=clip, text="")
|
||||
self.assertIsInstance(result, tuple)
|
||||
|
||||
def test_whitespace_only(self, mock_db, mock_integ, mock_tracker, mock_monitor):
|
||||
def test_whitespace_only(self, mock_db, mock_tracker, mock_monitor):
|
||||
"""Whitespace-only string should not raise UnboundLocalError."""
|
||||
node, clip = self._make_node_and_clip(
|
||||
mock_db, mock_integ, mock_tracker, mock_monitor
|
||||
)
|
||||
node, clip = self._make_node_and_clip(mock_db, mock_tracker, mock_monitor)
|
||||
result = node.encode_prompt(clip=clip, text=" ")
|
||||
self.assertIsInstance(result, tuple)
|
||||
|
||||
def test_empty_text_with_prepend_append(
|
||||
self, mock_db, mock_integ, mock_tracker, mock_monitor
|
||||
):
|
||||
def test_empty_text_with_prepend_append(self, mock_db, mock_tracker, mock_monitor):
|
||||
"""Empty main text with prepend/append should produce single space, not
|
||||
double."""
|
||||
node, clip = self._make_node_and_clip(
|
||||
mock_db, mock_integ, mock_tracker, mock_monitor
|
||||
)
|
||||
node, clip = self._make_node_and_clip(mock_db, mock_tracker, mock_monitor)
|
||||
result = node.encode_prompt(
|
||||
clip=clip, text="", prepend_text="before", append_text="after"
|
||||
)
|
||||
@@ -141,24 +123,16 @@ class TestPromptManagerEmptyInput(unittest.TestCase):
|
||||
self.assertEqual(result[1], "before after")
|
||||
self.assertNotIn(" ", result[1])
|
||||
|
||||
def test_valid_text_still_works(
|
||||
self, mock_db, mock_integ, mock_tracker, mock_monitor
|
||||
):
|
||||
def test_valid_text_still_works(self, mock_db, mock_tracker, mock_monitor):
|
||||
"""Normal text input should continue to work correctly."""
|
||||
node, clip = self._make_node_and_clip(
|
||||
mock_db, mock_integ, mock_tracker, mock_monitor
|
||||
)
|
||||
node, clip = self._make_node_and_clip(mock_db, mock_tracker, mock_monitor)
|
||||
result = node.encode_prompt(clip=clip, text="a beautiful sunset")
|
||||
self.assertIsInstance(result, tuple)
|
||||
self.assertEqual(result[1], "a beautiful sunset")
|
||||
|
||||
def test_prepend_text_append_all_present(
|
||||
self, mock_db, mock_integ, mock_tracker, mock_monitor
|
||||
):
|
||||
def test_prepend_text_append_all_present(self, mock_db, mock_tracker, mock_monitor):
|
||||
"""All three parts should combine with single spaces."""
|
||||
node, clip = self._make_node_and_clip(
|
||||
mock_db, mock_integ, mock_tracker, mock_monitor
|
||||
)
|
||||
node, clip = self._make_node_and_clip(mock_db, mock_tracker, mock_monitor)
|
||||
result = node.encode_prompt(
|
||||
clip=clip, text="main prompt", prepend_text="before", append_text="after"
|
||||
)
|
||||
@@ -167,7 +141,6 @@ class TestPromptManagerEmptyInput(unittest.TestCase):
|
||||
|
||||
@patch("prompt_manager_base.get_image_monitor")
|
||||
@patch("prompt_manager_base.get_prompt_tracker")
|
||||
@patch("prompt_manager_base.get_comfyui_integration")
|
||||
@patch("prompt_manager_base.PromptDatabase")
|
||||
class TestConnectedTextInput(unittest.TestCase):
|
||||
"""Test that externally connected text is preserved in output (issue #123).
|
||||
@@ -177,7 +150,7 @@ class TestConnectedTextInput(unittest.TestCase):
|
||||
value reaches Python; these tests verify Python handles it correctly.
|
||||
"""
|
||||
|
||||
def _make_text_node(self, mock_db_class, mock_integ, mock_tracker, mock_monitor):
|
||||
def _make_text_node(self, mock_db_class, mock_tracker, mock_monitor):
|
||||
mock_db = Mock()
|
||||
mock_db_class.return_value = mock_db
|
||||
mock_db.save_prompt.return_value = 1
|
||||
@@ -186,7 +159,7 @@ class TestConnectedTextInput(unittest.TestCase):
|
||||
|
||||
return PromptManagerText()
|
||||
|
||||
def _make_clip_node(self, mock_db_class, mock_integ, mock_tracker, mock_monitor):
|
||||
def _make_clip_node(self, mock_db_class, mock_tracker, mock_monitor):
|
||||
mock_db = Mock()
|
||||
mock_db_class.return_value = mock_db
|
||||
mock_db.save_prompt.return_value = 1
|
||||
@@ -199,10 +172,10 @@ class TestConnectedTextInput(unittest.TestCase):
|
||||
return PromptManager(), mock_clip
|
||||
|
||||
def test_text_node_connected_text_with_prepend_append(
|
||||
self, mock_db, mock_integ, mock_tracker, mock_monitor
|
||||
self, mock_db, mock_tracker, mock_monitor
|
||||
):
|
||||
"""Simulates screenshot: connected text + prepend + append must all appear."""
|
||||
node = self._make_text_node(mock_db, mock_integ, mock_tracker, mock_monitor)
|
||||
node = self._make_text_node(mock_db, mock_tracker, mock_monitor)
|
||||
result = node.process_text(
|
||||
text="This is the prompt",
|
||||
prepend_text="This is prepend text",
|
||||
@@ -213,21 +186,17 @@ class TestConnectedTextInput(unittest.TestCase):
|
||||
"This is prepend text This is the prompt This is append text",
|
||||
)
|
||||
|
||||
def test_text_node_connected_text_only(
|
||||
self, mock_db, mock_integ, mock_tracker, mock_monitor
|
||||
):
|
||||
def test_text_node_connected_text_only(self, mock_db, mock_tracker, mock_monitor):
|
||||
"""Connected text without prepend/append passes through unchanged."""
|
||||
node = self._make_text_node(mock_db, mock_integ, mock_tracker, mock_monitor)
|
||||
node = self._make_text_node(mock_db, mock_tracker, mock_monitor)
|
||||
result = node.process_text(text="This is the prompt")
|
||||
self.assertEqual(result[0], "This is the prompt")
|
||||
|
||||
def test_clip_node_connected_text_with_prepend_append(
|
||||
self, mock_db, mock_integ, mock_tracker, mock_monitor
|
||||
self, mock_db, mock_tracker, mock_monitor
|
||||
):
|
||||
"""Same scenario for PromptManager (CLIP variant)."""
|
||||
node, clip = self._make_clip_node(
|
||||
mock_db, mock_integ, mock_tracker, mock_monitor
|
||||
)
|
||||
node, clip = self._make_clip_node(mock_db, mock_tracker, mock_monitor)
|
||||
result = node.encode_prompt(
|
||||
clip=clip,
|
||||
text="This is the prompt",
|
||||
@@ -239,13 +208,9 @@ class TestConnectedTextInput(unittest.TestCase):
|
||||
"This is prepend text This is the prompt This is append text",
|
||||
)
|
||||
|
||||
def test_clip_node_connected_text_only(
|
||||
self, mock_db, mock_integ, mock_tracker, mock_monitor
|
||||
):
|
||||
def test_clip_node_connected_text_only(self, mock_db, mock_tracker, mock_monitor):
|
||||
"""Connected text without prepend/append passes through unchanged."""
|
||||
node, clip = self._make_clip_node(
|
||||
mock_db, mock_integ, mock_tracker, mock_monitor
|
||||
)
|
||||
node, clip = self._make_clip_node(mock_db, mock_tracker, mock_monitor)
|
||||
result = node.encode_prompt(clip=clip, text="This is the prompt")
|
||||
self.assertEqual(result[1], "This is the prompt")
|
||||
|
||||
|
||||
@@ -324,7 +324,6 @@ class CaptureTests(unittest.TestCase):
|
||||
node.db.save_prompt_pair.side_effect = RuntimeError("offline")
|
||||
node.logger = logging.getLogger("test.capture.nodes")
|
||||
node.prompt_tracker = Mock()
|
||||
node.comfyui_integration = Mock()
|
||||
node._inject_lora_trigger_words = lambda text: text + " TRIGGER"
|
||||
clip = Mock()
|
||||
clip.tokenize.side_effect = lambda text: text
|
||||
|
||||
@@ -24,7 +24,6 @@ class TestFocusedNode(unittest.TestCase):
|
||||
def test_focused_encodes_composition_and_tracks_main_text_and_graph(self):
|
||||
node = PromptManagerFocused.__new__(PromptManagerFocused)
|
||||
node.logger = logging.getLogger("test.focused")
|
||||
node.comfyui_integration = mock.Mock()
|
||||
node._inject_lora_trigger_words = lambda text: text
|
||||
node._track_prompt_execution = mock.Mock(return_value=42)
|
||||
clip = mock.Mock()
|
||||
|
||||
+1
-29
@@ -1,7 +1,7 @@
|
||||
"""
|
||||
Tests for hashing utilities.
|
||||
|
||||
Tests is_duplicate_prompt and generate_prompt_hash, which are
|
||||
Tests and generate_prompt_hash, which are
|
||||
currently untested. generate_prompt_hash is covered in test_basic.py
|
||||
but we add edge cases here.
|
||||
"""
|
||||
@@ -15,7 +15,6 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from tests.background_stops import stop_tracker # noqa: E402
|
||||
from utils.hashing import (
|
||||
generate_prompt_hash,
|
||||
is_duplicate_prompt,
|
||||
STRIPPED_CODE_POINTS,
|
||||
prompt_key,
|
||||
register_prompt_functions,
|
||||
@@ -69,33 +68,6 @@ class TestSingleHashImplementation(unittest.TestCase):
|
||||
self.assertFalse(os.path.exists(os.path.join(ROOT, "restart_gallery.py")))
|
||||
|
||||
|
||||
class TestIsDuplicatePrompt(unittest.TestCase):
|
||||
"""Test is_duplicate_prompt function."""
|
||||
|
||||
def test_identical_texts_are_duplicates(self):
|
||||
self.assertTrue(is_duplicate_prompt("hello", "hello"))
|
||||
|
||||
def test_case_insensitive(self):
|
||||
self.assertTrue(is_duplicate_prompt("Hello World", "hello world"))
|
||||
|
||||
def test_whitespace_insensitive(self):
|
||||
self.assertTrue(is_duplicate_prompt(" hello ", "hello"))
|
||||
|
||||
def test_different_texts_not_duplicates(self):
|
||||
self.assertFalse(is_duplicate_prompt("text A", "text B"))
|
||||
|
||||
def test_threshold_parameter_accepted(self):
|
||||
# threshold is unused but should not cause errors
|
||||
self.assertTrue(is_duplicate_prompt("same", "same", threshold=0.5))
|
||||
|
||||
def test_empty_strings_are_duplicates(self):
|
||||
self.assertTrue(is_duplicate_prompt("", ""))
|
||||
|
||||
def test_unicode_comparison(self):
|
||||
self.assertTrue(is_duplicate_prompt("日本語", "日本語"))
|
||||
self.assertFalse(is_duplicate_prompt("日本語", "中文"))
|
||||
|
||||
|
||||
class TestGeneratePromptHashEdgeCases(unittest.TestCase):
|
||||
"""Additional edge cases for generate_prompt_hash."""
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
# Mock ComfyUI server before importing anything that touches config. Only the
|
||||
# first test module to run installs the stub; nothing else in sys.modules is
|
||||
# replaced, so later modules (test_comfyui_integration) see the real package.
|
||||
# replaced, so later modules see the real package.
|
||||
_mock_server = MagicMock()
|
||||
_mock_server.PromptServer.instance.routes = MagicMock()
|
||||
sys.modules.setdefault("server", _mock_server)
|
||||
@@ -881,17 +881,6 @@ class TestOutputListener(unittest.TestCase):
|
||||
self.assertTrue(monitor.stop_monitoring())
|
||||
|
||||
|
||||
class TestModuleIsolation(unittest.TestCase):
|
||||
"""This module must not leave mocks behind for later test modules."""
|
||||
|
||||
def test_comfyui_integration_is_not_replaced_by_a_mock(self):
|
||||
module = sys.modules.get("utils.comfyui_integration")
|
||||
self.assertFalse(
|
||||
isinstance(module, MagicMock),
|
||||
"utils.comfyui_integration was replaced by a MagicMock in sys.modules",
|
||||
)
|
||||
|
||||
|
||||
class TestMonitorThreadSafety(unittest.TestCase):
|
||||
"""Singleton creation must be thread-safe."""
|
||||
|
||||
|
||||
@@ -406,8 +406,6 @@ class TestLegacyRoutesHoldTheRegistry(_LegacyHoldCase):
|
||||
)
|
||||
await self._busy(resp)
|
||||
self.assertTrue(victim.exists())
|
||||
# The duplicate family lock was not taken by the refused request.
|
||||
self.assertIsNotNone(self.api._acquire_long_job(self.api.LONG_JOB_DUPLICATES))
|
||||
|
||||
async def test_lora_scan_refused_while_scan_runs(self):
|
||||
from py.config import IntegrationConfig
|
||||
|
||||
@@ -506,6 +506,20 @@ class TestJobOutcomesAndLog(unittest.TestCase):
|
||||
self.assertFalse(report["changed"]["truncated"])
|
||||
self.assertEqual(report["errors"], {"count": 0, "rows": [], "truncated": False})
|
||||
|
||||
def test_record_into_a_complete_group_keeps_the_row_for_the_report(self):
|
||||
# #586: a group kept whole stays whole when later rows are recorded.
|
||||
job = Job("duplicate_cleanup", "Remove", groups=("kept",))
|
||||
rows = [
|
||||
{"path": f"f{i}.png", "detail": "Kept"} for i in range(MAX_OUTCOME_ROWS)
|
||||
]
|
||||
job.record_counted("kept", len(rows), rows, complete=True)
|
||||
job.record("kept", {"path": "late.png", "detail": "Kept copy"}, "= late.png")
|
||||
self.assertEqual(len(job.outcomes["kept"]), MAX_OUTCOME_ROWS)
|
||||
report = job.report()["outcomes"]["kept"]
|
||||
self.assertEqual(report["count"], MAX_OUTCOME_ROWS + 1)
|
||||
self.assertEqual(report["rows"][-1]["path"], "late.png")
|
||||
self.assertFalse(report["truncated"])
|
||||
|
||||
def test_thumbnail_job_groups(self):
|
||||
self.assertEqual(
|
||||
jobs_module.THUMBNAIL_GROUP_NAMES, ("generated", "skipped", "errors")
|
||||
|
||||
@@ -1061,17 +1061,78 @@ class LoraImportRefusalTests(LoraImportAPITestCase):
|
||||
job = await self._failed_job()
|
||||
self.assertEqual(job.scope["failure"], "backup-failed")
|
||||
|
||||
async def test_cancel_right_after_the_start_writes_nothing(self):
|
||||
async def test_cancel_while_it_stages_writes_nothing(self):
|
||||
# #575: held at a download, so the cancel lands while it stages.
|
||||
self._two_loras()
|
||||
before = _snapshot(self.api.db)
|
||||
resp, plan = await self._post(PREVIEW)
|
||||
resp, body = await self._post(APPLY, {"token": plan["token"]})
|
||||
job = self.api._jobs.get(body["job"]["id"])
|
||||
self.api._jobs.cancel(job.id) # at once, while it stages
|
||||
self.gate = threading.Event()
|
||||
_plan, job = await self._apply_reviewed()
|
||||
loop = asyncio.get_running_loop()
|
||||
self.assertTrue(await loop.run_in_executor(None, self.downloading.wait, 10))
|
||||
self.assertTrue(self.api._jobs.cancel(job.id))
|
||||
self.gate.set()
|
||||
await job.task
|
||||
self.assertEqual(job.state, "cancelled")
|
||||
self.assertEqual(_snapshot(self.api.db), before)
|
||||
|
||||
async def test_a_cancel_accepted_as_staging_ends_writes_nothing(self):
|
||||
# #575: staging already passed its last cancel check and returned; a
|
||||
# cancel accepted before the job resumes still stops it before the
|
||||
# commit, so a request answered "cancellation_requested" holds.
|
||||
self._two_loras()
|
||||
before = _snapshot(self.api.db)
|
||||
staged, release = threading.Event(), threading.Event()
|
||||
real_stage = lora_import.stage
|
||||
|
||||
def held_stage(*args, **kwargs):
|
||||
result = real_stage(*args, **kwargs)
|
||||
staged.set()
|
||||
release.wait(10)
|
||||
return result
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
with patch.object(lora_import, "stage", held_stage):
|
||||
_plan, job = await self._apply_reviewed()
|
||||
try:
|
||||
self.assertTrue(await loop.run_in_executor(None, staged.wait, 10))
|
||||
self.assertTrue(self.api._jobs.cancel(job.id))
|
||||
finally:
|
||||
release.set()
|
||||
await job.task
|
||||
self.assertEqual(job.state, "cancelled", job.log)
|
||||
self.assertEqual(_snapshot(self.api.db), before)
|
||||
|
||||
async def test_a_cancel_after_the_commit_point_is_refused(self):
|
||||
# #575: once staging is done the job is no longer cancellable; a
|
||||
# late cancel is refused (the route answers it as not requested) and
|
||||
# the job truthfully ends "finished" with the import written.
|
||||
self._two_loras()
|
||||
before = _snapshot(self.api.db)
|
||||
replacing, release = threading.Event(), threading.Event()
|
||||
real_replace = self.api.db.replace_category_prompts
|
||||
|
||||
def held_replace(*args, **kwargs):
|
||||
replacing.set()
|
||||
release.wait(10)
|
||||
return real_replace(*args, **kwargs)
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
with patch.object(self.api.db, "replace_category_prompts", held_replace):
|
||||
_plan, job = await self._apply_reviewed()
|
||||
try:
|
||||
self.assertTrue(await loop.run_in_executor(None, replacing.wait, 10))
|
||||
self.assertFalse(job.summary()["cancellable"])
|
||||
resp = await self.client.request(
|
||||
"POST", f"/prompt_manager/jobs/{job.id}/cancel"
|
||||
)
|
||||
body = json.loads(await resp.text())
|
||||
finally:
|
||||
release.set()
|
||||
await job.task
|
||||
self.assertFalse(body["cancellation_requested"])
|
||||
self.assertEqual(job.state, "finished", job.log)
|
||||
self.assertNotEqual(_snapshot(self.api.db), before)
|
||||
|
||||
|
||||
def plan_title():
|
||||
return lora_import.JOB_TITLE
|
||||
|
||||
@@ -171,7 +171,7 @@ class TestClassification(PairMetadataCase):
|
||||
self.assertEqual(self.preview().targets, [[prompt, True, True]])
|
||||
|
||||
def test_category_is_case_sensitive(self):
|
||||
_a, prompt = self.pair("case", "n1", category="Portrait")
|
||||
self.pair("case", "n1", category="Portrait")
|
||||
self.pair("case", "n2", category="portrait")
|
||||
draft = self.preview()
|
||||
self.assertEqual(draft.targets, [])
|
||||
@@ -292,7 +292,7 @@ class TestClassification(PairMetadataCase):
|
||||
)
|
||||
|
||||
def test_protected_rows_come_first_within_fifty(self):
|
||||
_c, conflict = self.pair("zz conflict", "n1", category="x")
|
||||
self.pair("zz conflict", "n1", category="x")
|
||||
self.pair("zz conflict", "n2", category="y")
|
||||
for n in range(60):
|
||||
self.pair(f"fill {n:02d}", category="portrait")
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
import sys
|
||||
import unittest
|
||||
from unittest import mock
|
||||
import unittest.mock as mock
|
||||
|
||||
from utils import native_memory
|
||||
|
||||
@@ -10,7 +10,9 @@ from utils import native_memory
|
||||
class ReleaseFreeMemoryTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
# Each test looks the symbol up again.
|
||||
patcher = mock.patch.multiple(native_memory, _trim=None, _looked_up=False)
|
||||
patcher = mock.patch.object(
|
||||
native_memory, "_trim", native_memory._NOT_LOOKED_UP
|
||||
)
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
|
||||
@@ -22,7 +24,7 @@ class ReleaseFreeMemoryTests(unittest.TestCase):
|
||||
def test_other_platforms_do_nothing(self):
|
||||
for platform in ("win32", "darwin", "cygwin"):
|
||||
with self.subTest(platform), mock.patch.object(sys, "platform", platform):
|
||||
native_memory._looked_up = False
|
||||
native_memory._trim = native_memory._NOT_LOOKED_UP
|
||||
with mock.patch.object(native_memory.ctypes, "CDLL") as cdll:
|
||||
self.assertFalse(native_memory.release_free_memory())
|
||||
cdll.assert_not_called()
|
||||
@@ -30,8 +32,12 @@ class ReleaseFreeMemoryTests(unittest.TestCase):
|
||||
def test_musl_does_nothing(self):
|
||||
with (
|
||||
mock.patch.object(sys, "platform", "linux"),
|
||||
# Windows has no os.confstr: this test fakes Linux there too.
|
||||
mock.patch.object(
|
||||
native_memory.os, "confstr", side_effect=ValueError("unknown name")
|
||||
native_memory.os,
|
||||
"confstr",
|
||||
side_effect=ValueError("unknown name"),
|
||||
create=True,
|
||||
),
|
||||
mock.patch.object(native_memory.ctypes, "CDLL") as cdll,
|
||||
):
|
||||
|
||||
@@ -179,7 +179,6 @@ class NodeTestCase(unittest.TestCase):
|
||||
self.node.logger = logging.getLogger("test.node")
|
||||
self.node.prompt_tracker = mock.Mock()
|
||||
self.node.image_monitor = mock.Mock()
|
||||
self.node.comfyui_integration = mock.Mock()
|
||||
|
||||
def tearDown(self):
|
||||
self.db.close_all() # Windows cannot unlink an open database file
|
||||
@@ -189,20 +188,16 @@ class NodeTestCase(unittest.TestCase):
|
||||
|
||||
|
||||
class TestBaseSaveAndSearch(NodeTestCase):
|
||||
def test_init_wires_database_tracker_monitor_and_integration(self):
|
||||
def test_init_wires_database_tracker_and_monitor(self):
|
||||
with (
|
||||
mock.patch.object(prompt_manager_base, "PromptDatabase") as db_cls,
|
||||
mock.patch.object(prompt_manager_base, "get_prompt_tracker") as tracker,
|
||||
mock.patch.object(prompt_manager_base, "get_image_monitor") as monitor,
|
||||
mock.patch.object(
|
||||
prompt_manager_base, "get_comfyui_integration"
|
||||
) as integration,
|
||||
):
|
||||
node = PromptManagerBase()
|
||||
self.assertIs(node.db, db_cls.return_value)
|
||||
tracker.assert_called_once_with(db_cls.return_value)
|
||||
monitor.assert_called_once_with(db_cls.return_value, tracker.return_value)
|
||||
self.assertIs(node.comfyui_integration, integration.return_value)
|
||||
monitor.return_value.start_monitoring.assert_called_once_with()
|
||||
|
||||
def test_saving_an_existing_prompt_updates_its_metadata(self):
|
||||
@@ -242,31 +237,18 @@ class TestBaseSaveAndSearch(NodeTestCase):
|
||||
self.node._save_prompt_to_database("a cat on a mat")
|
||||
self.assertEqual(self.node._search_prompts(""), [])
|
||||
self.assertEqual(self.node._search_prompts(" "), [])
|
||||
self.assertEqual(len(self.node.search_prompts_api("cat")), 1)
|
||||
self.assertEqual(len(self.node.get_recent_prompts_api(limit=5)["prompts"]), 1)
|
||||
self.assertEqual(len(self.node._search_prompts("cat")), 1)
|
||||
|
||||
def test_search_and_recent_errors_return_empty_lists(self):
|
||||
self.node.db = mock.Mock()
|
||||
self.node.db.search_prompts.side_effect = RuntimeError("locked")
|
||||
self.node.db.get_recent_prompts.side_effect = RuntimeError("locked")
|
||||
with self.assertLogs("test.node", level="ERROR"):
|
||||
self.assertEqual(self.node._search_prompts("cat"), [])
|
||||
with self.assertLogs("test.node", level="ERROR"):
|
||||
self.assertEqual(self.node.get_recent_prompts_api(), [])
|
||||
|
||||
def test_gallery_system_start_failure_is_logged(self):
|
||||
self.node.image_monitor.start_monitoring.side_effect = OSError("no inotify")
|
||||
with self.assertLogs("test.node", level="ERROR"):
|
||||
self.node._start_gallery_system()
|
||||
self.node.image_monitor.get_status.return_value = {"running": False}
|
||||
self.node.prompt_tracker.get_status.return_value = {"active_prompts_count": 0}
|
||||
self.assertEqual(
|
||||
self.node.get_gallery_status(),
|
||||
{
|
||||
"image_monitor": {"running": False},
|
||||
"prompt_tracker": {"active_prompts_count": 0},
|
||||
},
|
||||
)
|
||||
self.node.cleanup_gallery_system()
|
||||
self.node.image_monitor.stop_monitoring.assert_not_called()
|
||||
|
||||
@@ -376,7 +358,6 @@ class TestPromptManagerEncoding(NodeTestCase):
|
||||
self.assertEqual(saved["text"], "a cat")
|
||||
self.assertIn("prepend:best", saved["tags"])
|
||||
self.assertIn("append:8k", saved["tags"])
|
||||
self.node.comfyui_integration.register_prompt.assert_called_once()
|
||||
|
||||
def test_tracking_failure_does_not_break_encoding(self):
|
||||
self.node._track_prompt_execution = mock.Mock(side_effect=RuntimeError("db"))
|
||||
|
||||
@@ -19,7 +19,6 @@ def bare(node_cls):
|
||||
"""A node without its database, tracker or gallery side effects."""
|
||||
node = node_cls.__new__(node_cls)
|
||||
node.logger = __import__("logging").getLogger("test.wiring")
|
||||
node.comfyui_integration = mock.Mock()
|
||||
node._inject_lora_trigger_words = lambda text: text
|
||||
node._track_prompt_execution = mock.Mock(return_value=42)
|
||||
return node
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user