Compare commits
17
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
996bcd3372 | ||
|
|
1b9e581bd7 | ||
|
|
212f3162bb | ||
|
|
32bd5569e6 | ||
|
|
b2a305d3f3 | ||
|
|
323023d59c | ||
|
|
5379fe8b94 | ||
|
|
3be0ada6a1 | ||
|
|
48b1292115 | ||
|
|
fb9134f081 | ||
|
|
f9835aa4d6 | ||
|
|
863d717435 | ||
|
|
8252d9d226 | ||
|
|
28ad3a1a1f | ||
|
|
a905cc2316 | ||
|
|
9ab25f9aaa | ||
|
|
2964a3b632 |
@@ -19,6 +19,9 @@ jobs:
|
||||
security:
|
||||
name: bandit (high severity)
|
||||
runs-on: ubuntu-latest
|
||||
# ceil(1.5 x slowest of three green main runs) + 3, never under 5 (test.yml): runs 40257217665,
|
||||
# 40254137380, 40240706583 took 23s / 16s / 23s -> 5 min.
|
||||
timeout-minutes: 5
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
@@ -19,10 +19,22 @@ concurrency:
|
||||
env:
|
||||
PYTHONFAULTHANDLER: "1"
|
||||
|
||||
# Every job has a time limit: ceil(1.5 x the slowest of three recent green runs on main) + 3
|
||||
# minutes, never under 5. The 3 minutes leave room for the Python runner's watchdog (120 s, see
|
||||
# tests/run_suite.py) to print its stack dump before GitHub cancels the job. A limit changes only
|
||||
# with new measurements written here (TESTING.md "Time limits and hangs").
|
||||
# Measured 2026-10-26 on runs 40257217681 (3be0ada), 40240706568 (fb9134f), 40228167328 (f9835aa):
|
||||
# py matrix, slowest py3.10 on windows-latest: 12m43s / 15m54s / 14m53s -> 27 min
|
||||
# (ubuntu 5m27s-6m27s, macos 6m14s-8m30s, windows 11m42s-15m54s)
|
||||
# real ComfyUI compatibility: 13m32s / 8m55s / 11m46s -> 24 min
|
||||
# coverage gate (85%): 7m26s / 7m22s / 7m54s -> 15 min
|
||||
# perf budget 30s / 29s / 27s, black and flake8 39s / 37s / 39s,
|
||||
# frontend modules 23s / 19s / 29s -> 5 min (the floor)
|
||||
jobs:
|
||||
test:
|
||||
name: py${{ matrix.python-version }} on ${{ matrix.os }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 27
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -45,12 +57,12 @@ jobs:
|
||||
pip install -r requirements-dev.txt
|
||||
|
||||
- name: Run tests
|
||||
run: python -m unittest discover -s tests -v
|
||||
run: python -m tests.run_suite -v
|
||||
|
||||
comfyui:
|
||||
name: real ComfyUI compatibility
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 25
|
||||
timeout-minutes: 24
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
@@ -127,6 +139,7 @@ jobs:
|
||||
coverage:
|
||||
name: coverage gate (85%)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
@@ -143,7 +156,7 @@ jobs:
|
||||
|
||||
- name: Measure coverage
|
||||
run: |
|
||||
python -m coverage run -m unittest discover -s tests
|
||||
python -m coverage run -m tests.run_suite
|
||||
python -m coverage combine
|
||||
python -m coverage report --fail-under=85
|
||||
|
||||
@@ -154,6 +167,7 @@ jobs:
|
||||
name: perf budget
|
||||
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
@@ -176,6 +190,7 @@ jobs:
|
||||
lint:
|
||||
name: black and flake8
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/setup-python@v7
|
||||
@@ -188,6 +203,7 @@ jobs:
|
||||
js-test:
|
||||
name: frontend modules
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ repos:
|
||||
|
||||
- id: tests
|
||||
name: tests
|
||||
entry: python -m unittest discover tests/ -v
|
||||
entry: python -m tests.run_suite -q
|
||||
language: system
|
||||
pass_filenames: false
|
||||
stages: [pre-push]
|
||||
|
||||
@@ -501,6 +501,8 @@ To add your key:
|
||||
2. Paste it into the **CivitAI API Key** field in the Integrations settings
|
||||
3. Save and re-import to download any previously skipped images
|
||||
|
||||
The key is sent only over HTTPS to CivitAI's own hosts. CivitAI serves images through a redirect to its storage host; PromptManager follows such a redirect only over HTTPS to a host on CivitAI's own `civitai.com`, `civitai.red` or `civitai.green` domains, at most three hops, and never sends the key with it. When downloads fail, the import's status line says how many; when none could be downloaded, it starts with a warning. The server log has one warning per failed image with the reason (HTTP status, refused redirect, size cap, unreadable image, timeout or network error).
|
||||
|
||||
> **Note:** Re-importing is safe — PromptManager skips LoRAs that were already imported. To do a fresh import, the previous `lora-manager` data is cleared automatically before re-scanning.
|
||||
|
||||
### 🌐 Web Interface Features
|
||||
@@ -922,12 +924,12 @@ db.model.backup_database("backup_prompts.db")
|
||||
```bash
|
||||
cd ComfyUI_PromptManager
|
||||
pip install -r requirements-dev.txt
|
||||
python -m unittest discover -s tests # Python (what CI runs)
|
||||
python -m tests.run_suite # Python (what CI runs: unittest discovery plus a hang watchdog)
|
||||
node --test tests/js/*.test.js # frontend pure modules
|
||||
python -m coverage run -m unittest discover -s tests && python -m coverage combine && python -m coverage report
|
||||
python -m coverage run -m tests.run_suite && python -m coverage combine && python -m coverage report
|
||||
```
|
||||
|
||||
CI runs the suite on Linux, Windows and macOS for Python 3.10 to 3.12 with an 85% coverage gate. Tests that delete a database file first check that nothing in the process still holds it open (`tests/open_handles.py`), so a connection leak fails on Linux the way it would on Windows.
|
||||
CI runs the suite on Linux, Windows and macOS for Python 3.10 to 3.13 with an 85% coverage gate. `python -m unittest discover -s tests` still works; see `TESTING.md` ("Time limits and hangs") for the watchdog. Tests that delete a database file first check that nothing in the process still holds it open (`tests/open_handles.py`), so a connection leak fails on Linux the way it would on Windows.
|
||||
|
||||
### Code Style
|
||||
|
||||
|
||||
+661
-15
@@ -31,7 +31,7 @@ its reset endpoint exists only in the test harness.
|
||||
Use a Python environment containing `requirements-dev.txt` and Node.js 24:
|
||||
|
||||
```sh
|
||||
python -m unittest discover -s tests -q
|
||||
python -m tests.run_suite -q # what CI runs; see "Time limits and hangs"
|
||||
npm ci --prefix tools/vendor --ignore-scripts
|
||||
npx --prefix tools/vendor playwright install --with-deps chromium firefox webkit
|
||||
npm run --prefix tools/vendor test
|
||||
@@ -39,6 +39,9 @@ npm run --prefix tools/vendor test:coverage
|
||||
npm run --prefix tools/vendor test:workflows -- --project firefox --project webkit
|
||||
```
|
||||
|
||||
`python -m unittest discover -s tests` still works and finds the same tests;
|
||||
it just has no hang watchdog.
|
||||
|
||||
`PM_TEST_PYTHON` selects the Python executable; set it when Python is outside
|
||||
PATH. Every Playwright worker starts its own `tests/browser/server.py --port 0`,
|
||||
which binds a port the OS picks and prints it as `PM_TEST_SERVER_PORT=<n>`, with
|
||||
@@ -204,6 +207,62 @@ not certify every existing feature, arbitrary host versions,
|
||||
GPU/model changes, cross-pack installation or visual design changes. Those still
|
||||
need targeted validation.
|
||||
|
||||
## Time limits and hangs
|
||||
|
||||
CI and the pre-push hook run the Python suite as `python -m tests.run_suite`
|
||||
(`-v`, `-q`, `-s START` and `-p PATTERN` as in `unittest discover`), and the
|
||||
coverage job as `python -m coverage run -m tests.run_suite`. The runner
|
||||
(`tests/run_suite.py`) discovers the same tests with the same `sys.path` and
|
||||
warning filters as `python -m unittest discover -s tests`, exits 0, 1, or 5 when
|
||||
no tests ran (Python 3.12+ behaviour, on every version), and adds a watchdog: it writes `start: <test id>` to stderr before every test, at every
|
||||
verbosity, and arms `faulthandler.dump_traceback_later`. If a test, its teardown,
|
||||
its event-loop shutdown or interpreter exit stops making progress, the stack of
|
||||
every thread goes to stderr (`Timeout (0:02:00)!` then `Thread 0x...` blocks) and
|
||||
the process exits non-zero.
|
||||
|
||||
- `PM_TEST_HANG_LIMIT` (default 120 s) bounds each test from its `start:` line to
|
||||
the next one, including the class and module fixtures (`tearDownClass`,
|
||||
`setUpModule`, ...) that run in between. `PM_TEST_SHUTDOWN_LIMIT` (default
|
||||
120 s) bounds the end of the run: the summary, non-daemon threads, atexit
|
||||
handlers and coverage's save. Both must be
|
||||
positive integers; anything else stops the runner with exit 2. 120 s stays below
|
||||
asyncio's 300 s wait for default-executor threads on Python 3.12 and 3.13 (3.10
|
||||
and 3.11 wait without limit), so a hang in an `IsolatedAsyncioTestCase` loop
|
||||
shutdown is dumped inside the test that hung.
|
||||
- Reading a dump: the last `start:` line names the test. An `ok` after it means
|
||||
the test body passed and the hang came after it: in its teardown or event-loop
|
||||
shutdown (where `IsolatedAsyncioTestCase` joins executor threads), or in a
|
||||
class or module fixture before the next test; the stack shows which. A dump
|
||||
after the summary is a hang at interpreter exit. The dump shows thread stacks, not asyncio task
|
||||
stacks.
|
||||
- When the watchdog fires under coverage, that run's coverage data is lost; the
|
||||
job fails anyway and the dump is what matters.
|
||||
|
||||
Every job in `test.yml`, `code-quality.yml` and `viewerjs-check.yml` has
|
||||
`timeout-minutes` (`tests/vendor/ci-workflow.test.mjs` checks it). The rule is
|
||||
ceil(1.5 x the slowest of three recent green runs on `main`) + 3 minutes, never
|
||||
under 5; the three run ids, their times and the resulting limits are written
|
||||
in a comment above `jobs:`. The 3 minutes leave room for one watchdog firing, so
|
||||
the dump prints before GitHub cancels the job. A job that hits its limit, or a
|
||||
watchdog dump, is a failure like any other: read the dump, find the blocking
|
||||
call, fix it. Never raise a limit or rerun to get green; a limit changes only
|
||||
with new measurements written next to it.
|
||||
|
||||
### After a push
|
||||
|
||||
Follow the run for the commit you pushed, never an older one:
|
||||
|
||||
```sh
|
||||
gh run list --commit <pushed sha> --workflow test.yml # the run id for that commit
|
||||
gh run watch <run id> --exit-status # returns when the run ends
|
||||
gh run view <run id> --json jobs # shows a job that already failed
|
||||
gh run view --job <job id> --log-failed # that job's failed log, at once
|
||||
```
|
||||
|
||||
Start fixing as soon as a failed log shows the cause, but push the fix only after
|
||||
the run for that commit has finished: pull-request runs use `cancel-in-progress`,
|
||||
so a push cancels the running jobs and can hide a second failure.
|
||||
|
||||
## Performance budgets
|
||||
|
||||
A spec performance budget gets two tests:
|
||||
@@ -802,8 +861,8 @@ Recorded shell deviations:
|
||||
`file_size` equal to the file's size in this walk. Both paths go through
|
||||
`normalize_image_path` (absolute, normalized, `os.path.normcase`); a row without
|
||||
`file_path` matches by its `image_path`.
|
||||
- Read again: files linked without a pair (the monitor's fallback and FIFO links, the
|
||||
legacy positive-only import), linked to more than one prompt or pair, not linked, a
|
||||
- Read again: files linked without a pair (the monitor's queue and run-key links,
|
||||
fallback links made before #342, the legacy positive-only import), linked to more than one prompt or pair, not linked, a
|
||||
file whose size changed (one replaced at the same path, e.g. a reused ComfyUI
|
||||
counter name, is read and its conflict lands under Needs review), and a row with no
|
||||
stored size (unknown is not known). The scan now stores each file's size when it
|
||||
@@ -2378,7 +2437,8 @@ paints nothing. Until it answers, the section shows "Loading the overview…"; a
|
||||
shows "The overview could not be loaded. Your saved data has not changed." and Retry,
|
||||
never zeros. The thumbnail row is a separate read of `/images/thumbnails/status` (a
|
||||
walk of every source), so the other values never wait for it: "Checking thumbnails…"
|
||||
until it answers. The health check (`POST /prompt_manager/system/health`) is a `read`
|
||||
until it answers. The Backup status row is likewise its own read of
|
||||
`/prompt_manager/backup/status` (S9). The health check (`POST /prompt_manager/system/health`) is a `read`
|
||||
registry job of kind `health`: one at a time, beside a scan, not cancellable and with
|
||||
no file outcomes. The page tracks the started job in the Activity store (its event
|
||||
stream; the store marks it lost if the server stops listing it), never by polling. A press while one runs (another tab) waits
|
||||
@@ -2399,7 +2459,7 @@ the connection's `total_changes` is 0 at close and the data reads back unchanged
|
||||
`-shm` bytes are not compared: every WAL reader rewrites its read marks there, so no
|
||||
reader can keep it byte-identical (`test_health_check_never_writes`).
|
||||
|
||||
Recorded deviations (plan D60–D67, D79–D81 renumbered D92–D99 and D111–D113; D121–D131 assigned):
|
||||
Recorded deviations (plan D60–D75, D79–D82 renumbered D92–D107 and D111–D114; D121–D131 assigned):
|
||||
|
||||
- **D92** (shell): no "Sample configuration" chip. The Save status is truthful: "No
|
||||
unsaved changes.", "Unsaved changes.", "Saved in this browser." / "Saved to the server
|
||||
@@ -2416,7 +2476,10 @@ Recorded deviations (plan D60–D67, D79–D81 renumbered D92–D99 and D111–D
|
||||
`needs_review.pending`) with the concept's line; its Review metadata action opens
|
||||
`#/review` (the concept's `go-review`). A count
|
||||
that is not an integer reads "Ambiguous image links" / "Not available yet." (Q6).
|
||||
"Backup status" reads "No retained backup record." until verified backups exist (S8).
|
||||
"Backup status" says what `GET /prompt_manager/backup/status` reports, in Backup &
|
||||
export's own status line (D103, S9): "Checking backups…" until its own read answers,
|
||||
then the session's last download and the newest safety backup, "No backup recorded."
|
||||
or "The backup status could not be read.".
|
||||
The thumbnail row is "Checking thumbnails…" until its own read answers, then "No
|
||||
missing thumbnails", "N missing thumbnails" (the pending count of
|
||||
`/images/thumbnails/status`, followed by " · M outdated" when previews are in an
|
||||
@@ -2442,7 +2505,8 @@ Recorded deviations (plan D60–D67, D79–D81 renumbered D92–D99 and D111–D
|
||||
available" (an unavailable source), "Not available yet" (Ambiguous metadata, Q6) or
|
||||
"Could not check". A `p.small` after the list says when it was checked and whether
|
||||
every check passed, some could not run, or issues need review. Close / Review missing
|
||||
links (to Maintenance, its hand-off until S7). Nothing is changed by a check.
|
||||
links, which reviews a fresh Missing-file links plan (D100, S7); it never cleans up.
|
||||
Nothing is changed by a check.
|
||||
- **D96** (Statistics): "Usage statistics" without the sample note: the concept's
|
||||
"Runs by category" table (Category / Runs / Avg. rating) with the real runs and
|
||||
average rating of eligible parents per category ("Uncategorized", "Not rated"),
|
||||
@@ -2458,9 +2522,9 @@ Recorded deviations (plan D60–D67, D79–D81 renumbered D92–D99 and D111–D
|
||||
- **D122** (shell): the section nav's `aria-label` is "System sections"; the concept's
|
||||
literal label is its prototype action name "system-tab sections".
|
||||
- **D123** (Maintenance): the concept's Maintenance notice
|
||||
("Check first. Review changes. Then apply." + Run health checks) is delivered ahead of
|
||||
the rest of Maintenance, above its D94 hand-off, as the native place that runs the
|
||||
health check. A press shows "Checking…" (`aria-disabled`) and opens the report; a
|
||||
("Check first. Review changes. Then apply." + Run health checks) was delivered ahead of
|
||||
the rest of Maintenance (S2), as the native place that runs the health check; since S7
|
||||
the section's two surfaces follow it and its D94 hand-off is gone. A press shows "Checking…" (`aria-disabled`) and opens the report; a
|
||||
failed check toasts "The health check could not finish. Nothing was changed."
|
||||
- **D97** (Appearance): added rows for the legacy gallery settings the concept leaves
|
||||
out, after its own rows: "Show image details on cards" (`showImageInfo`; off hides a
|
||||
@@ -2497,7 +2561,7 @@ Recorded deviations (plan D60–D67, D79–D81 renumbered D92–D99 and D111–D
|
||||
dialog, since size and quality change no count. The primary is "Start" (the
|
||||
concept's "Simulate thumbnail job"). Overview's Review thumbnails opens the same
|
||||
dialog (the concept's row starts a sample "Thumbnail check" job directly). Maintenance's
|
||||
Thumbnail cache row takes the dialog with S7. `thumbnail dialog matches the concept
|
||||
Thumbnail cache → Review options opens it too (S7). `thumbnail dialog matches the concept
|
||||
from sources|overview` (system-concept.spec, both themes and sizes) maps the line and
|
||||
"Start"; `thumbnail dialog comparison detects mutations` proves they are the only
|
||||
difference and catches a renamed option, swapped options, swapped actions and a
|
||||
@@ -2519,11 +2583,226 @@ Recorded deviations (plan D60–D67, D79–D81 renumbered D92–D99 and D111–D
|
||||
`tests/js/workspace-activity-kinds.test.js`, and in `workspace-activity.spec.mjs` "a
|
||||
thumbnail job shows Generated Skipped Errors, its pill verb and its toasts" and "a held
|
||||
legacy job and the health check show no tiles, no Results and no made-up percentage".
|
||||
- **D111, S7 kinds**: a maintenance apply (`maintenance`, titled after its operation,
|
||||
not cancellable except "Store each workflow once") reads its title in the pill (no
|
||||
total, so no percentage), counts Changed (green), Protected and Errors, keeps
|
||||
"Maintenance log", and toasts "<title> finished · N changed" or, when it failed,
|
||||
"<title> failed · <the server's status line>" (a plan-changed apply says "The library
|
||||
changed since the review; nothing was changed. Review the plan again."). The duplicate
|
||||
search (`duplicates`) reads "Finding duplicates · N% · a / b", counts Groups found
|
||||
(rows open their first copy), says "Duplicate search started." and "Duplicate search
|
||||
finished · N groups found", with "Search log" and "Cancel search". The duplicate
|
||||
cleanup (`duplicate_cleanup`, not cancellable) reads "Removing duplicate files", counts
|
||||
Removed (green), Kept and Errors (Kept rows open the viewer), and toasts "Duplicate
|
||||
cleanup finished · N removed · M errors" or, when it failed, "Duplicate cleanup failed ·
|
||||
<the server's status line>". A refused start names the job the server named, not the
|
||||
first running one. Every job keeps Download report. Tests: the
|
||||
three S7 cases in `tests/js/workspace-activity-kinds.test.js`.
|
||||
- **D112** (Activity, S5): a legacy-page operation held on the registry (not
|
||||
cancellable, no outcome groups) reads as its server title ("Database restore (admin
|
||||
page)", "Thumbnail clear (gallery page)"), with no tiles, no Results, no Cancel, an
|
||||
indeterminate bar while it runs and "<title> finished." when it ends, whatever its
|
||||
kind (the gallery's clear is a `thumbnails` hold).
|
||||
- **D100** (Maintenance, S7): the concept's `maintenancePlan` dialog ("Review maintenance
|
||||
plan") on the server's read-only plan (`POST /maintenance/preview`), opened by each
|
||||
Review plan and by the health report's Review missing links. No sample note. The `h3`
|
||||
is the operation's row title (the concept prints its `data-plan` name, "duplicate
|
||||
prompts"); the `dl.meta-grid` keeps Scope / Protected / Backup /
|
||||
Media on disk with the plan's values: its scope label, its protected reasons joined
|
||||
with " · " ("Nothing is protected" when none), "Create and verify before changes
|
||||
(<db>.backup_<date and time>)" or "No safety backup (the policy is off)", and its
|
||||
media line. (A `workflow_storage` plan would read "Store each workflow once" and
|
||||
"Always created and verified first (…)", but Maintenance has no row for it: the
|
||||
concept has none, so the migration is reviewed and applied through the API and only
|
||||
its jobs appear, in Activity.) A
|
||||
`details` "Listed records (N)" ("(N of M)" past the server's 50) holds the records the
|
||||
plan names in a `pre`. The `p.small` says "Nothing changes until you apply. This
|
||||
review holds for N minutes." (from `expires_at`), plus "Not enough free disk space:
|
||||
about X MB is needed and Y MB is free." when the plan's disk check is short. Then the
|
||||
D55 backup box ("Back up the database first (recommended)", "Back up now") and Cancel /
|
||||
"Apply reviewed operation" (the concept's "Simulate reviewed operation"). A plan with
|
||||
nothing to do (scope 0, "No duplicate prompt text") shows its rows, "There is nothing to
|
||||
apply; nothing was changed." and only Close, never an Apply that would start an empty
|
||||
job (`a plan with nothing to do shows only Close, never Apply`). Apply sends
|
||||
only the token; the dialog closes, "<title> started." is toasted and the job opens in
|
||||
Activity. A refused apply stays and says why: "Another job is running: <title>."
|
||||
(D57), "This plan was already applied: <job>." (`plan-used`, linking the job), the
|
||||
server's line for an expired or replaced review (then the plan is reviewed again at
|
||||
once), for a short disk, for nothing to do and for the server's own pre-start failure
|
||||
(`apply-failed`). Only our server's refusal is definitive: a known code, or a 4xx whose
|
||||
body says `success:false`. A dropped connection, a proxy's 5xx (no code) or a 2xx whose
|
||||
body cannot be read may follow a started job, so they say "Outcome not confirmed. Check
|
||||
Activity before applying it again." and read the Activity list again
|
||||
(`lost apply response…` runs the apply and then drops its answer; `a proxy error or an
|
||||
unreadable answer is not a refusal` answers a 504 after the server started the job). A
|
||||
review button holds ("Reviewing…", `aria-disabled`) while its preview is read, so a
|
||||
double press sends one preview (`a review button pressed twice sends one preview`). A
|
||||
failed apply records its code as the job's `scope.failure`; when the plan was overtaken
|
||||
(`plan-changed`, `storage-changed`, `plan-expired`), its Activity view offers "Review
|
||||
again", which reviews the same operation afresh (a duplicate cleanup whose sources
|
||||
changed goes back to Maintenance to search again).
|
||||
Cancel and Escape write nothing (`Cancel writes nothing`: a read-only dump of the rows
|
||||
and the media hashes, `GET /__test__/system/digest`). `plan dialogs match the concept`
|
||||
(system-maintenance-concept.spec, both themes and sizes) compares each of the four
|
||||
operations' dialogs with the concept's, mapped by these values only.
|
||||
- **D101** (Maintenance, S7): Duplicate image files → Find duplicates starts the
|
||||
duplicate search (Activity, D111); its job view adds "Review groups" after Download
|
||||
report once it finished with groups. The concept's `duplicates` dialog ("Review
|
||||
duplicate image files") shows the real groups ten at a time: a `p.small[role=status]`
|
||||
"Groups 1–10 of N", then per group a `p.small` "N identical files · S each" and one
|
||||
`.compare-grid` with a `.compare-item` per copy (its preview as the concept's
|
||||
`img.sample-fluid`, and a radio "Keep <source>/<path>", the oldest copy checked, its
|
||||
label wrapping anywhere (CSS under D101) so long dataset paths never make the review
|
||||
scroll sideways (`long copy paths wrap; the duplicates review never scrolls sideways`,
|
||||
desktop and phone); a copy
|
||||
gone since the search is disabled and says so), "Previous groups" / "Next groups" when
|
||||
there are more than ten, the `p.small` "Removing a copy deletes its file from disk;
|
||||
its database links move to the kept copy. History keeps the removed paths in its run
|
||||
records. Nothing is deleted until you review and apply the cleanup.", and Cancel /
|
||||
"Review duplicate cleanup". Choices survive paging; the review sends every group shown
|
||||
so far as `{group_id, root_index, path}` (never a bare path: two sources may hold a
|
||||
file at the same path) and opens the D100 plan for "Duplicate image files". The
|
||||
primary stays the concept's plain button; the plan it opens has the primary.
|
||||
`duplicates dialog matches the concept` and `duplicates comparison detects mutations`
|
||||
(system-maintenance-concept.spec) pin the mapping.
|
||||
- **D102** (Maintenance, S7, Q5, #222): Metadata cache clears only the browser entries
|
||||
Prompt Manager owns as a cache (`web/js/cache-ownership.js`). The inventory of every
|
||||
storage writer has none (see "Browser cache ownership" below), so the row's line adds
|
||||
"Nothing is cached in this browser." and its Review plan is `aria-disabled` and opens
|
||||
nothing; a browser that refused to list its storage says "This browser did not allow
|
||||
checking its storage." instead. With owned entries (a test names one through the module's inventory), Review
|
||||
plan opens "Review maintenance plan" › "Metadata cache" (Scope "N cached entries in
|
||||
this browser", Protected "Preferences and other tools’ data in this browser", Backup
|
||||
"Not needed: the database is not changed", Media "No files are removed") and "Clear
|
||||
browser cache" clears exactly those and toasts what it cleared. The row's dimmed button
|
||||
is D93's style. `maintenance matches the concept` (both themes and sizes) maps only this
|
||||
row; `maintenance comparison detects mutations` proves it is the only difference and
|
||||
catches a removed row, a renamed operation, a lost danger mark and a swapped token.
|
||||
- **D103** (Backup & export, S9): "Back up" (the concept's "Simulate backup") downloads
|
||||
the verified copy of `GET /prompt_manager/backup` through the api client (the same
|
||||
`downloadDatabaseBackup` as every backup box; a copy whose `X-PromptManager-Verify` is
|
||||
not `ok` toasts "Database backup downloaded, but it did not pass verification:
|
||||
<reason>"), then reads the status again. The `p.small#backup-status` under the
|
||||
actions shows only what `GET /prompt_manager/backup/status` reports: "Downloaded
|
||||
backup · <date, time> · verified." (or "not verified") for this session's last
|
||||
download and "Safety backup · <date, time> · N on disk." for the newest
|
||||
`<db>.backup_*`, else "No backup recorded."; "Checking backups…" while read and "The
|
||||
backup status could not be read." when it failed (never "no backup"). Portable
|
||||
export's "Export JSON" (the concept's "Export sample JSON") downloads the Library
|
||||
document (`/library/export`); a second `.row-actions.spaced` adds "Export all prompts
|
||||
JSON" (`/export`, the legacy admin page's Export of every prompt) and "Export saved pairs JSON"
|
||||
(`/pairs/export`), plain buttons as the concept's; a failed export toasts "The export
|
||||
could not be downloaded. Nothing was saved to this device." "Import / inspect file"
|
||||
opens the shared "Import & inspect" (D15, D55), as Images does.
|
||||
- **D104** (Backup & export, S9, Q4): Recovery policy keeps the concept's "Proposed" chip.
|
||||
"Create a safety backup before maintenance" is the server's `safety_backup`
|
||||
(`maintenance.safety_backup`) policy. Its line says what it does ("Keep a verified copy
|
||||
of the database on disk before maintenance, cleanups and LoRA imports change records. A
|
||||
restore always keeps one."); the concept's "Preview a verified backup before applying
|
||||
operations that change records." describes nothing the server does. It is drafted behind the Save row and saved with
|
||||
`expected_revision` (a configuration changed elsewhere is the conflict status); it is
|
||||
disabled with "Server settings could not be read." when the read failed. "Scheduled
|
||||
database backups" offers Manual only; Daily and Weekly stay listed but disabled
|
||||
(there is no scheduler), the row's line is "Daily and Weekly are not available yet:
|
||||
back up with Back up whenever you choose." (the concept's proposal line is
|
||||
prototype copy) and nothing is drafted for it. The warning note reads "Restores are
|
||||
staged, migrated and validated before replacing live data."
|
||||
- **D105** (Backup & export, S9): "Review restore…" picks a file (no type filter: the
|
||||
safety backups `<db>.backup_<time>` have no `.db` extension; the server validates) and stages it
|
||||
(`POST /prompt_manager/restore/preview`, multipart through `fetchUrl`; the button reads
|
||||
"Reviewing…" meanwhile). "Restore preview" is the shared review: no sample note; the
|
||||
concept's `dl.meta-grid` Snapshot (file name · size · date), Integrity ("Verified" plus
|
||||
the stage's document repairs), Migration ("Current schema" / "Upgraded to the current
|
||||
schema"), Safety backup ("Created before replacing (<label>)": the restore always
|
||||
keeps one; the Snapshot size is whole KB below 1 MB), then "In the backup" and "In the library now" (prompts, image links, saved
|
||||
pairs, runs); a `p.small` "Restore replaces the whole library with this backup and
|
||||
runs alone in Activity. Nothing changes until you restore. This review holds for N
|
||||
minutes."; the D55 backup box ("Downloads a copy of the prompts database before the
|
||||
restore replaces it."); Cancel / "Restore" (the concept's "Simulate restore", still
|
||||
`danger`). Leaving it any way but Restore (Cancel, ×, Escape, another dialog, the page
|
||||
leaving) discards the stage (`DELETE …/preview/<token>`). Restore starts the "Database
|
||||
restore" Activity job (exclusive, not cancellable, indeterminate; its descriptor ends
|
||||
with the server's line) and opens its view; a review the server asks to redo (expired,
|
||||
library changed) stages the same file again. A Restore refused as `busy` (another job
|
||||
runs; the server keeps the token) opens the same review again with "Discard" / "Restore":
|
||||
Restore retries, Discard (or leaving) releases it. A job whose leases did not drain
|
||||
hands its review back (`scope.review_kept`, "…The review was reopened in the page that
|
||||
started the restore: restore it again there, or discard it."): the page that started it
|
||||
opens the review again over the page shown now, with Discard / Restore. Every other path
|
||||
that abandons a token after Restore (a used, expired or replaced review, a lost answer)
|
||||
sends its DELETE, which changes nothing for a token the server dropped or a job took. A
|
||||
third review while two are held elsewhere toasts "Two restores are already being
|
||||
reviewed in another window. Discard one there, or try again when it expires (15 minutes
|
||||
after it was reviewed)." An upload whose answer is lost because the section was left
|
||||
mid-response is not discarded by the page: the server's 15-minute expiry removes it. When a restore the page saw running
|
||||
finishes, the cached Images and Library visits are dropped, the sidebar counts and
|
||||
saved views are read again and the open route is shown again (the Activity job view
|
||||
stays). A refused file toasts "This file can't be restored: <validator reason>.". A
|
||||
review value may wrap anywhere (`#modal-body > dl.meta-grid dd`, CSS under D105), so a
|
||||
long file name keeps the concept's two equal columns at phone width.
|
||||
- **D106** (Integrations, S10, Q8): the send bridge always asks for the ComfyUI window
|
||||
and node, and the nodes always follow ComfyUI's own renderer, so "Default send
|
||||
destination" offers only "Ask each time" and "Host compatibility" only "Follow host
|
||||
renderer"; the concept's other options stay listed but disabled, each row says why in a
|
||||
`p.small` under its description, and nothing is stored for them. "Return to canvas
|
||||
after send" is off and disabled with "Not available: sending applies the text in
|
||||
ComfyUI and keeps this page open." Host compatibility adds a `p.small[role=status]`
|
||||
with the renderer the open ComfyUI windows report over the bridge (`renderer`:
|
||||
`classic` or `nodes2` on each Prompt Manager destination,
|
||||
`web/js/prompt-bridge-adapter.js`; `#renderer-status`): "Detected: Nodes 2.0 (1 ComfyUI window).", "…
|
||||
Classic and Nodes 2.0 (2 ComfyUI windows).", "ComfyUI is open; add a Prompt Manager
|
||||
node to see which renderer it uses." or "No ComfyUI window is open in this browser, so
|
||||
its renderer is not known."; it looks again when the page becomes visible, and only the
|
||||
newest check paints. The two selects never shrink for the reason lines and, at phone
|
||||
width, may grow to 160px (the concept caps them at 120px and cuts "Follow host
|
||||
renderer"), so each shows its chosen option whole (`selectsFit` in the comparison). "LoraManager
|
||||
integration" replaces the concept's sample line with the detection ("Detected:
|
||||
custom_nodes/ComfyUI-Lora-Manager." / "Not detected in ComfyUI’s custom_nodes
|
||||
folders." / "Found at …, but outside ComfyUI’s custom_nodes folders, so it can’t be
|
||||
enabled." for a linked install the server would refuse, from `/lora/status`'s
|
||||
`detected_usable`); the legacy read-only "LoraManager path" row follows it; "Automatically
|
||||
include trigger words" adds "Trigger words are (not) loaded." (the trigger cache, a
|
||||
separate fact). The notes read as the row description (`.setting-row p`). Pinned by
|
||||
`integrations matches the concept` (both themes and sizes) and its mutation check
|
||||
(system-integrations-concept.spec), the D106 tests of
|
||||
workspace-system-integrations.spec, and `each destination reports its window
|
||||
renderer, and System › Integrations shows it (S10)` on the real host
|
||||
(tests/comfyui/prompt-bridge.spec).
|
||||
- **D107** (Integrations, S10): "LoRA reference data" reads "Review import" (the
|
||||
concept's button only simulated a job) and opens "Review LoRA import" in the concept's
|
||||
record-changing review (its `maintenancePlan` shell): `h3` "LoRA reference data", the
|
||||
`dl.meta-grid` Scope ("Replace N imported prompts with M LoRAs"), Found ("N metadata
|
||||
files · M readable"), Protected (the saved pairs that block it), Network ("Downloads
|
||||
up to N example images from CivitAI[ with your saved key]" / "No downloads"), Backup
|
||||
and Media on disk, a `details` "LoRAs found (N)", the hold line plus "Example images
|
||||
downloaded before a failure stay in the image cache." in the `p.small`, the D55 backup
|
||||
box and Cancel / "Start import" (`danger` when it would remove every import). A review
|
||||
that saved pairs block, or with nothing to import, shows its rows and only Close.
|
||||
Start runs the "LoRA reference import" Activity job (D111 descriptor: Imported, Linked
|
||||
to existing, Skipped, Images downloaded, Download errors, Errors; Cancel only while it
|
||||
downloads; a failure ends with the server's reason). Pinned by `LoRA import review
|
||||
matches the concept` and its mutation check (system-integrations-concept.spec) and
|
||||
the import tests of workspace-system-integrations.spec.
|
||||
- **D114** (Integrations, S10): the CivitAI key field is never filled in (the page never
|
||||
receives the key). Typing drafts a new key, a blank field keeps the saved one (no
|
||||
draft), and "Clear saved key" (shown only while a key is saved) drafts its removal.
|
||||
A `p.small#civitai-key-status[role=status]` (it also describes the field) says "A key
|
||||
is saved." / "No key saved." / "The new key replaces the saved one when you save." /
|
||||
"The saved key is removed when you save."; pressing "Clear saved key" keeps the
|
||||
keyboard on the key field, and the line is rewritten only when its text changes. A typed
|
||||
key is dropped when the section is left (the field comes back empty); leaving without one
|
||||
keeps the last Save's outcome, so Sources still marks a refused source. The field is `autocomplete=new-password`, so browsers never offer it
|
||||
as a login; the concept's
|
||||
prototype warning is dropped. The LoRA rows save through the one Save row:
|
||||
`POST /lora/enable` with `expected_revision` (a config.json changed elsewhere is
|
||||
`settings-changed`), then `/settings` for any other host change with the revision the
|
||||
first save returned (`writeHostInParts`). When only one part is written, the store keeps
|
||||
what saved, adopts its revision and highlights only the rest: the status is the
|
||||
server's reason (an unusable source) or "Some changes were saved to the server
|
||||
configuration; the highlighted ones were not.", and the next Save needs no conflict
|
||||
(`a save where only the LoRA part is written says so; the next save needs no conflict`). Pinned by `key never prefilled; blank keeps; Clear saved key
|
||||
clears; saved through Save preferences (D114)` and `a LoRA save over settings changed
|
||||
elsewhere is a conflict, never a blind write` (workspace-system-integrations.spec).
|
||||
- **D113** (Sources): the Worker threads range ends at the server's `cpu_count` (the
|
||||
concept's fixed `max=16`) and shows the server's real `worker_threads` (the concept's
|
||||
sample 4).
|
||||
@@ -2961,6 +3240,103 @@ Restoring a deduped backup on a version before this one is not supported: that
|
||||
version's restore accepts the file and then reads rewritten images without their
|
||||
workflow (README changelog).
|
||||
|
||||
### LoRA reference import contract (#221, #249 System S10)
|
||||
|
||||
The LoRA import is staged (`py/lora_import.py`): discover the LoraManager
|
||||
`*.metadata.json` files and fingerprint them (path, size, modification time); stage each
|
||||
record (CivitAI example images go into the image cache first, best effort, counted in
|
||||
`downloaded` / `download_errors`; each image is written to a temp file beside its cache
|
||||
entry and moved in place, so a half-written image is never seen as cached); the files are
|
||||
listed and fingerprinted again (a file changed, removed or added while staging aborts with
|
||||
nothing written); then `PromptDatabase.replace_category_prompts`
|
||||
(`database/category_import.py`) replaces the `lora-manager` category in one
|
||||
`BEGIN IMMEDIATE` transaction. Any failure leaves the previous prompts, tags and image
|
||||
links exactly as they were; images already downloaded stay in the cache and the job says
|
||||
so. A record whose preparation fails is skipped (legacy); a prompt a saved pair uses
|
||||
blocks the whole replacement (`PairDependencyError`, nothing deleted); a replacement with
|
||||
no record over a non-empty collection needs `allow_empty` (a reviewed import), so the
|
||||
legacy scan refuses it.
|
||||
|
||||
`POST /prompt_manager/lora/import/preview` (body `{}`) writes nothing and requests
|
||||
nothing (`test_preview_writes_nothing_and_makes_no_requests`: DB and `-wal` bytes, `-shm`
|
||||
size, rows, no cache folder, the downloader never called) and answers the maintenance
|
||||
`Plan` shape plus `lora_files`, `with_metadata`, `current_imports`, `blocked_by_pairs`,
|
||||
`planned_downloads` and `network`; a blocked plan has no token. `POST
|
||||
/prompt_manager/lora/import/apply {token}` starts the `lora_import` job (202): title "LoRA
|
||||
reference import", admission `records`, cancellable only while it stages, the safety
|
||||
backup the policy promises before the replace. Refusals: `plan-changed` (409, the files
|
||||
changed since the review, or the integration was turned off or moved to another install),
|
||||
`plan-expired`, `plan-used`, `storage-changed`, `busy`,
|
||||
`insufficient-space`, `apply-failed`. The job fails with `plan-changed` (the files or the
|
||||
imports changed meanwhile), `blocked-by-pairs`, `nothing-to-import`, `backup-failed` or
|
||||
`import-failed`, each with nothing written; the job keeps what staging skipped, by name,
|
||||
whatever happened after. The tests change real files behind a held download
|
||||
(`test_files_changed_while_staging_fail_the_job`, removed and added files too). The
|
||||
replace binds the category, not one parameter per prompt, so SQLite's variable limit
|
||||
never caps a collection. Cached CivitAI images are written to a `.pm-*.part` file in the
|
||||
same folder and renamed into place, with the umask's mode as a plain save gives; each import
|
||||
first removes `.pm-*.part` files older than an hour from the image cache and its LoRA
|
||||
folders (`TestStaleDownloadSweep`). A download starts only at an HTTPS `civitai.com`,
|
||||
`civitai.red` or `civitai.green` host from the fixed `CIVITAI_HOSTS` list (the key goes
|
||||
with that first request only); a redirect is followed only over HTTPS to one of those
|
||||
domains or a subdomain (parsed host name, default port, no credentials), at most three
|
||||
hops, without the key (`TestCivitaiRedirects`, `TestCivitaiRedAndGreenDomains`). Each
|
||||
failed download logs one warning naming the URL without its query and a short reason
|
||||
(HTTP status, refused redirect, hop limit, size cap, decode or save error, timeout or
|
||||
network error), never the key (`TestDownloadFailureWarnings`). The legacy `POST /lora/scan` runs the same stages under its SSE
|
||||
event shape; a page that closes its connection stops staging before its next LoRA, and the
|
||||
scan keeps its records hold until staging has stopped, then ends `cancelled` in Activity
|
||||
(`test_a_disconnected_legacy_scan_stops_and_holds_until_it_stopped`) (`test_legacy_scan_events_unchanged_on_success`), holds
|
||||
the registry as "LoRA import (admin page)" and takes no safety backup. Both imports are
|
||||
`records` jobs, so a restore (`exclusive`, #249 S8) is refused while one runs and they are
|
||||
refused while a restore runs; the review is a request lease ("LoRA import review"), refused
|
||||
(409 `busy`) during a restore, and a restore waits for it (`LoraImportRestoreTests`). `POST /lora/enable`
|
||||
accepts the optional `expected_revision`. LoraManager detection also searches every
|
||||
`custom_nodes` folder ComfyUI registers through `folder_paths` (#144), main folder first.
|
||||
|
||||
`tests/browser/workspace_integrations_fixture.py` (`/__test__/integrations/*`) makes a fake
|
||||
LoraManager install and LoRA folder under the server's temp root, holds or fails the
|
||||
import, saves a blocking pair and answers CivitAI downloads with a stub opener, so no
|
||||
test reaches the network.
|
||||
|
||||
```sh
|
||||
python3 -m unittest tests.test_lora_import tests.test_api_lora tests.test_lora_database \
|
||||
tests.test_lora_utils -v
|
||||
node --test tests/js/workspace-system-integrations.test.js tests/js/prompt-bridge-adapter.test.js
|
||||
npm run --prefix tools/vendor test:workflows -- workspace-system-integrations.spec.mjs \
|
||||
system-integrations-concept.spec.mjs \
|
||||
--project chromium --project touch --project firefox --project webkit
|
||||
```
|
||||
|
||||
### Browser cache ownership (#222)
|
||||
|
||||
The legacy gallery's Clear cache used to delete every localStorage key starting `pm_` or
|
||||
`promptManager_` (the workspace theme and the dismissed update notice among them) and
|
||||
every CacheStorage cache whose name held "gallery" or "thumbnail" (another tool's, on
|
||||
ComfyUI's shared origin). `web/js/cache-ownership.js` names what a clear may remove
|
||||
from an inventory of the real writers (2026-10-24): every key a Prompt Manager page
|
||||
writes is a preference (`gallerySettings`, `pm_workspace_theme`,
|
||||
`pm_update_notice_dismissed`) and no page opens CacheStorage or IndexedDB (the
|
||||
`web/sw.js` service worker that did never shipped). So nothing is removed, and the
|
||||
toast says "Nothing of Prompt Manager's is cached in this browser." `the inventory covers
|
||||
every browser storage writer in web/` (`tests/js/cache-ownership.test.js`) fails when a
|
||||
page writes a key that is neither a preference nor inventoried, writes one it cannot name
|
||||
(a computed `setItem` key, a bracket or property assignment on a Storage), or opens
|
||||
CacheStorage or IndexedDB; `the storage scan reports every way of writing browser
|
||||
storage` pins those patterns. It skips `web/lib` (third-party builds) and the pinned
|
||||
Toolkit build (`web/js/shared/toolkit`, never hand-edited, verified by `npm run check`;
|
||||
its storage belongs to the Toolkit repository and a new pin is reviewed on its own). The
|
||||
gallery's server thumbnail clear stays its own request and runs first: refused or
|
||||
unreachable, it changes nothing and says so (the old code toasted success when the
|
||||
request failed); refused browser storage is reported, never hidden
|
||||
(`tests/js/legacy-refusals.test.js`; the browser case `the gallery cache clear keeps
|
||||
preferences and other tools caches` in `workflows.spec.mjs`).
|
||||
|
||||
The legacy duplicates dialog's View Full now names its source (`?root=<index>`, with the
|
||||
path URL-encoded) when the gallery's output folder is a listed source; before, the serve
|
||||
route opened the first source holding a file at that path
|
||||
(`test_view_full_names_the_gallery_folder_source`).
|
||||
|
||||
### Library location contract (#249 System S6b)
|
||||
|
||||
`utils/library_location.py` is the one resolver of every data path: the database
|
||||
@@ -3035,14 +3411,14 @@ generation did not change meanwhile (`PromptModel.get_connection`).
|
||||
Writes racing the switch: a background write that was waiting on the old file's lock
|
||||
when the switch closed its connection fails, and run counting (`count_runs`,
|
||||
`count_pair_runs`) and the image monitor's link (`ImageGenerationHandler.
|
||||
link_image_to_prompt`: the resolved-pair ingest and the fallback or recent-prompt link)
|
||||
link_image_to_prompt`: the resolved-pair ingest and the plain prompt link)
|
||||
run it once more through `PromptDatabase.survive_library_switch` when the model's
|
||||
generation changed during the attempt; the retry opens the new library. The monitor
|
||||
links through `PromptDatabase.link_image_to_prompt_or_raise`, which raises database
|
||||
errors (`link_image_to_prompt` keeps its log-and-return-0 contract for other callers);
|
||||
each attempt is one transaction, so a failed attempt rolls back and a committed one
|
||||
does not retry, and the image is linked once. `TestBackgroundWritesSurviveTheSwitch`
|
||||
races a real UPDATE (run counting) and a real fallback image link against the lock and
|
||||
races a real UPDATE (run counting) and a real plain image link against the lock and
|
||||
asserts each lands in the new file only. Any other failure is not retried. Known limit: that
|
||||
blocked writer, and the mover with it, waits out the writer's busy timeout (30 s)
|
||||
before its connection closes: SQLite holds the connection's mutex during the busy
|
||||
@@ -3071,6 +3447,95 @@ are closed before `os.replace`; renames stay inside one folder; folder identity
|
||||
folder are refused. The browser test server closes its own log handlers before its temp
|
||||
root (which now holds the logs) is removed.
|
||||
|
||||
### Backup and restore contract (#249 System S8)
|
||||
|
||||
A restore is staged, then published (`database/restore_stage.py`). The upload is
|
||||
streamed to `.pm-restore-upload-*.tmp` and staged in a `.pm-restore-*` folder, both in
|
||||
the library's own folder (never the system temp folder); free space there must hold
|
||||
the upload three times (upload, stage, WAL growth) plus the current database
|
||||
(`restore_room`). The stage checks the upload without reading every workflow document,
|
||||
copies it with the backup API (rollback-journal and WAL files alike), migrates an older
|
||||
schema, repairs damaged workflow documents (read one row at a time: a body that inflates
|
||||
completely while only its recorded hash or only its length disagrees is re-keyed or merged
|
||||
into the document holding that text; any other damaged body is re-stored from an inline
|
||||
copy with the same SHA-256, else dropped with its images' references cleared; repaired,
|
||||
cleared and distinct images are counted), rotates the
|
||||
history epoch, makes the copy standalone, verifies it with every document read and
|
||||
records its SHA-256. Publishing rechecks that digest (`StageChanged`), keeps the
|
||||
current database as `<db>.backup_<YYYYmmdd_HHMMSS_ffffff>[_n]` (`database/safety_backup.py`:
|
||||
standalone, fsynced, 0600, not verified because the database being replaced may be the
|
||||
damaged one), copies the stage over the live pages through the backup API (#226: no
|
||||
file copy) and checkpoints the WAL with TRUNCATE, waiting up to 30 s for readers and
|
||||
logging one that stays (`tests/test_restore_stage.py`).
|
||||
|
||||
`POST /prompt_manager/restore/preview` answers `{token, expires_at, summary, schema,
|
||||
integrity, documents, live, size, safety_backup}` and writes nothing live; at most two
|
||||
stages are held (429 `too-large`) for 15 minutes, and a timer removes an expired stage
|
||||
without another request; `DELETE .../preview/{token}` removes one. A preview that is
|
||||
refused, fails or is cancelled (even while the worker still stages) removes its stage. The
|
||||
store of reviewed previews changes only on the event loop; stage folders are removed in
|
||||
the executor. `POST /prompt_manager/restore/apply {token}` rechecks the live fingerprint (409
|
||||
`plan-changed`, stage dropped) and starts "Database restore": kind `restore`, admission
|
||||
`exclusive` (it runs alone, like Move library and Optimize), not cancellable,
|
||||
indeterminate. Its runner waits for request leases (when they outlast the wait the job
|
||||
fails `busy` and the review is kept under its token, flagged `scope.review_kept` for the
|
||||
page that started it), rechecks the fingerprint,
|
||||
publishes, swaps `self.db`, prunes the pool and logs the safety backup by name, any
|
||||
document repair and the restored counts; a failed page copy keeps the safety backup and
|
||||
says so. Once the copy replaced the library the restore succeeded: pruning, reopening
|
||||
`self.db`, dropping the old library's previews and closing the previous object only log
|
||||
their failures, in the job and the legacy route alike. The fingerprint covers row counts
|
||||
and the newest row and edit of prompts, images, pairs, runs and saved views, not every
|
||||
edit (a tag or favorite change on an older row); those survive in the safety backup. Tokens are single use (409 `plan-used` with the job), expire (410
|
||||
`plan-expired`) and are bound to the database identity (409 `storage-changed`). The
|
||||
legacy `/restore` runs the same stage and publish under the same `exclusive` kind with
|
||||
its old response; a repair adds `workflow_documents` and a sentence to `message`
|
||||
(`tests/test_restore_plans.py`, `tests/test_api_backup_restore.py`).
|
||||
|
||||
Short request-scoped work takes a registry lease (`JobRegistry.lease`): a Needs review
|
||||
decision, a maintenance preview, the restore preview's read of the live library (not its
|
||||
upload or staging) and the apply's check. A lease is not
|
||||
listed, never refuses another lease or a job, and is refused (409 `busy`) while an
|
||||
`exclusive` job runs; the restore waits for running leases before it switches the file
|
||||
(`tests/test_job_leases.py`). `GET /prompt_manager/backup` holds `backup` ("Database
|
||||
backup", `guarded`) only while it writes a standalone copy beside the database, so a
|
||||
stalled download never blocks other work. A copy that is not SQLite or fails `PRAGMA
|
||||
integrity_check` is a 500 with no bytes. A copy that fails only a semantic check
|
||||
(`verify_database_file`, workflow documents not read) is still sent, so a user can always
|
||||
get a backup: the reason is logged, sent as `X-PromptManager-Verify` (`ok` otherwise) and
|
||||
recorded as `verified:false`. The file is streamed outside the hold, then closed and
|
||||
removed. `GET
|
||||
/prompt_manager/backup/status` lists up to ten `<db>.backup_*` files newest first (name,
|
||||
bytes, modified time; no paths) and this session's last verified download, or null.
|
||||
|
||||
System › Backup & export (`system-backups.mjs`, S9) is the concept's "backups" body on
|
||||
these routes (D103–D105 above); Overview's Backup status row reads the same status. The
|
||||
admin page's Backup and Restore are unchanged. `tests/browser/system-backups-concept.spec.mjs`
|
||||
compares the section, the Restore preview and the Import & inspect dialog with the
|
||||
concept (both themes, 1600×1100 and 390×844), mapping only D103–D105 (D15 and D55 for the
|
||||
shared Import dialog); unmapped, exactly those paths differ, and `backup & export
|
||||
comparison detects mutations` renames a label, removes a row, drops a danger mark and
|
||||
swaps tokens. `tests/browser/workspace-system-backups.spec.mjs` covers Back up and its
|
||||
revoked object URL, the status line against the server (none, a safety backup, a failed
|
||||
read), Overview's row, the restore review against the preview (and that it writes
|
||||
nothing), Cancel and Escape discarding the stage, a restore through Activity that
|
||||
reloads the counts and the open Library, a busy refusal reopened and retried, two
|
||||
refusals then a third review, ×, a section and a route leave discarding, a safety-backup
|
||||
file name, an invalid file, the three exports and a
|
||||
failed one, Import / inspect (Rescan while running, Scan for metadata, Inspect), the
|
||||
safety-backup policy with its revision and a conflict, the schedule, a failed settings
|
||||
read and the legacy routes. The fixture (`workspace_system_fixture.py`) writes a safety
|
||||
backup (`/__test__/system/safety-backup`), lists the backups and held stages
|
||||
(`GET /__test__/system/backups`), and its reset drops held reviews, removes the safety
|
||||
backups and forgets the last download.
|
||||
|
||||
```sh
|
||||
node --test tests/js/workspace-system-backups.test.js tests/js/workspace-system-overview.test.js
|
||||
npm run --prefix tools/vendor test:workflows -- workspace-system-backups.spec.mjs \
|
||||
system-backups-concept.spec.mjs \
|
||||
--project chromium --project touch --project firefox --project webkit
|
||||
```
|
||||
|
||||
## Workspace data fields for the concept pages (#249)
|
||||
|
||||
These fields are additive. The legacy `/prompt_manager/images/output` listing (no
|
||||
@@ -3552,10 +4017,191 @@ Viewer, including nested modal focus, mobile layouts and per-role sending.
|
||||
real Classic and Nodes 2.0 widgets and verifies workflow serialization without
|
||||
loading a workflow or queueing generation.
|
||||
|
||||
## Image attribution: certain or unlinked (#342)
|
||||
|
||||
The image monitor and the output scan link an output only to a prompt that
|
||||
provably produced it; anything uncertain stays unlinked and the monitor logs it
|
||||
at info ("No certain prompt for <file>; left unlinked"). There is no recency
|
||||
fallback (`_get_fallback_prompt` is gone) and no unproven tracker entry.
|
||||
|
||||
- **Empty editor.** Prepend and append text are never stored as the prompt
|
||||
(maintainer's product decision; #343 tracks letting a link drive `text`). A
|
||||
PromptManager, Text, Focused, Split or Pair node whose `text` is empty saves,
|
||||
counts and tracks nothing for the positive side, logs at info (no "Could not
|
||||
save or track" warning), and marks its run key empty, so no earlier entry of
|
||||
that key can be used and a batch item keeps its queue place
|
||||
(`PromptTracker.mark_empty`). The queue hook counts nothing for it either. A
|
||||
save or tracking failure of a positive node marks its run key the same way, so
|
||||
an older entry of that key never stands in for the failed run.
|
||||
- **Run keys** (`utils/run_keys.py`): sha256 of a prompt node plus its whole
|
||||
upstream closure (`class_type` and `inputs`) in the API graph. Nodes register
|
||||
tracker entries under it; the monitor computes it from the image's embedded
|
||||
graph. The snapshot taken when the file appears names the key's latest run; a
|
||||
queued entry is taken only by key and from that run (batch items in order, a
|
||||
placeholder leaves its image unlinked, same-key entries of earlier runs ahead
|
||||
of it are stale and dropped, entries of later runs that executed during the
|
||||
monitor's delay wait for their own images). Without a queued entry the
|
||||
snapshot's entry itself is used only when its run id is known
|
||||
(`comfy_execution.utils.get_executing_context`) and that run registered one
|
||||
entry for the key, which covers a cached re-run (sampler seed changed) without
|
||||
applying a batch's last item to every image. When the key's latest entry is a
|
||||
placeholder the image stays unlinked without consulting history (the same
|
||||
rule as a popped placeholder). More than one positive PromptManager node in
|
||||
the image graph, or any positive node without a key: never linked from the
|
||||
tracker. Links are followed exactly as ComfyUI's `is_link` defines them (a
|
||||
`str` node id and an `int` or `float` slot in a two-item list). Equal keys do
|
||||
not always mean equal text: the key can't see `IS_CHANGED`, so an upstream
|
||||
node that forces re-runs gives two runs one key; that is why the run named by
|
||||
the snapshot is required as well. A file event delivered after the next run
|
||||
already re-registered the key is the one remaining exposure (see the ledger).
|
||||
- **Recorded history** (`database/run_attribution.py`): a file is attributed when
|
||||
exactly one recorded run reported its output location (root key, subfolder,
|
||||
name), its mtime lies within that run (dequeue to terminal or callback, +-2 s),
|
||||
and every positive context names the same stored prompt (the live
|
||||
`positive_prompt_id` hint). The monitor tries it last; the scan tries it when the
|
||||
#170 browser-parser floor finds nothing, and also for files with no prompt
|
||||
metadata at all (`--disable-metadata`, savers that embed none); it never
|
||||
creates a prompt. Lookups are batched (`stored_run_prompts`): one
|
||||
parameterized `filename IN (...)` query per chunk of at most 500 names
|
||||
(`LOOKUP_CHUNK`), matched in memory, so a scan batch asks history once per
|
||||
chunk, never once per file. The history schema is unchanged (no extra index),
|
||||
so older builds still open the library.
|
||||
- **After the run ends.** An image with no run keys (no embedded API graph) that
|
||||
nothing identifies waits for its run: when the run's outputs are recorded
|
||||
(`write_execution_outputs_for_finish` notifies `subscribe_outputs_sealed`
|
||||
listeners after commit), each waiting image the run reported gets exactly one
|
||||
history-only re-check on the monitor's worker. At most 1000 images wait, each
|
||||
for at most an hour; shutdown forgets them. The image is registered before the
|
||||
first history check, so a run sealed in between is not missed; a re-check that
|
||||
can't be scheduled yet (the image's own pass still pending, or the pending set
|
||||
full) is kept due and scheduled when a pending image finishes. A keyless image
|
||||
outside ComfyUI's output folder never waits (history can't attribute it).
|
||||
- **Spellings.** History keeps ComfyUI's own spelling of its output folder (the
|
||||
root key). A file is matched through that spelling first, then with both sides
|
||||
resolved (`realpath`), so an output folder reached through a symlink or a
|
||||
Windows junction still attributes the scan's resolved file paths; waiting
|
||||
files are keyed by their resolved path for the same reason. The scan makes
|
||||
its batch's history lookup only when the first file in the batch needs it.
|
||||
- **Blank positive.** A resolved pair whose stored positive is empty (the #342
|
||||
image with its Pair execution record) is `unresolved` for the scan and for
|
||||
`ingest_resolved_pair` (it used to raise in `_save_pair`).
|
||||
- **Parity floor.** Metadata linking, the scan's resolver and parser floor are
|
||||
unchanged for every non-blank result; only guesses are gone.
|
||||
|
||||
Tests: `tests/test_prompt_attribution.py` (nodes, tracker, monitor, run keys),
|
||||
`tests/test_run_attribution.py` (history attribution, monitor and scan),
|
||||
`tests/test_review_image_links_tool.py`, and the real-host
|
||||
`tests/comfyui/image-attribution.spec.mjs` (both renderers): a typed Pair image
|
||||
links, a Pair image whose positive is only linked prepend text from an unknown
|
||||
node stays unlinked with nothing stored, an unknown node feeding `text` links
|
||||
through its run key, and its cached re-run with a new sampler seed links again
|
||||
(the host's `execution_cached` event names the PromptManager node, under the
|
||||
fixture's Classic cache). A second case saves with metadata disabled and holds
|
||||
the run open behind the fixture gate after SaveImage: the output is waiting and
|
||||
unlinked while the run is open, and links to the stored prompt once the run's
|
||||
outputs are recorded. The managed host config disables monitoring, so these
|
||||
cases start and stop the plugin's monitor through the owned fixture route
|
||||
`/pm_execution_fixture/monitor` (its GET lists the waiting files).
|
||||
On origin/main the prepend image was linked to an unrelated prompt.
|
||||
|
||||
Repair: `tools/review_image_links.py <db> --output-dir <ComfyUI output> --ids
|
||||
<ids>` (or `--prompt <id>`) judges links by these rules (`keep`, `relink`,
|
||||
`unlink`, `conflict`, `missing`) on a backup copy and changes nothing.
|
||||
`--apply` writes `relink`/`unlink` only for ids named with `--ids`, after saving
|
||||
`<db>.pre-relink_<stamp>`, and only while each image is still linked to the
|
||||
prompt it was judged on (otherwise it is counted as skipped). `relink` points the
|
||||
link record at the other prompt and drops its pair links. `unlink` removes only
|
||||
the image's link record (its `generated_images` row, the metadata snapshot stored
|
||||
in it and its pair links): that is how the app represents an unlinked image, the
|
||||
same as a file the scan could not attribute, so the file stays on disk and in the
|
||||
gallery as unlinked, and its favorite and tags (kept by file) are untouched. Every
|
||||
run prints this. Queue-linked batch images can't be proven after the fact and read
|
||||
`unlink`, so review before applying. Stop ComfyUI first.
|
||||
## Linked prompt text (#343)
|
||||
|
||||
A link drives `text` on every PromptManager node, and `negative_text` on the Pair,
|
||||
as it drives any core widget. The Focused, Split and Pair editors hide their widgets,
|
||||
and the frontend only draws a widget's socket while the widget is visible, so
|
||||
`web/js/text-links.js` detaches those two inputs from their hidden widgets. They become
|
||||
ordinary sockets; the schema, widget order and saved `widgets_values` do not change. A
|
||||
link dropped on a prompt box (or its link note) fills that box's input: Nodes 2.0 resolves
|
||||
it through the box's `.lg-node-widget` and `[data-slot-key]`, Classic through the node's
|
||||
`getInputOnPos`. An exact STRING link dropped elsewhere on the editor fills the main prompt
|
||||
first, then the negative, before the frontend's own choice (it used to pick `prepend_text`,
|
||||
which caused #342). A link being moved off a text input counts as free, as the frontend
|
||||
counts it, so dropping it back on the editor keeps it on `text`. Wildcard links (a
|
||||
frontend Primitive before it is typed) are not steered.
|
||||
|
||||
The frontend's legacy Primitive is virtual: it writes its value into the target widget,
|
||||
which the editor text no longer is. For the editors' detached inputs the `graphToPrompt`
|
||||
wrapper puts the Primitive's value (through virtual reroutes, also inside subgraphs, where
|
||||
the queued id is `<subgraph node>:<inner node>`) into the queued `text`/`negative_text`,
|
||||
resolved as the host resolves a value written into the widget: the Primitive's text
|
||||
replacement when its "Run widget replace on values" option is on, then the hidden widget's
|
||||
own serializer (dynamic prompts: `{red|blue}` becomes one choice). Each queued output is
|
||||
filled once (the queue hook only fills an output that did not come through the wrapper),
|
||||
so a wildcard is drawn once per run. Inputs still bound to their widget (the Classic
|
||||
nodes) are left as the host resolved them; as on main, the Classic text widget's own
|
||||
serializer queues its text as written, wildcards included (specs wait on the node's
|
||||
`_pmTextSerializerReady` before queuing). A Primitive drives the text as before, also in
|
||||
workflows saved before #343. On every Run the host logs "Invalid widget or widget name"
|
||||
for a Primitive wired to a detached editor input: its own widget write finds no widget
|
||||
binding there. That is expected and harmless. A linked
|
||||
Classic text box gives its canvas row back while linked (the core keeps an empty gap) and
|
||||
gets its size back on disconnect. The metadata parsers map a known node's positional
|
||||
widget values through its widget table when its widget-bound inputs do not line up with
|
||||
it (the editors' text inputs are sockets now).
|
||||
|
||||
Not covered, by ruling: linking a subgraph input to an inner editor's `text` gives the
|
||||
subgraph a plain STRING socket, not a promoted typeable widget as a core widget input
|
||||
would. The socket was unreachable before #343, so nothing regressed.
|
||||
While linked, each editor says the value comes from a link, and the typed text is kept
|
||||
in the workflow for when the link is disconnected. That includes the Classic nodes,
|
||||
which used to save `null` under a link.
|
||||
|
||||
- `tests/js/text-links.test.js`: socket exposure, drop preference (exact STRING, moved
|
||||
links), box hit-test, Primitive values through reroutes, replacement and the widget
|
||||
serializer (bound inputs left to the host), stored values, the Classic row.
|
||||
- `tests/test_generation_params.py` (`workflow-only-detached-*` cases, with JS parity
|
||||
vectors): editor text inputs without widget bindings keep their positional values, and so
|
||||
does a core KSampler whose only widget-bound input (`cfg`) is out of schema order.
|
||||
- `tests/browser/{focused,pair,classic}-node.spec.mjs`: the linked note, typed text
|
||||
kept through save and configure (also with a workflow's widget binding restored), the
|
||||
Classic "use prompt" guard while linked.
|
||||
- `tests/comfyui/text-link.spec.mjs` (both startup projects): a real mouse drag of a
|
||||
STRING output onto each node's prompt box (and onto the Pair's negative socket); the
|
||||
link, the note and the typed text survive renderer switches both ways, a reload,
|
||||
disconnect and node removal. `fixtures/workflow-before-343.json` and
|
||||
`workflow-before-343-primitive.json` were serialized by the pre-#343 code on the pinned
|
||||
host and must load unchanged; the second one's legacy Primitives keep driving the text
|
||||
through a real Run. Also: a Primitive dropped on the Main prompt box and a string on the
|
||||
Negative prompt box, a moved link released on the editor body, paste, clone and undo, a
|
||||
legacy Primitive inside a subgraph, and wildcard Primitives on every node type queued
|
||||
through a real Run as one choice, the same one another `graphToPrompt` wrapper sees.
|
||||
- `tests/comfyui/execution-text-link.spec.mjs` (execution host): workflows built by the
|
||||
real frontend through the sockets, with text from `PrimitiveString`, the legacy
|
||||
Primitive or through `PreviewAny`, for PromptManager, PromptManagerText, Focused, Split
|
||||
and the Pair (positive, negative, both, neither linked, and a Primitive negative). A
|
||||
wildcard Primitive on PromptManager (queued as written) and Focused (one choice) saves
|
||||
and counts what was queued. The linked text is saved, counted once, recorded as
|
||||
the run's raw positive/negative, and after a scan the run's image links to it. Text
|
||||
only known at run time (`PreviewAny`) cannot be read from the image; the case only
|
||||
requires that it is never linked to another prompt (its attribution from the recorded
|
||||
run belongs to #342).
|
||||
|
||||
```sh
|
||||
node --test tests/js/text-links.test.js
|
||||
npm run --prefix tools/vendor test:workflows -- classic-node.spec.mjs focused-node.spec.mjs pair-node.spec.mjs \
|
||||
--project chromium --project touch --project firefox --project webkit
|
||||
PM_COMFY_ROOT=... PM_TEST_PYTHON=... npm run --prefix tools/vendor test:comfyui -- text-link
|
||||
PM_COMFY_ROOT=... PM_TEST_PYTHON=... npm run --prefix tools/vendor test:executions -- execution-text-link
|
||||
```
|
||||
|
||||
## Durable execution ledger
|
||||
|
||||
`npm run --prefix tools/vendor test:executions` runs 46 real execution cases
|
||||
(23 each in Classic and Nodes 2.0, retaining all 42 prior execution cases),
|
||||
`npm run --prefix tools/vendor test:executions` runs 52 real execution cases
|
||||
(26 each in Classic and Nodes 2.0, retaining all 48 prior execution cases and adding
|
||||
the two #342 image-attribution cases),
|
||||
alongside the existing 80 stock host cases (the original 78 plus the native
|
||||
editor host case) and six pack compatibility cases. Run
|
||||
these suites serially because they share report cleanup and owned host resources. Use the same pinned disposable `PM_COMFY_ROOT`,
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
"""Atomic replacement of one imported category (#221, #249 System S10).
|
||||
|
||||
The LoRA reference import used to delete its whole category and then save
|
||||
each record in its own transaction, so a failure part way left the previous
|
||||
collection deleted and the new one partial. ``replace_category_prompts``
|
||||
does both in one ``BEGIN IMMEDIATE`` transaction: the category's current
|
||||
prompts (with their tags and image links) are replaced by the staged
|
||||
records, or, on any error, nothing changes.
|
||||
|
||||
A staged record is ``{text, hash, tags, images}``. A record whose hash a
|
||||
prompt outside the category already has links its images to that prompt
|
||||
(the legacy import did the same); a second record with the hash of an
|
||||
earlier one links its images to that one. Neither is a new prompt.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Optional, Sequence
|
||||
|
||||
try:
|
||||
from .maintenance import PlanChanged
|
||||
from .pairs import PairDependencyError
|
||||
from .workflow_documents import documents_of_rows, release_documents
|
||||
except ImportError: # pragma: no cover - flat test imports
|
||||
from database.maintenance import PlanChanged
|
||||
from database.pairs import PairDependencyError
|
||||
from database.workflow_documents import documents_of_rows, release_documents
|
||||
|
||||
|
||||
class EmptyReplacement(ValueError):
|
||||
"""The replacement has no record and would remove every current import."""
|
||||
|
||||
|
||||
# The category's prompt ids, as a subquery (one bound parameter).
|
||||
_IN_CATEGORY = "SELECT id FROM prompts WHERE category = ?"
|
||||
|
||||
# Each record's outcome and the result counter it adds to.
|
||||
_COUNTERS = {"imported": "imported", "linked": "linked", "duplicate": "duplicates"}
|
||||
|
||||
|
||||
def _placeholders(values: Sequence[Any]) -> str:
|
||||
return ",".join("?" * len(values))
|
||||
|
||||
|
||||
class CategoryImportOperations:
|
||||
"""Read the state of an imported category and replace it atomically."""
|
||||
|
||||
def category_import_state(self, category: str) -> Dict[str, Any]:
|
||||
"""``{ids, paired}``: the category's prompt ids (ascending) and how
|
||||
many of them a saved pair uses (those block a replacement)."""
|
||||
with self.model.get_connection() as conn:
|
||||
ids = [
|
||||
row[0]
|
||||
for row in conn.execute(
|
||||
"SELECT id FROM prompts WHERE category = ? ORDER BY id",
|
||||
(category,),
|
||||
).fetchall()
|
||||
]
|
||||
paired = conn.execute(
|
||||
"SELECT COUNT(*) FROM prompts WHERE category = ? AND id IN "
|
||||
"(SELECT positive_prompt_id FROM prompt_pairs UNION "
|
||||
"SELECT negative_prompt_id FROM prompt_pairs)",
|
||||
(category,),
|
||||
).fetchone()[0]
|
||||
return {"ids": ids, "paired": paired}
|
||||
|
||||
def existing_prompt_hashes(
|
||||
self, hashes: Sequence[str], exclude_category: str
|
||||
) -> set:
|
||||
"""The given hashes a prompt outside ``exclude_category`` already has."""
|
||||
found = set()
|
||||
wanted = list(dict.fromkeys(hashes))
|
||||
with self.model.get_connection() as conn:
|
||||
for start in range(0, len(wanted), 500):
|
||||
chunk = wanted[start : start + 500]
|
||||
rows = conn.execute(
|
||||
f"SELECT hash FROM prompts WHERE hash IN ({_placeholders(chunk)}) "
|
||||
"AND (category IS NULL OR category != ?)",
|
||||
[*chunk, exclude_category],
|
||||
).fetchall()
|
||||
found.update(row[0] for row in rows)
|
||||
return found
|
||||
|
||||
def replace_category_prompts(
|
||||
self,
|
||||
category: str,
|
||||
records: List[Dict[str, Any]],
|
||||
*,
|
||||
expected_ids: Optional[List[int]] = None,
|
||||
allow_empty: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
"""Replace every prompt of ``category`` with ``records`` in one transaction.
|
||||
|
||||
Raises (and changes nothing): ``PlanChanged`` when ``expected_ids`` is
|
||||
given and the category's prompt ids differ from it;
|
||||
``PairDependencyError`` when a saved pair uses one of its prompts;
|
||||
``EmptyReplacement`` when ``records`` is empty while the category has
|
||||
prompts, unless ``allow_empty`` (an empty replacement that was
|
||||
reviewed). Any other error rolls everything back too.
|
||||
|
||||
Returns ``{removed, imported, linked, duplicates, images, rows}``:
|
||||
prompts removed and created, records linked to an existing prompt or
|
||||
to an earlier record with the same text, image links added, and per
|
||||
record ``{name, outcome}`` (``imported``/``linked``/``duplicate``).
|
||||
"""
|
||||
with self._pair_transaction() as conn:
|
||||
ids = [
|
||||
row[0]
|
||||
for row in conn.execute(
|
||||
"SELECT id FROM prompts WHERE category = ? ORDER BY id",
|
||||
(category,),
|
||||
).fetchall()
|
||||
]
|
||||
if expected_ids is not None and ids != list(expected_ids):
|
||||
raise PlanChanged("The imported prompts changed since the review")
|
||||
if not records and ids and not allow_empty:
|
||||
raise EmptyReplacement(
|
||||
"Nothing to import; the previous imports were kept"
|
||||
)
|
||||
self._ensure_category_unpaired(conn, category)
|
||||
removed = self._delete_category_rows(conn, category, ids)
|
||||
result = {
|
||||
"removed": removed,
|
||||
"imported": 0,
|
||||
"linked": 0,
|
||||
"duplicates": 0,
|
||||
"images": 0,
|
||||
"rows": [],
|
||||
}
|
||||
created = {}
|
||||
for record in records:
|
||||
self._import_record(conn, category, record, created, result)
|
||||
return result
|
||||
|
||||
def _delete_category_rows(self, conn, category: str, ids: List[int]) -> int:
|
||||
"""Delete the category's prompts with their image links and tags.
|
||||
|
||||
No commit. The rows are selected by category (just verified against
|
||||
``ids`` under the write lock), so no statement binds one parameter per
|
||||
prompt and SQLite's variable limit never caps a collection.
|
||||
"""
|
||||
if not ids:
|
||||
return 0
|
||||
documents = documents_of_rows(conn, "prompt_id", ids) # chunked
|
||||
conn.execute(
|
||||
f"DELETE FROM generated_images WHERE prompt_id IN ({_IN_CATEGORY})",
|
||||
(category,),
|
||||
)
|
||||
release_documents(conn, documents)
|
||||
conn.execute(
|
||||
f"DELETE FROM prompt_tags WHERE prompt_id IN ({_IN_CATEGORY})",
|
||||
(category,),
|
||||
)
|
||||
return conn.execute(
|
||||
"DELETE FROM prompts WHERE category = ?", (category,)
|
||||
).rowcount
|
||||
|
||||
@staticmethod
|
||||
def _ensure_category_unpaired(conn, category: str) -> None:
|
||||
"""Raise PairDependencyError when a saved pair uses a category prompt."""
|
||||
row = conn.execute(
|
||||
f"SELECT id FROM prompt_pairs WHERE positive_prompt_id IN ({_IN_CATEGORY}) "
|
||||
f"OR negative_prompt_id IN ({_IN_CATEGORY}) LIMIT 1",
|
||||
(category, category),
|
||||
).fetchone()
|
||||
if row:
|
||||
raise PairDependencyError(
|
||||
f"Prompt belongs to saved pair {row[0]}; delete the pair first "
|
||||
"or save changed text as a new prompt"
|
||||
)
|
||||
|
||||
def _import_record(self, conn, category, record, created, result) -> None:
|
||||
"""Save one staged record on ``conn``, or link it to the prompt it repeats."""
|
||||
prompt_hash = record["hash"]
|
||||
name = record.get("name") or record["text"]
|
||||
if prompt_hash in created:
|
||||
prompt_id, outcome = created[prompt_hash], "duplicate"
|
||||
else:
|
||||
row = conn.execute(
|
||||
"SELECT id FROM prompts WHERE hash = ?", (prompt_hash,)
|
||||
).fetchone()
|
||||
if row is not None:
|
||||
prompt_id, outcome = row[0], "linked"
|
||||
else:
|
||||
prompt_id = self._insert_prompt(
|
||||
conn,
|
||||
record["text"],
|
||||
category,
|
||||
list(record.get("tags") or []),
|
||||
None,
|
||||
None,
|
||||
prompt_hash,
|
||||
)
|
||||
outcome = "imported"
|
||||
created[prompt_hash] = prompt_id
|
||||
result[_COUNTERS[outcome]] += 1
|
||||
for image in record.get("images") or []:
|
||||
if self._insert_image_link(conn, prompt_id, image):
|
||||
result["images"] += 1
|
||||
result["rows"].append({"name": name, "outcome": outcome})
|
||||
@@ -4,6 +4,8 @@ import sqlite3
|
||||
from contextlib import ExitStack
|
||||
from typing import Callable, ContextManager
|
||||
|
||||
from .run_attribution import notify_outputs_sealed
|
||||
|
||||
try:
|
||||
from ..utils.execution_history_events import RunFinish, encode_command
|
||||
from ..utils.execution_outputs import OutputBundle, validate_output_bundle
|
||||
@@ -113,6 +115,10 @@ class ExecutionOutputOperations:
|
||||
"VALUES(?,?,?,?,?,?,?,?)",
|
||||
[(run_id, *reference) for reference in prepared],
|
||||
)
|
||||
# Committed: images waiting for this run may now be attributed (#342)
|
||||
notify_outputs_sealed(
|
||||
run_id, [dict(zip(_REFERENCE_FIELDS, row)) for row in prepared]
|
||||
)
|
||||
return True
|
||||
|
||||
def get_execution_outputs(self, run_id: str) -> dict:
|
||||
|
||||
+52
-62
@@ -9,7 +9,6 @@ import math
|
||||
import sqlite3
|
||||
import os
|
||||
import threading
|
||||
import tempfile
|
||||
import weakref
|
||||
from contextlib import closing
|
||||
from pathlib import Path
|
||||
@@ -20,6 +19,7 @@ from .managed_connection import (
|
||||
await_closed,
|
||||
retire_connections,
|
||||
)
|
||||
from . import restore_stage
|
||||
from .library_schema import library_schema_error, migrate_library_state
|
||||
from .image_favorites import image_favorites_schema_error, migrate_image_favorites
|
||||
from .annotation_schema import annotation_schema_error, migrate_annotation_state
|
||||
@@ -39,7 +39,6 @@ from .workflow_documents import (
|
||||
from .execution_history_schema import (
|
||||
history_schema_error,
|
||||
migrate_execution_history,
|
||||
rotate_history_epoch,
|
||||
)
|
||||
|
||||
# Import logging system
|
||||
@@ -555,7 +554,10 @@ class PromptModel:
|
||||
"WHERE pp.positive_prompt_id IS NOT gi.prompt_id LIMIT 1"
|
||||
).fetchone():
|
||||
return "Image pair association has a different positive prompt owner"
|
||||
for row in conn.execute("SELECT provenance FROM image_prompt_pairs"):
|
||||
# fetchone, not iteration: a managed connection's cursor prefetches
|
||||
# ITER_CHUNK rows when iterated, and each provenance may be ~4 MB.
|
||||
cursor = conn.execute("SELECT provenance FROM image_prompt_pairs")
|
||||
while (row := cursor.fetchone()) is not None:
|
||||
error = pair_provenance_error(row[0])
|
||||
if error:
|
||||
return error
|
||||
@@ -1132,7 +1134,7 @@ class PromptModel:
|
||||
|
||||
@staticmethod
|
||||
def verify_database_file(
|
||||
path: str, *, require_library_state: bool = False
|
||||
path: str, *, require_library_state: bool = False, documents: bool = True
|
||||
) -> Tuple[bool, str]:
|
||||
"""
|
||||
Check that a file is a healthy prompts database before restoring it.
|
||||
@@ -1146,6 +1148,9 @@ class PromptModel:
|
||||
database has (Library state, image favorites, saved views,
|
||||
image annotations and the review queue), as a migrated
|
||||
restore stage must; old backups may lack them.
|
||||
documents: Read every workflow document and refuse a damaged
|
||||
one. A restore checks its upload without it and repairs
|
||||
damaged documents on the stage (``database/restore_stage.py``).
|
||||
|
||||
Returns:
|
||||
(ok, reason): reason is "ok" when the file is acceptable, otherwise
|
||||
@@ -1221,7 +1226,7 @@ class PromptModel:
|
||||
if review_error:
|
||||
return False, review_error
|
||||
documents_error = workflow_documents_schema_error(
|
||||
conn, allow_missing=not require_library_state, documents=True
|
||||
conn, allow_missing=not require_library_state, documents=documents
|
||||
)
|
||||
if documents_error:
|
||||
return False, documents_error
|
||||
@@ -1255,17 +1260,43 @@ class PromptModel:
|
||||
finally:
|
||||
source.close()
|
||||
|
||||
def stage_restore(self, src_path: str):
|
||||
"""Stage ``src_path`` beside this database for a restore (blocking).
|
||||
|
||||
Returns a ``StagedRestore`` (``database/restore_stage.py``): a
|
||||
migrated, repaired and verified copy in the library's folder, its
|
||||
digest and what it holds. Neither the live database nor the source
|
||||
is touched. Raises ValueError when either does not verify.
|
||||
"""
|
||||
return restore_stage.stage_restore(PromptModel, self.db_path, src_path)
|
||||
|
||||
def publish_staged(self, staged) -> str:
|
||||
"""Replace the live database with ``staged``; return the safety backup.
|
||||
|
||||
See ``restore_stage.publish_staged``: the staged bytes are rechecked,
|
||||
the current database is kept as ``<db_path>.backup_<stamp>`` and the
|
||||
WAL is truncated afterwards.
|
||||
"""
|
||||
backup_path = restore_stage.publish_staged(self, staged)
|
||||
self.logger.info("Database restored from a staged copy")
|
||||
from .execution_history import notify_history_restore
|
||||
|
||||
notify_history_restore(self.db_path)
|
||||
return backup_path
|
||||
|
||||
def restore_from_file(self, src_path: str) -> str:
|
||||
"""
|
||||
Replace the live database with the file at src_path.
|
||||
|
||||
A disposable SQLite backup of the source is migrated and verified first.
|
||||
Only then is the current database backed up next to itself as
|
||||
``<db_path>.backup_<YYYYmmdd_HHMMSS>`` (WAL content included), and the
|
||||
staged pages replace the live pages through SQLite's backup API.
|
||||
Migration failures never replace live content or modify the source.
|
||||
An online replacement failure leaves the live database intact and
|
||||
propagates; the completed safety backup remains available.
|
||||
``stage_restore`` then ``publish_staged``: a disposable copy of the
|
||||
source is migrated, repaired and verified in the library's folder
|
||||
first. Only then is the current database backed up next to itself as
|
||||
``<db_path>.backup_<YYYYmmdd_HHMMSS_ffffff>`` (WAL content included),
|
||||
and the staged pages replace the live pages through SQLite's backup
|
||||
API. Staging failures never replace live content or modify the
|
||||
source. An online replacement failure leaves the live database intact
|
||||
and propagates; the completed safety backup remains available. The
|
||||
stage is removed either way.
|
||||
|
||||
Args:
|
||||
src_path: SQLite file to restore
|
||||
@@ -1275,55 +1306,14 @@ class PromptModel:
|
||||
there was no database to back up.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If the previous database could not be backed up
|
||||
ValueError: If the source or its migrated copy does not verify
|
||||
SafetyBackupFailed: If the previous database could not be backed up
|
||||
sqlite3.OperationalError: If SQLite refuses online replacement
|
||||
"""
|
||||
ok, reason = self.verify_database_file(src_path)
|
||||
if not ok:
|
||||
raise ValueError(reason)
|
||||
with tempfile.TemporaryDirectory(prefix="prompt-manager-restore-") as directory:
|
||||
staged_path = str(Path(directory) / "staged.db")
|
||||
source = sqlite3.connect(
|
||||
Path(src_path).resolve().as_uri() + "?mode=ro", uri=True
|
||||
)
|
||||
try:
|
||||
with closing(sqlite3.connect(staged_path)) as destination:
|
||||
source.backup(destination)
|
||||
finally:
|
||||
source.close()
|
||||
staged = None
|
||||
try:
|
||||
staged = PromptModel(staged_path)
|
||||
rotate_history_epoch(staged.get_connection())
|
||||
ok, reason = self.verify_database_file(
|
||||
staged_path, require_library_state=True
|
||||
)
|
||||
if not ok:
|
||||
raise ValueError(reason)
|
||||
staged.close_all()
|
||||
backup_path = ""
|
||||
if os.path.exists(self.db_path):
|
||||
timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
backup_path = f"{self.db_path}.backup_{timestamp}"
|
||||
if not self.backup_database(backup_path):
|
||||
raise RuntimeError("Could not back up the current database")
|
||||
self.logger.info(f"Current database backed up to: {backup_path}")
|
||||
try:
|
||||
self._replace_contents_from(staged_path)
|
||||
except sqlite3.OperationalError as exc:
|
||||
exc._prompt_manager_restore_backup_created = bool(backup_path)
|
||||
raise
|
||||
with PromptModel._init_lock:
|
||||
PromptModel._initialized_paths.add(self._schema_key())
|
||||
self.logger.info(f"Database restored from {src_path}")
|
||||
from .execution_history import notify_history_restore
|
||||
|
||||
notify_history_restore(self.db_path)
|
||||
return backup_path
|
||||
finally:
|
||||
if staged is not None:
|
||||
staged.close_all()
|
||||
with PromptModel._init_lock:
|
||||
PromptModel._initialized_paths.discard(
|
||||
os.path.normcase(os.path.realpath(staged_path))
|
||||
)
|
||||
staged = self.stage_restore(src_path)
|
||||
try:
|
||||
backup_path = self.publish_staged(staged)
|
||||
finally:
|
||||
staged.discard()
|
||||
self.logger.info(f"Database restored from {src_path}")
|
||||
return backup_path
|
||||
|
||||
+100
-70
@@ -26,6 +26,7 @@ from .history_reads import HistoryReadOperations
|
||||
from .health import HealthOperations, iter_consistency_issues
|
||||
from .duplicate_copies import DuplicateCopyOperations
|
||||
from .workflow_storage import WorkflowStorageOperations
|
||||
from .category_import import CategoryImportOperations, EmptyReplacement # noqa: F401
|
||||
from .workflow_documents import (
|
||||
DocumentReader,
|
||||
documents_of_rows,
|
||||
@@ -366,6 +367,7 @@ class PromptDatabase(
|
||||
MaintenanceOperations,
|
||||
DuplicateCopyOperations,
|
||||
WorkflowStorageOperations,
|
||||
CategoryImportOperations,
|
||||
):
|
||||
"""Database operations class for managing prompts."""
|
||||
|
||||
@@ -455,26 +457,9 @@ class PromptDatabase(
|
||||
|
||||
try:
|
||||
with self.model.get_connection() as conn:
|
||||
cursor = conn.execute(
|
||||
"""
|
||||
INSERT INTO prompts (
|
||||
text, category, tags, rating, notes, hash, created_at,
|
||||
updated_at
|
||||
) VALUES (?, ?, NULL, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
text.strip(),
|
||||
category,
|
||||
rating,
|
||||
notes,
|
||||
prompt_hash,
|
||||
datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
),
|
||||
prompt_id = self._insert_prompt(
|
||||
conn, text, category, tags, rating, notes, prompt_hash
|
||||
)
|
||||
prompt_id = cursor.lastrowid
|
||||
if tags:
|
||||
self._sync_prompt_tags(conn, prompt_id, tags)
|
||||
conn.commit()
|
||||
self.logger.debug(f"Successfully saved prompt with ID: {prompt_id}")
|
||||
return prompt_id
|
||||
@@ -490,6 +475,32 @@ class PromptDatabase(
|
||||
)
|
||||
return existing["id"]
|
||||
|
||||
def _insert_prompt(
|
||||
self,
|
||||
conn: sqlite3.Connection,
|
||||
text: str,
|
||||
category: Optional[str],
|
||||
tags: Optional[List[str]],
|
||||
rating: Optional[int],
|
||||
notes: Optional[str],
|
||||
prompt_hash: Optional[str],
|
||||
) -> int:
|
||||
"""Insert one prompt row and its tags on ``conn`` (no commit)."""
|
||||
now = datetime.datetime.now(datetime.timezone.utc).isoformat()
|
||||
cursor = conn.execute(
|
||||
"""
|
||||
INSERT INTO prompts (
|
||||
text, category, tags, rating, notes, hash, created_at,
|
||||
updated_at
|
||||
) VALUES (?, ?, NULL, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(text.strip(), category, rating, notes, prompt_hash, now, now),
|
||||
)
|
||||
prompt_id = cursor.lastrowid
|
||||
if tags:
|
||||
self._sync_prompt_tags(conn, prompt_id, tags)
|
||||
return prompt_id
|
||||
|
||||
def record_prompt_use(self, prompt_id: int, times: int = 1) -> bool:
|
||||
"""
|
||||
Count runs of a prompt and mark it as most recently used.
|
||||
@@ -1670,64 +1681,83 @@ class PromptDatabase(
|
||||
return 0
|
||||
|
||||
# Proceed with linking
|
||||
if not isinstance(metadata, dict):
|
||||
metadata = {}
|
||||
filename = os.path.basename(image_path)
|
||||
file_path = normalize_image_path(image_path)
|
||||
file_size, width, height, image_format = _coerce_file_info(metadata)
|
||||
# Each workflow and prompt graph is stored once (new form).
|
||||
documents = stored_values(
|
||||
conn,
|
||||
{
|
||||
"workflow_data": dumps_strict(metadata.get("workflow", {})),
|
||||
"prompt_metadata": dumps_strict(metadata.get("prompt", {})),
|
||||
},
|
||||
image_id = self._insert_image_link(
|
||||
conn, prompt_id_int, image_path, metadata
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
# INSERT OR IGNORE skips a file already linked to this prompt
|
||||
# (same prompt_id + normalised full path)
|
||||
cursor = conn.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO generated_images
|
||||
(prompt_id, image_path, filename, file_path, file_size, width,
|
||||
height, format, workflow_data, prompt_metadata, parameters,
|
||||
workflow_document_id, prompt_document_id)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
if not image_id:
|
||||
self.logger.debug(
|
||||
f"Image {normalize_image_path(image_path)} already linked to "
|
||||
f"prompt {prompt_id_int}"
|
||||
)
|
||||
return 0
|
||||
|
||||
return image_id
|
||||
|
||||
def _insert_image_link(
|
||||
self,
|
||||
conn: sqlite3.Connection,
|
||||
prompt_id_int: int,
|
||||
image_path: str,
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
) -> int:
|
||||
"""Insert one image link on ``conn`` (no commit).
|
||||
|
||||
Returns the new image row's id, or 0 when the file is already linked
|
||||
to this prompt (``INSERT OR IGNORE``).
|
||||
"""
|
||||
if not isinstance(metadata, dict):
|
||||
metadata = {}
|
||||
filename = os.path.basename(image_path)
|
||||
file_path = normalize_image_path(image_path)
|
||||
file_size, width, height, image_format = _coerce_file_info(metadata)
|
||||
# Each workflow and prompt graph is stored once (new form).
|
||||
documents = stored_values(
|
||||
conn,
|
||||
{
|
||||
"workflow_data": dumps_strict(metadata.get("workflow", {})),
|
||||
"prompt_metadata": dumps_strict(metadata.get("prompt", {})),
|
||||
},
|
||||
)
|
||||
|
||||
# INSERT OR IGNORE skips a file already linked to this prompt
|
||||
# (same prompt_id + normalised full path)
|
||||
cursor = conn.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO generated_images
|
||||
(prompt_id, image_path, filename, file_path, file_size, width,
|
||||
height, format, workflow_data, prompt_metadata, parameters,
|
||||
workflow_document_id, prompt_document_id)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
prompt_id_int,
|
||||
image_path,
|
||||
filename,
|
||||
file_path,
|
||||
file_size,
|
||||
width,
|
||||
height,
|
||||
image_format,
|
||||
documents["workflow_data"],
|
||||
documents["prompt_metadata"],
|
||||
dumps_strict(metadata.get("parameters", {})),
|
||||
documents["workflow_document_id"],
|
||||
documents["prompt_document_id"],
|
||||
),
|
||||
)
|
||||
if cursor.rowcount == 0:
|
||||
# Already linked: a document made only for this row goes.
|
||||
release_documents(
|
||||
conn,
|
||||
(
|
||||
prompt_id_int,
|
||||
image_path,
|
||||
filename,
|
||||
file_path,
|
||||
file_size,
|
||||
width,
|
||||
height,
|
||||
image_format,
|
||||
documents["workflow_data"],
|
||||
documents["prompt_metadata"],
|
||||
dumps_strict(metadata.get("parameters", {})),
|
||||
documents["workflow_document_id"],
|
||||
documents["prompt_document_id"],
|
||||
),
|
||||
)
|
||||
if cursor.rowcount == 0:
|
||||
# Already linked: a document made only for this row goes.
|
||||
release_documents(
|
||||
conn,
|
||||
(
|
||||
documents["workflow_document_id"],
|
||||
documents["prompt_document_id"],
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
if cursor.rowcount == 0:
|
||||
self.logger.debug(
|
||||
f"Image {file_path} already linked to prompt {prompt_id_int}"
|
||||
)
|
||||
return 0
|
||||
|
||||
return cursor.lastrowid
|
||||
return 0
|
||||
return cursor.lastrowid
|
||||
|
||||
def get_prompt_images(self, prompt_id: str) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
|
||||
@@ -23,10 +23,26 @@ except ImportError:
|
||||
from utils.prompt_pair_resolver import RESOLVER_VERSION
|
||||
|
||||
|
||||
def blank_positive(resolution: Dict[str, Any]) -> bool:
|
||||
"""A resolved pair whose stored positive text is empty (#342).
|
||||
|
||||
A PromptManager node whose editor is empty encodes only its prepend and
|
||||
append text, which is never stored as a prompt: such a result names no
|
||||
prompt to link.
|
||||
"""
|
||||
if resolution.get("state") != "resolved":
|
||||
return False
|
||||
positive = resolution.get("positive") or {}
|
||||
text = positive.get("storage_text", positive.get("text"))
|
||||
return not (isinstance(text, str) and text.strip())
|
||||
|
||||
|
||||
def ingest_resolved_pair(
|
||||
db: Any, image_path: str, metadata: Dict[str, Any], resolution: Dict[str, Any]
|
||||
) -> Dict[str, Any]:
|
||||
state = resolution.get("state", "unresolved")
|
||||
if state == "resolved" and blank_positive(resolution):
|
||||
state = "unresolved"
|
||||
if state != "resolved":
|
||||
return {"status": state, "pairs_added": 0, "links_added": 0}
|
||||
with db._pair_transaction() as conn:
|
||||
|
||||
@@ -0,0 +1,365 @@
|
||||
"""A restore in two steps: stage the upload, then publish it (#249 System S8).
|
||||
|
||||
**Stage** (``stage_restore``) never touches the live database or the source.
|
||||
In a new folder ``.pm-restore-*`` in the library's own folder (#318: never
|
||||
the system temp folder, which may be another disk), it:
|
||||
|
||||
1. checks the source (``verify_database_file`` without reading every
|
||||
workflow document: a damaged one is repaired below, not refused);
|
||||
2. copies it with SQLite's backup API (a rollback-journal file and a WAL file
|
||||
alike) and opens the copy as a ``PromptModel``, which migrates an older
|
||||
schema;
|
||||
3. repairs damaged workflow documents (``repair_damaged_documents``),
|
||||
rotates the history epoch and counts what the copy holds;
|
||||
4. makes the copy one self-contained file and verifies it fully (every
|
||||
current table, every document);
|
||||
5. records the copy's SHA-256, so publishing binds exactly these bytes.
|
||||
|
||||
**Publish** (``publish_staged``) re-hashes the stage (``StageChanged`` if it
|
||||
differs), keeps the current database as a uniquely named, standalone safety
|
||||
backup beside it (``database/safety_backup.py``; not verified, because the
|
||||
database being replaced may be the damaged one), copies the stage over the
|
||||
live pages through the backup API (no file is swapped under open handles; an
|
||||
``OperationalError`` there leaves the live data intact, #226), then
|
||||
checkpoints the WAL with TRUNCATE so the copied pages do not leave a
|
||||
database-sized ``-wal`` behind. A reader that keeps a snapshot past the busy
|
||||
wait is logged, not fatal: the restore itself is done.
|
||||
|
||||
Every handle on the stage is closed before its folder is removed (Windows).
|
||||
``sweep_restore_temps`` removes what an interrupted restore left (startup).
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
import tempfile
|
||||
from contextlib import closing
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Dict, Optional
|
||||
|
||||
from . import workflow_documents as wd
|
||||
from .execution_history_schema import rotate_history_epoch
|
||||
from .safety_backup import take_safety_backup
|
||||
|
||||
logger = logging.getLogger("prompt_manager.database.restore_stage")
|
||||
|
||||
STAGE_PREFIX = ".pm-restore-"
|
||||
UPLOAD_PREFIX = ".pm-restore-upload-"
|
||||
STAGED_NAME = "staged.db"
|
||||
# How long the post-restore TRUNCATE checkpoint waits for readers to leave.
|
||||
CHECKPOINT_BUSY_MS = 30_000
|
||||
_HASH_CHUNK = 1024 * 1024
|
||||
SUMMARY_KEYS = (
|
||||
"prompts",
|
||||
"positives",
|
||||
"pairs",
|
||||
"archived_pairs",
|
||||
"image_links",
|
||||
"runs",
|
||||
"saved_views",
|
||||
)
|
||||
|
||||
|
||||
class StageChanged(ValueError):
|
||||
"""The staged file is no longer the bytes that were reviewed."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class StagedRestore:
|
||||
"""A verified, migrated copy of an upload, ready to publish."""
|
||||
|
||||
path: str
|
||||
digest: str
|
||||
size: int
|
||||
summary: Dict[str, int]
|
||||
schema: Dict[str, bool]
|
||||
documents: Dict[str, int] = field(
|
||||
default_factory=lambda: {"repaired": 0, "cleared": 0, "images": 0}
|
||||
)
|
||||
|
||||
@property
|
||||
def directory(self) -> str:
|
||||
return os.path.dirname(self.path)
|
||||
|
||||
def discard(self) -> None:
|
||||
"""Remove the stage folder (idempotent; its handles are all closed)."""
|
||||
remove_stage_folder(self.directory)
|
||||
|
||||
|
||||
def remove_stage_folder(directory: str) -> None:
|
||||
"""Remove a stage folder and its files, logging what cannot be removed."""
|
||||
if not directory or not os.path.isdir(directory):
|
||||
return
|
||||
for name in os.listdir(directory):
|
||||
try:
|
||||
os.remove(os.path.join(directory, name))
|
||||
except OSError as error:
|
||||
logger.warning(f"Could not remove restore stage file {name}: {error}")
|
||||
try:
|
||||
os.rmdir(directory)
|
||||
except OSError as error:
|
||||
logger.warning(f"Could not remove the restore stage folder: {error}")
|
||||
|
||||
|
||||
def file_digest(path: str) -> str:
|
||||
"""``sha256:<hex>`` of the file's bytes (blocking)."""
|
||||
digest = hashlib.sha256()
|
||||
with open(path, "rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(_HASH_CHUNK), b""):
|
||||
digest.update(chunk)
|
||||
return "sha256:" + digest.hexdigest()
|
||||
|
||||
|
||||
def _readonly(path: str) -> sqlite3.Connection:
|
||||
return sqlite3.connect(Path(path).resolve().as_uri() + "?mode=ro", uri=True)
|
||||
|
||||
|
||||
def _schema(path: str) -> list:
|
||||
"""The file's schema objects, to tell whether staging migrated it."""
|
||||
with closing(_readonly(path)) as conn:
|
||||
return sorted(
|
||||
conn.execute(
|
||||
"SELECT type, name, tbl_name, sql FROM sqlite_master "
|
||||
"WHERE name NOT LIKE 'sqlite_%'"
|
||||
).fetchall(),
|
||||
key=repr,
|
||||
)
|
||||
|
||||
|
||||
def _count(conn: sqlite3.Connection, table: str, where: str = "") -> int:
|
||||
exists = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)
|
||||
).fetchone()
|
||||
if not exists:
|
||||
return 0
|
||||
# ``table`` and ``where`` are constants of library_summary, never input.
|
||||
return conn.execute(f"SELECT COUNT(*) FROM {table} {where}").fetchone()[0] # nosec
|
||||
|
||||
|
||||
def library_summary(conn: sqlite3.Connection) -> Dict[str, int]:
|
||||
"""What a library holds, for a restore's review (reads only).
|
||||
|
||||
``positives`` are the Library's prompts (``library_counts``'s
|
||||
``positive_prompts``); ``pairs`` the active saved pairs and
|
||||
``archived_pairs`` the archived ones; ``image_links`` the indexed image
|
||||
rows; ``runs`` the recorded executions.
|
||||
"""
|
||||
from .library import _ELIGIBLE
|
||||
|
||||
has_state = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND "
|
||||
"name='prompt_library_state'"
|
||||
).fetchone()
|
||||
has_pairs = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='prompt_pairs'"
|
||||
).fetchone()
|
||||
if has_state and has_pairs:
|
||||
positives = conn.execute(
|
||||
"SELECT COUNT(*) FROM prompts p WHERE " + _ELIGIBLE
|
||||
).fetchone()[0]
|
||||
else:
|
||||
positives = _count(conn, "prompts")
|
||||
return {
|
||||
"prompts": _count(conn, "prompts"),
|
||||
"positives": positives,
|
||||
"pairs": _count(conn, "prompt_pairs", "WHERE archived=0"),
|
||||
"archived_pairs": _count(conn, "prompt_pairs", "WHERE archived<>0"),
|
||||
"image_links": _count(conn, "generated_images"),
|
||||
"runs": _count(conn, "execution_runs"),
|
||||
"saved_views": _count(conn, "saved_views"),
|
||||
}
|
||||
|
||||
|
||||
# The newest row of each table a restore would overwrite: with the counts,
|
||||
# a write after the review (a new run, image, prompt or edit) changes it.
|
||||
_FINGERPRINT_SQL = (
|
||||
("prompts", "SELECT MAX(id), MAX(updated_at), MAX(last_used_at) FROM prompts"),
|
||||
("generated_images", "SELECT MAX(id) FROM generated_images"),
|
||||
("prompt_pairs", "SELECT MAX(id), MAX(updated_at) FROM prompt_pairs"),
|
||||
("execution_runs", "SELECT COUNT(*), MAX(rowid) FROM execution_runs"),
|
||||
("saved_views", "SELECT MAX(rowid) FROM saved_views"),
|
||||
)
|
||||
|
||||
|
||||
def live_fingerprint(conn: sqlite3.Connection):
|
||||
"""``(summary, fingerprint)`` of a live library, read in one snapshot.
|
||||
|
||||
The fingerprint changes when rows are added to or removed from the
|
||||
tables a restore replaces, or when a prompt, pair, run or saved view is
|
||||
the newest edited (counts plus the newest row and edit of each table);
|
||||
the apply refuses with ``plan-changed`` then. It does not see every
|
||||
edit (a tag or favorite change on an older row); those are kept in the
|
||||
restore's safety backup. A read transaction the caller already has is
|
||||
used and left open; one opened here is ended here.
|
||||
"""
|
||||
import json
|
||||
|
||||
opened = not conn.in_transaction
|
||||
if opened:
|
||||
conn.execute("BEGIN") # one read snapshot for every count
|
||||
try:
|
||||
summary = library_summary(conn)
|
||||
newest = []
|
||||
for table, sql in _FINGERPRINT_SQL:
|
||||
try:
|
||||
newest.append([table, list(conn.execute(sql).fetchone())])
|
||||
except sqlite3.OperationalError:
|
||||
newest.append([table, None]) # no such table or column
|
||||
finally:
|
||||
if opened:
|
||||
conn.rollback() # it only read
|
||||
text = json.dumps([summary, newest], sort_keys=True, default=str)
|
||||
return summary, "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _copy(src_path: str, staged_path: str) -> None:
|
||||
source = _readonly(src_path)
|
||||
try:
|
||||
with closing(sqlite3.connect(staged_path)) as destination:
|
||||
source.backup(destination)
|
||||
finally:
|
||||
source.close()
|
||||
|
||||
|
||||
def _standalone(path: str) -> None:
|
||||
with closing(sqlite3.connect(path)) as conn:
|
||||
conn.execute("PRAGMA journal_mode=DELETE").fetchone()
|
||||
|
||||
|
||||
def stage_restore(model_cls, db_path: str, src_path: str) -> StagedRestore:
|
||||
"""Stage ``src_path`` beside ``db_path`` (blocking); see the module docstring.
|
||||
|
||||
Raises ValueError with a readable reason when the source or the migrated
|
||||
stage does not verify; the stage folder is then already removed.
|
||||
"""
|
||||
ok, reason = model_cls.verify_database_file(src_path, documents=False)
|
||||
if not ok:
|
||||
raise ValueError(reason)
|
||||
before = _schema(src_path)
|
||||
with closing(_readonly(src_path)) as conn:
|
||||
library_state = bool(
|
||||
conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND "
|
||||
"name='prompt_library_state'"
|
||||
).fetchone()
|
||||
)
|
||||
folder = os.path.dirname(os.path.abspath(db_path))
|
||||
directory = tempfile.mkdtemp(prefix=STAGE_PREFIX, dir=folder)
|
||||
staged_path = os.path.join(directory, STAGED_NAME)
|
||||
staged_key = os.path.normcase(os.path.realpath(staged_path))
|
||||
try:
|
||||
_copy(src_path, staged_path)
|
||||
staged = None
|
||||
try:
|
||||
staged = model_cls(staged_path)
|
||||
conn = staged.get_connection()
|
||||
documents = wd.repair_damaged_documents(conn)
|
||||
rotate_history_epoch(conn)
|
||||
summary = library_summary(conn)
|
||||
finally:
|
||||
if staged is not None:
|
||||
staged.close_all()
|
||||
_standalone(staged_path)
|
||||
ok, reason = model_cls.verify_database_file(
|
||||
staged_path, require_library_state=True
|
||||
)
|
||||
if not ok:
|
||||
raise ValueError(reason)
|
||||
result = StagedRestore(
|
||||
path=staged_path,
|
||||
digest=file_digest(staged_path),
|
||||
size=os.path.getsize(staged_path),
|
||||
summary=summary,
|
||||
schema={
|
||||
"migrated": _schema(staged_path) != before,
|
||||
"library_state": library_state,
|
||||
},
|
||||
documents=documents,
|
||||
)
|
||||
except BaseException:
|
||||
remove_stage_folder(directory)
|
||||
raise
|
||||
finally:
|
||||
with model_cls._init_lock:
|
||||
model_cls._initialized_paths.discard(staged_key)
|
||||
return result
|
||||
|
||||
|
||||
def checkpoint_truncate(conn: sqlite3.Connection) -> bool:
|
||||
"""Checkpoint the WAL and truncate it to 0 bytes; False when a reader held it."""
|
||||
previous = conn.execute("PRAGMA busy_timeout").fetchone()[0]
|
||||
try:
|
||||
conn.execute(f"PRAGMA busy_timeout = {int(CHECKPOINT_BUSY_MS)}")
|
||||
busy = conn.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone()[0]
|
||||
finally:
|
||||
conn.execute(f"PRAGMA busy_timeout = {int(previous)}")
|
||||
return not busy
|
||||
|
||||
|
||||
def publish_staged(model, staged: StagedRestore) -> str:
|
||||
"""Publish ``staged`` over ``model``'s database; return the safety backup path.
|
||||
|
||||
"" when there was no database to back up. Raises StageChanged before
|
||||
anything is written when the stage's bytes changed since staging, and
|
||||
SafetyBackupFailed when the current database could not be kept. An
|
||||
``OperationalError`` from the page copy propagates with
|
||||
``_prompt_manager_restore_backup_created`` set; the live data is intact.
|
||||
"""
|
||||
if not os.path.isfile(staged.path) or file_digest(staged.path) != staged.digest:
|
||||
raise StageChanged("The staged restore changed since it was reviewed")
|
||||
backup_path = ""
|
||||
if os.path.exists(model.db_path):
|
||||
backup_path = take_safety_backup(model, verify=False)
|
||||
logger.info(f"Current database backed up to: {os.path.basename(backup_path)}")
|
||||
try:
|
||||
model._replace_contents_from(staged.path)
|
||||
except sqlite3.OperationalError as exc:
|
||||
exc._prompt_manager_restore_backup_created = bool(backup_path)
|
||||
raise
|
||||
with model._init_lock:
|
||||
model._initialized_paths.add(model._schema_key())
|
||||
_truncate_wal(model)
|
||||
return backup_path
|
||||
|
||||
|
||||
def _truncate_wal(model) -> Optional[bool]:
|
||||
"""TRUNCATE-checkpoint the live WAL after a publish (never raises)."""
|
||||
try:
|
||||
conn = model.get_connection()
|
||||
with conn.using():
|
||||
truncated = checkpoint_truncate(conn)
|
||||
except sqlite3.Error as error:
|
||||
logger.warning(f"The database log could not be truncated: {error}")
|
||||
return None
|
||||
if not truncated:
|
||||
logger.warning(
|
||||
"The database log could not be truncated after the restore: a "
|
||||
"reader kept it open. It shrinks at the next checkpoint with no "
|
||||
"reader."
|
||||
)
|
||||
return truncated
|
||||
|
||||
|
||||
def sweep_restore_temps(db_path: str) -> list:
|
||||
"""Remove the stages and uploads an interrupted restore left (startup)."""
|
||||
folder = os.path.dirname(os.path.abspath(db_path))
|
||||
removed = []
|
||||
try:
|
||||
names = os.listdir(folder)
|
||||
except OSError:
|
||||
return removed
|
||||
for name in names:
|
||||
path = os.path.join(folder, name)
|
||||
if name.startswith(UPLOAD_PREFIX) and os.path.isfile(path):
|
||||
try:
|
||||
os.remove(path)
|
||||
removed.append(name)
|
||||
except OSError:
|
||||
continue
|
||||
elif name.startswith(STAGE_PREFIX) and os.path.isdir(path):
|
||||
remove_stage_folder(path)
|
||||
if not os.path.exists(path):
|
||||
removed.append(name)
|
||||
return removed
|
||||
@@ -380,15 +380,21 @@ def _structure_error(conn: sqlite3.Connection) -> Optional[str]:
|
||||
|
||||
|
||||
def _evidence_error(conn: sqlite3.Connection) -> Optional[str]:
|
||||
"""Stream every evidence document once; one row is held at a time."""
|
||||
for evidence_hash, text in conn.execute(
|
||||
"""Stream every evidence document once; one row is held at a time.
|
||||
|
||||
Rows are read with ``fetchone``: iterating a managed connection's cursor
|
||||
would prefetch ``ITER_CHUNK`` (128) rows (``database/managed_connection.py``).
|
||||
"""
|
||||
items = conn.execute(
|
||||
f"SELECT evidence_hash, evidence FROM {ITEMS_TABLE}" # nosec B608
|
||||
):
|
||||
)
|
||||
while (row := items.fetchone()) is not None:
|
||||
evidence_hash, text = row[0], row[1]
|
||||
if not _is_canonical(text) or evidence_digest(text) != evidence_hash:
|
||||
return f"Review queue table {ITEMS_TABLE} contains invalid evidence"
|
||||
for (text,) in conn.execute(
|
||||
f"SELECT evidence FROM {DECISIONS_TABLE}" # nosec B608
|
||||
):
|
||||
decisions = conn.execute(f"SELECT evidence FROM {DECISIONS_TABLE}") # nosec B608
|
||||
while (row := decisions.fetchone()) is not None:
|
||||
text = row[0]
|
||||
if not _is_canonical(text):
|
||||
return f"Review queue table {DECISIONS_TABLE} contains invalid evidence"
|
||||
return None
|
||||
|
||||
@@ -0,0 +1,244 @@
|
||||
"""Attribute an output file to a stored prompt through recorded history (#342).
|
||||
|
||||
Execution history records, per run, the files ComfyUI reported
|
||||
(``execution_run_outputs``) and the PromptManager texts the run used
|
||||
(``execution_run_contexts``, whose ``positive_prompt_id`` hint is set only
|
||||
while the raw text matches exactly one stored prompt). A file is attributed
|
||||
only when all of this holds:
|
||||
|
||||
- exactly one recorded run reported the file's output location (same output
|
||||
root, subfolder and name);
|
||||
- the file was last written while that run was executing, from its dequeue
|
||||
to its terminal event (else the finish callback): a later run that history
|
||||
did not record may have reused the name;
|
||||
- every positive context of that run names the same stored prompt.
|
||||
|
||||
Anything else returns None: the caller leaves the file unlinked. Read-only.
|
||||
|
||||
Lookups are batched (``stored_run_prompts``): one parameterized query per
|
||||
chunk of at most ``LOOKUP_CHUNK`` distinct file names, matched in memory. The
|
||||
history schema is left exactly as it is (no extra index), so older builds can
|
||||
still open the library.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
import threading
|
||||
import weakref
|
||||
from typing import Any, Callable, Dict, Iterable, List, Optional
|
||||
|
||||
try:
|
||||
from ..utils.execution_outputs import snapshot_output_roots
|
||||
from ..utils.library_location import import_folder_paths
|
||||
except ImportError:
|
||||
from utils.execution_outputs import snapshot_output_roots
|
||||
from utils.library_location import import_folder_paths
|
||||
|
||||
# File times may trail or lead the recorded run times a little (coarse file
|
||||
# system clocks, the write finishing just after the host reported it).
|
||||
MTIME_SLACK_MS = 2000
|
||||
|
||||
# Distinct file names per lookup query: well under SQLite's bound-parameter
|
||||
# limit (999 on old builds), so a scan batch costs one query per chunk.
|
||||
LOOKUP_CHUNK = 500
|
||||
|
||||
_RUNS_FOR_NAMES = (
|
||||
"SELECT DISTINCT o.filename, o.subfolder, o.root_key, o.path_style, o.run_id, "
|
||||
"r.observed_dequeue_at, COALESCE(r.terminal_event_at, r.observed_callback_at) "
|
||||
"FROM execution_run_outputs o JOIN execution_runs r ON r.run_id = o.run_id "
|
||||
"WHERE o.folder_type = 'output' AND o.filename IN ({marks})"
|
||||
)
|
||||
|
||||
|
||||
def host_output_directory() -> Optional[str]:
|
||||
"""ComfyUI's output directory, or None outside ComfyUI."""
|
||||
folder_paths = import_folder_paths()
|
||||
if folder_paths is None:
|
||||
return None
|
||||
try:
|
||||
value = folder_paths.get_output_directory()
|
||||
except Exception:
|
||||
return None
|
||||
return value if isinstance(value, str) and value else None
|
||||
|
||||
|
||||
def _inside(image_path: str, output_root: str) -> Optional[str]:
|
||||
"""``image_path`` relative to ``output_root``, or None when outside it."""
|
||||
try:
|
||||
relative = os.path.relpath(image_path, output_root)
|
||||
except ValueError: # another drive on Windows
|
||||
return None
|
||||
if os.path.isabs(relative) or relative == os.pardir:
|
||||
return None
|
||||
if relative.startswith(os.pardir + os.sep):
|
||||
return None
|
||||
return relative
|
||||
|
||||
|
||||
def _resolved(path: str) -> str:
|
||||
try:
|
||||
return os.path.realpath(path)
|
||||
except (OSError, ValueError):
|
||||
return os.path.abspath(path)
|
||||
|
||||
|
||||
def _location(image_path: str, output_root: str, path_style: str):
|
||||
"""(root key, subfolder, filename) as history records them, or None.
|
||||
|
||||
The root key is always the host's own spelling of its output folder,
|
||||
because history recorded that one. The file may be spelled through the
|
||||
same folder or through its resolved path (a symlinked or junction output
|
||||
folder, as scan roots are): the lexical spelling is tried first, then
|
||||
both sides resolved.
|
||||
"""
|
||||
root_key = snapshot_output_roots(path_style, output_root, output_root).output_key
|
||||
if root_key is None:
|
||||
return None
|
||||
relative = _inside(os.path.abspath(image_path), os.path.abspath(output_root))
|
||||
if relative is None:
|
||||
relative = _inside(_resolved(image_path), _resolved(output_root))
|
||||
if relative is None:
|
||||
return None
|
||||
subfolder, filename = os.path.split(relative)
|
||||
if not filename:
|
||||
return None
|
||||
return root_key, subfolder, filename
|
||||
|
||||
|
||||
def in_output_folder(image_path: str, output_root: Optional[str]) -> bool:
|
||||
"""Whether history could ever attribute ``image_path`` (inside the host's
|
||||
output folder, in either spelling)."""
|
||||
if not image_path or not output_root:
|
||||
return False
|
||||
style = "windows" if os.name == "nt" else "posix"
|
||||
return _location(image_path, output_root, style) is not None
|
||||
|
||||
|
||||
def _written_during(image_path: str, started: Any, finished: Any) -> bool:
|
||||
if not isinstance(started, int) or not isinstance(finished, int):
|
||||
return False
|
||||
try:
|
||||
written = os.stat(image_path).st_mtime_ns // 1_000_000
|
||||
except OSError:
|
||||
return False
|
||||
return started - MTIME_SLACK_MS <= written <= finished + MTIME_SLACK_MS
|
||||
|
||||
|
||||
def stored_run_prompt(
|
||||
db: Any,
|
||||
image_path: str,
|
||||
output_root: Optional[str],
|
||||
path_style: Optional[str] = None,
|
||||
) -> Optional[int]:
|
||||
"""The stored prompt id history attributes to ``image_path``, else None."""
|
||||
if not image_path:
|
||||
return None
|
||||
return stored_run_prompts(db, [image_path], output_root, path_style).get(image_path)
|
||||
|
||||
|
||||
def _runs_by_location(db: Any, names: List[str]) -> Optional[Dict[tuple, dict]]:
|
||||
"""``{(filename, subfolder, root_key, style): {run_id: (start, end)}}``.
|
||||
|
||||
One query per ``LOOKUP_CHUNK`` names; None without history tables.
|
||||
"""
|
||||
found: Dict[tuple, dict] = {}
|
||||
conn = db.model.get_connection()
|
||||
for start in range(0, len(names), LOOKUP_CHUNK):
|
||||
chunk = names[start : start + LOOKUP_CHUNK]
|
||||
query = _RUNS_FOR_NAMES.format(marks=",".join("?" for _ in chunk))
|
||||
try:
|
||||
rows = conn.execute(query, chunk).fetchall() # nosec B608
|
||||
except sqlite3.Error:
|
||||
return None # no history tables (a library without history)
|
||||
for name, subfolder, root_key, style, run_id, began, ended in rows:
|
||||
runs = found.setdefault((name, subfolder, root_key, style), {})
|
||||
runs[run_id] = (began, ended)
|
||||
return found
|
||||
|
||||
|
||||
def _named_prompt(db: Any, run_id: str) -> Optional[int]:
|
||||
contexts = db.get_execution_contexts(run_id).get("contexts") or []
|
||||
named = {
|
||||
context.get("positive_prompt_id")
|
||||
for context in contexts
|
||||
if context.get("positive") is not None
|
||||
}
|
||||
if len(named) != 1:
|
||||
return None
|
||||
(prompt_id,) = named
|
||||
return prompt_id if isinstance(prompt_id, int) else None
|
||||
|
||||
|
||||
def stored_run_prompts(
|
||||
db: Any,
|
||||
image_paths: Iterable[str],
|
||||
output_root: Optional[str],
|
||||
path_style: Optional[str] = None,
|
||||
) -> Dict[str, Optional[int]]:
|
||||
"""``{path: stored prompt id or None}`` for many files in few queries.
|
||||
|
||||
The output lookup is one query per ``LOOKUP_CHUNK`` distinct names; each
|
||||
run that alone reported a file is then read once for its contexts.
|
||||
"""
|
||||
paths = list(dict.fromkeys(path for path in image_paths if path))
|
||||
result: Dict[str, Optional[int]] = dict.fromkeys(paths)
|
||||
if not paths or not output_root:
|
||||
return result
|
||||
style = path_style or ("windows" if os.name == "nt" else "posix")
|
||||
located = {}
|
||||
for path in paths:
|
||||
location = _location(path, output_root, style)
|
||||
if location is not None:
|
||||
located[path] = location
|
||||
if not located:
|
||||
return result
|
||||
names = sorted({filename for _key, _sub, filename in located.values()})
|
||||
runs = _runs_by_location(db, names)
|
||||
if not runs:
|
||||
return result
|
||||
prompts: Dict[str, Optional[int]] = {}
|
||||
for path, (root_key, subfolder, filename) in located.items():
|
||||
reported = runs.get((filename, subfolder, root_key, style), {})
|
||||
if len(reported) != 1:
|
||||
continue
|
||||
((run_id, (began, ended)),) = reported.items()
|
||||
if not _written_during(path, began, ended):
|
||||
continue
|
||||
if run_id not in prompts:
|
||||
prompts[run_id] = _named_prompt(db, run_id)
|
||||
result[path] = prompts[run_id]
|
||||
return result
|
||||
|
||||
|
||||
_LOGGER = logging.getLogger("prompt_manager.run_attribution")
|
||||
_SEALED_LISTENERS: List[weakref.WeakMethod] = []
|
||||
_SEALED_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def subscribe_outputs_sealed(method: Callable[[str, List[dict]], None]) -> None:
|
||||
"""Call bound ``method(run_id, references)`` after a run's outputs are recorded.
|
||||
|
||||
Held weakly: a listener that is garbage collected is dropped.
|
||||
"""
|
||||
with _SEALED_LOCK:
|
||||
_SEALED_LISTENERS.append(weakref.WeakMethod(method))
|
||||
|
||||
|
||||
def notify_outputs_sealed(run_id: str, references: Iterable[dict]) -> None:
|
||||
"""Tell listeners a run's output references were just committed.
|
||||
|
||||
A listener failure is logged and never reaches the history writer.
|
||||
"""
|
||||
references = list(references)
|
||||
with _SEALED_LOCK:
|
||||
alive = [ref for ref in _SEALED_LISTENERS if ref() is not None]
|
||||
_SEALED_LISTENERS[:] = alive
|
||||
listeners = [ref() for ref in alive]
|
||||
for listener in listeners:
|
||||
if listener is None:
|
||||
continue
|
||||
try:
|
||||
listener(run_id, references)
|
||||
except Exception:
|
||||
_LOGGER.warning("Output seal listener failed", exc_info=True)
|
||||
@@ -0,0 +1,185 @@
|
||||
"""Safety backups beside the live database: one writer for every caller.
|
||||
|
||||
A safety backup is ``<db>.<label>_<YYYYmmdd_HHMMSS_ffffff>[_n]`` in the
|
||||
library's own folder. It is written by SQLite's backup API into a temp file
|
||||
beside the database (``SAFETY_TEMP_PREFIX``, swept at startup after a crash),
|
||||
made one self-contained file (rollback journal), optionally verified, fsynced,
|
||||
made private (``0o600`` on POSIX) and only then given its unique name, so a
|
||||
backup name never carries a half-written file. The names are unique within
|
||||
one second (microseconds, then a counter): on Windows two backups in the same
|
||||
second must not overwrite each other.
|
||||
|
||||
Maintenance applies take a verified backup (``py/api/maintenance.py``); a
|
||||
restore takes an unverified one, because the database it replaces may be the
|
||||
damaged one the user is recovering from (``database/restore_stage.py``).
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import os
|
||||
import sqlite3
|
||||
import tempfile
|
||||
from contextlib import closing
|
||||
|
||||
SAFETY_TEMP_PREFIX = ".pm-safety-"
|
||||
_NAME_ATTEMPTS = 1000
|
||||
|
||||
|
||||
class SafetyBackupFailed(Exception):
|
||||
"""The safety backup could not be written, verified or put in place.
|
||||
|
||||
``reason`` is the verify reason when the copy did not verify.
|
||||
"""
|
||||
|
||||
def __init__(self, message, reason=None):
|
||||
super().__init__(message)
|
||||
self.reason = reason
|
||||
|
||||
|
||||
def _now():
|
||||
return datetime.datetime.now()
|
||||
|
||||
|
||||
def remove_quietly(*paths):
|
||||
"""Remove each existing path, ignoring errors (cleanup after a failure)."""
|
||||
for path in paths:
|
||||
try:
|
||||
if path and os.path.exists(path):
|
||||
os.remove(path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def backup_names(db_path, label="backup"):
|
||||
"""Candidate names, unique within one second (microseconds, then ``_n``)."""
|
||||
base = f"{db_path}.{label}_{_now().strftime('%Y%m%d_%H%M%S_%f')}"
|
||||
for attempt in range(_NAME_ATTEMPTS):
|
||||
yield base if attempt == 0 else f"{base}_{attempt}"
|
||||
|
||||
|
||||
def reserve_backup_name(db_path, label="backup"):
|
||||
"""Create and return an unused ``<db>.<label>_<stamp>[_n]`` (O_EXCL)."""
|
||||
for candidate in backup_names(db_path, label):
|
||||
try:
|
||||
fd = os.open(candidate, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
|
||||
except FileExistsError:
|
||||
continue
|
||||
os.close(fd)
|
||||
return candidate
|
||||
raise SafetyBackupFailed("No free backup name")
|
||||
|
||||
|
||||
def place_backup(temp, db_path, label="backup"):
|
||||
"""Give the finished ``temp`` its unique backup name; return that path.
|
||||
|
||||
``os.link`` is atomic and refuses an existing name, so no empty or half
|
||||
file ever carries a backup name, even across a crash. A filesystem
|
||||
without hard links falls back to reserving the name (``O_EXCL``) and
|
||||
``os.replace``. The temp file is removed by the caller.
|
||||
"""
|
||||
for candidate in backup_names(db_path, label):
|
||||
try:
|
||||
os.link(temp, candidate)
|
||||
except FileExistsError:
|
||||
continue
|
||||
except (OSError, NotImplementedError):
|
||||
break # no hard links here
|
||||
else:
|
||||
return candidate
|
||||
else:
|
||||
raise SafetyBackupFailed("No free backup name")
|
||||
final = reserve_backup_name(db_path, label)
|
||||
try:
|
||||
os.replace(temp, final)
|
||||
except OSError:
|
||||
remove_quietly(final)
|
||||
raise
|
||||
return final
|
||||
|
||||
|
||||
def sweep_safety_temps(db_path):
|
||||
"""Remove the temp copies an interrupted safety backup left (blocking).
|
||||
|
||||
Called once at startup, before any job can take a backup. Returns the
|
||||
names removed.
|
||||
"""
|
||||
folder = os.path.dirname(os.path.abspath(db_path))
|
||||
removed = []
|
||||
try:
|
||||
names = os.listdir(folder)
|
||||
except OSError:
|
||||
return removed
|
||||
for name in names:
|
||||
if not (name.startswith(SAFETY_TEMP_PREFIX) and ".tmp" in name):
|
||||
continue
|
||||
try:
|
||||
os.remove(os.path.join(folder, name))
|
||||
removed.append(name)
|
||||
except OSError:
|
||||
continue
|
||||
return removed
|
||||
|
||||
|
||||
def make_standalone(path):
|
||||
"""Make the copy one self-contained file (rollback journal, not WAL).
|
||||
|
||||
The backup API copies the live database's WAL flag; a WAL file opened
|
||||
read-only later (a verify, a restore preview) would leave ``-wal`` and
|
||||
``-shm`` sidecars beside it.
|
||||
"""
|
||||
try:
|
||||
with closing(sqlite3.connect(path)) as conn:
|
||||
conn.execute("PRAGMA journal_mode=DELETE").fetchone()
|
||||
except sqlite3.Error as exc:
|
||||
raise SafetyBackupFailed("The backup could not be finished") from exc
|
||||
|
||||
|
||||
def fsync_file(path):
|
||||
with open(path, "rb+") as handle:
|
||||
os.fsync(handle.fileno())
|
||||
|
||||
|
||||
def safety_temp(db_path):
|
||||
"""A new, closed temp file beside the database (``SAFETY_TEMP_PREFIX``)."""
|
||||
fd, temp = tempfile.mkstemp(
|
||||
dir=os.path.dirname(os.path.abspath(db_path)),
|
||||
prefix=SAFETY_TEMP_PREFIX,
|
||||
suffix=".tmp",
|
||||
)
|
||||
os.close(fd)
|
||||
return temp
|
||||
|
||||
|
||||
def take_safety_backup(model, label="backup", *, verify=True):
|
||||
"""Write (and by default verify) a safety backup; return its path (blocking).
|
||||
|
||||
The copy is made in a temp file beside the database and only renamed
|
||||
onto its final, unique name ``<db>.<label>_<stamp>`` once it is complete
|
||||
(and verified), so a backup file on disk is always a whole one. ``label``
|
||||
"backup" joins the pruned pool; another label (the workflow storage
|
||||
migration's ``pre-workflow-storage``) is kept until the user removes it.
|
||||
Raises SafetyBackupFailed, leaving no file.
|
||||
"""
|
||||
db_path = os.path.abspath(model.db_path)
|
||||
temp = safety_temp(db_path)
|
||||
final = None
|
||||
try:
|
||||
if not model.backup_database(temp):
|
||||
raise SafetyBackupFailed("The backup could not be written")
|
||||
make_standalone(temp)
|
||||
if verify:
|
||||
ok, reason = model.verify_database_file(temp)
|
||||
if not ok:
|
||||
raise SafetyBackupFailed("The backup did not verify", reason)
|
||||
fsync_file(temp)
|
||||
if os.name != "nt":
|
||||
os.chmod(temp, 0o600)
|
||||
final = place_backup(temp, db_path, label)
|
||||
return final
|
||||
except OSError as exc:
|
||||
remove_quietly(final)
|
||||
raise SafetyBackupFailed("The backup could not be put in place") from exc
|
||||
except SafetyBackupFailed:
|
||||
remove_quietly(final)
|
||||
raise
|
||||
finally:
|
||||
remove_quietly(temp, temp + "-journal", temp + "-wal", temp + "-shm")
|
||||
@@ -475,3 +475,178 @@ def release_documents(conn: sqlite3.Connection, ids: Iterable[Optional[int]]) ->
|
||||
def sweep_documents(conn: sqlite3.Connection) -> int:
|
||||
"""Delete every document no image row references; returns the count."""
|
||||
return conn.execute(SWEEP_SQL).rowcount
|
||||
|
||||
|
||||
# ── repair (a restore's stage) ────────────────────────────────────────
|
||||
|
||||
INLINE_TEXTS_SQL = (
|
||||
"SELECT workflow_data, prompt_metadata FROM generated_images "
|
||||
"WHERE workflow_data IS NOT NULL OR prompt_metadata IS NOT NULL"
|
||||
)
|
||||
REKEY_SQL = "UPDATE workflow_documents SET sha256=?, byte_length=? WHERE id=?"
|
||||
|
||||
|
||||
def _rows(conn: sqlite3.Connection, sql: str, params=()):
|
||||
"""Each row of ``sql``, read one at a time.
|
||||
|
||||
Iterating a managed connection's cursor prefetches ``ITER_CHUNK`` (128)
|
||||
rows, and a document or inline text can be megabytes.
|
||||
"""
|
||||
cursor = conn.execute(sql, params)
|
||||
while (row := cursor.fetchone()) is not None:
|
||||
yield row
|
||||
|
||||
|
||||
def _inflated(data) -> Optional[bytes]:
|
||||
"""The complete text bytes ``data`` inflates to, or None.
|
||||
|
||||
Complete means the zlib stream ends (its Adler-32 checked) within
|
||||
MAX_DOCUMENT_BYTES with nothing after it: the body is intact even when
|
||||
the row's recorded hash or length is not.
|
||||
"""
|
||||
try:
|
||||
inflater = zlib.decompressobj()
|
||||
raw = inflater.decompress(bytes(data), MAX_DOCUMENT_BYTES + 1)
|
||||
if not inflater.eof or inflater.unconsumed_tail or inflater.unused_data:
|
||||
return None
|
||||
if len(raw) > MAX_DOCUMENT_BYTES:
|
||||
return None
|
||||
raw.decode("utf-8")
|
||||
return raw
|
||||
except (zlib.error, TypeError, UnicodeDecodeError):
|
||||
return None
|
||||
|
||||
|
||||
def _one_column_off(row, raw: bytes) -> bool:
|
||||
"""Whether only the recorded hash or only the recorded length is wrong.
|
||||
|
||||
The two columns check the body independently. When the body inflates
|
||||
completely and one of them still agrees with it, that column vouches for
|
||||
the body and the other one is the damaged value. When both disagree, the
|
||||
body itself is the suspect (another valid stream), so it is not trusted.
|
||||
"""
|
||||
hash_ok = hashlib.sha256(raw).digest() == bytes(row[0])
|
||||
length_ok = row[1] == len(raw)
|
||||
return hash_ok != length_ok
|
||||
|
||||
|
||||
def _inline_copies(conn: sqlite3.Connection, wanted: Dict[bytes, int]):
|
||||
"""``{document id: text}`` for each wanted hash an inline row still holds."""
|
||||
found: Dict[int, str] = {}
|
||||
for row in _rows(conn, INLINE_TEXTS_SQL):
|
||||
for text in row:
|
||||
if not isinstance(text, str) or not storable(text):
|
||||
continue
|
||||
document_id = wanted.get(hashlib.sha256(text.encode("utf-8")).digest())
|
||||
if document_id is not None and document_id not in found:
|
||||
found[document_id] = text
|
||||
if len(found) == len(wanted):
|
||||
break
|
||||
return found
|
||||
|
||||
|
||||
def _rekey(conn: sqlite3.Connection, document_id: int, raw: bytes) -> Optional[int]:
|
||||
"""Give an intact body its true key, or merge it into the document that has it.
|
||||
|
||||
Returns the holder's id when its own body was damaged and has just been
|
||||
rewritten from ``raw`` (it is repaired, never to be cleared), else None.
|
||||
"""
|
||||
digest = hashlib.sha256(raw).digest()
|
||||
holder = conn.execute(LOOKUP_SQL, (digest,)).fetchone()
|
||||
if holder is None or holder[0] == document_id:
|
||||
conn.execute(REKEY_SQL, (digest, len(raw), document_id))
|
||||
return None
|
||||
for column in DOCUMENT_ID_COLUMNS:
|
||||
# ``column`` is one of DOCUMENT_ID_COLUMNS, never caller data.
|
||||
conn.execute(
|
||||
f"UPDATE generated_images SET {column}=? WHERE {column}=?", # nosec B608
|
||||
(holder[0], document_id),
|
||||
)
|
||||
conn.execute("DELETE FROM workflow_documents WHERE id=?", (document_id,))
|
||||
if _decode(holder[1:], holder[0], quiet=True) is None:
|
||||
conn.execute(
|
||||
REPAIR_SQL, (len(raw), zlib.compress(raw, COMPRESSION_LEVEL), holder[0])
|
||||
)
|
||||
return holder[0]
|
||||
return None
|
||||
|
||||
|
||||
def repair_damaged_documents(conn: sqlite3.Connection) -> Dict[str, int]:
|
||||
"""Repair or clear every document failing its checks; commit (writes).
|
||||
|
||||
For a restore's stage only, never the live library. In order:
|
||||
|
||||
1. A document whose body inflates completely and whose recorded hash or
|
||||
length (not both) disagrees is re-keyed from its text, or merged
|
||||
into the document that already holds that text (``_one_column_off``).
|
||||
2. A damaged body whose exact text an inline row still holds (the same
|
||||
SHA-256, its key) is stored again from that text.
|
||||
3. Any other damaged document is unreadable for good: its images'
|
||||
references are cleared (they already showed no workflow) and the
|
||||
document is removed, so the stage verifies.
|
||||
|
||||
Returns ``{repaired, cleared, images}``: documents repaired (1 and 2),
|
||||
documents cleared and the distinct image rows whose reference was
|
||||
cleared. Rows are read one at a time.
|
||||
"""
|
||||
damaged: Dict[bytes, int] = {}
|
||||
intact: Dict[int, bytes] = {}
|
||||
for row in _rows(conn, ALL_DOCUMENTS_SQL):
|
||||
if _decode(row[:3], row[3], quiet=True) is None:
|
||||
raw = _inflated(row[2])
|
||||
if raw is not None and _one_column_off(row, raw):
|
||||
intact[row[3]] = raw
|
||||
else:
|
||||
damaged[bytes(row[0])] = row[3]
|
||||
result = {"repaired": 0, "cleared": 0, "images": 0}
|
||||
if not damaged and not intact:
|
||||
return result
|
||||
with conn:
|
||||
healed = []
|
||||
for document_id, raw in intact.items():
|
||||
holder = _rekey(conn, document_id, raw)
|
||||
if holder is not None and holder in damaged.values():
|
||||
# Rewritten from the merged body: repaired, not to be cleared
|
||||
# (it now also holds the merged images' references).
|
||||
damaged = {k: v for k, v in damaged.items() if v != holder}
|
||||
healed.append(holder)
|
||||
copies = _inline_copies(conn, damaged) if damaged else {}
|
||||
for document_id, text in copies.items():
|
||||
raw = text.encode("utf-8")
|
||||
data = zlib.compress(raw, COMPRESSION_LEVEL)
|
||||
conn.execute(REPAIR_SQL, (len(raw), data, document_id))
|
||||
lost = sorted(set(damaged.values()) - set(copies))
|
||||
images: Set[int] = set()
|
||||
for chunk in _chunks(lost):
|
||||
marks = ",".join("?" * len(chunk))
|
||||
# ``marks`` is only "?" placeholders; the ids are bound.
|
||||
for (image_id,) in _rows(
|
||||
conn,
|
||||
"SELECT id FROM generated_images WHERE " # nosec B608
|
||||
f"workflow_document_id IN ({marks}) OR prompt_document_id IN ({marks})",
|
||||
chunk + chunk,
|
||||
):
|
||||
images.add(image_id)
|
||||
for column in DOCUMENT_ID_COLUMNS:
|
||||
conn.execute(
|
||||
f"UPDATE generated_images SET {column}=NULL " # nosec B608
|
||||
f"WHERE {column} IN ({marks})",
|
||||
chunk,
|
||||
)
|
||||
conn.execute(
|
||||
f"DELETE FROM workflow_documents WHERE id IN ({marks})", # nosec B608
|
||||
chunk,
|
||||
)
|
||||
for document_id in list(intact) + healed + list(copies):
|
||||
logger.warning(f"Workflow document {document_id} was damaged; repaired")
|
||||
if lost:
|
||||
logger.warning(
|
||||
f"{len(lost)} damaged workflow document(s) could not be repaired; "
|
||||
f"{len(images)} image(s) show no workflow"
|
||||
)
|
||||
result.update(
|
||||
repaired=len(intact) + len(healed) + len(copies),
|
||||
cleared=len(lost),
|
||||
images=len(images),
|
||||
)
|
||||
return result
|
||||
|
||||
@@ -238,6 +238,9 @@ class PromptManager(PromptManagerBase, ComfyNodeABC):
|
||||
except Exception as e:
|
||||
# Log error but don't fail the encoding
|
||||
self.logger.warning(f"Failed to save prompt to database: {e}")
|
||||
self._mark_untracked(prompt, unique_id)
|
||||
else:
|
||||
self._skip_empty_prompt(prompt, unique_id)
|
||||
|
||||
# Perform standard CLIP text encoding using the combined text
|
||||
self.logger.debug(
|
||||
|
||||
+56
-1
@@ -24,6 +24,7 @@ try:
|
||||
from .utils.prompt_tracker import get_prompt_tracker
|
||||
from .utils.hashing import generate_prompt_hash
|
||||
from .utils.prompt_graph import resolve_text, run_prompt_nodes
|
||||
from .utils.run_keys import executing_run_id, prompt_node_key
|
||||
from .utils.usage_tracking import PENDING_FIRST_USE
|
||||
except ImportError:
|
||||
import sys
|
||||
@@ -35,6 +36,7 @@ except ImportError:
|
||||
from utils.prompt_tracker import get_prompt_tracker
|
||||
from utils.hashing import generate_prompt_hash
|
||||
from utils.prompt_graph import resolve_text, run_prompt_nodes
|
||||
from utils.run_keys import executing_run_id, prompt_node_key
|
||||
from utils.usage_tracking import PENDING_FIRST_USE
|
||||
|
||||
|
||||
@@ -168,14 +170,67 @@ class PromptManagerBase:
|
||||
prompt_id = self._save_prompt_to_database(
|
||||
text=text, category=category, tags=tags, count_run=batch_item
|
||||
)
|
||||
if prompt_id and role["is_run_prompt"]:
|
||||
if role["is_run_prompt"] and not prompt_id:
|
||||
# The save failed: an older entry of this key must not stand in (#342)
|
||||
self._mark_untracked(prompt_graph, unique_id)
|
||||
elif role["is_run_prompt"]:
|
||||
self.prompt_tracker.set_current_prompt(
|
||||
prompt_text=encoding_text,
|
||||
additional_data={**additional_data, "prompt_id": prompt_id},
|
||||
push_to_queue=batch_item or not role["known"],
|
||||
**self._run_identity(prompt_graph, unique_id),
|
||||
)
|
||||
return prompt_id
|
||||
|
||||
@staticmethod
|
||||
def _run_identity(prompt_graph: Any, unique_id: Any) -> Dict[str, Any]:
|
||||
"""The tracker's proof that an image belongs to this execution (#342)."""
|
||||
return {
|
||||
"run_key": prompt_node_key(prompt_graph, unique_id),
|
||||
"run_id": executing_run_id(),
|
||||
}
|
||||
|
||||
def _skip_empty_prompt(self, prompt_graph: Any, unique_id: Any) -> None:
|
||||
"""The ``text`` editor is empty: save and track nothing (#342).
|
||||
|
||||
Prepend and append text are deliberately never stored as the prompt, so
|
||||
an empty editor has no prompt to save, count or link. For the run's
|
||||
positive node this is logged at info and the node's run key is marked
|
||||
empty (``_mark_untracked``).
|
||||
"""
|
||||
self._mark_untracked(
|
||||
prompt_graph,
|
||||
unique_id,
|
||||
"Prompt text is empty (prepend and append text are not stored): "
|
||||
"nothing saved or tracked, and this run's images stay unlinked",
|
||||
)
|
||||
|
||||
def _mark_untracked(
|
||||
self, prompt_graph: Any, unique_id: Any, message: Optional[str] = None
|
||||
) -> None:
|
||||
"""This execution of a positive node has no prompt to link (#342).
|
||||
|
||||
An empty editor, or a save or tracking failure. The node's run key is
|
||||
marked with a placeholder, so no earlier prompt of that key can be
|
||||
linked to this run's images, and a batch item keeps its queue place.
|
||||
``message`` is logged at info for a positive node.
|
||||
"""
|
||||
try:
|
||||
role = self._prompt_role(prompt_graph, unique_id, "")
|
||||
if not role["is_run_prompt"]:
|
||||
if message:
|
||||
self.logger.debug("Empty text on a non-positive node; not saved")
|
||||
return
|
||||
if message:
|
||||
self.logger.info(message)
|
||||
self.prompt_tracker.mark_empty(
|
||||
push_to_queue=role["known"],
|
||||
**self._run_identity(prompt_graph, unique_id),
|
||||
)
|
||||
except Exception as error:
|
||||
# Tracking must never block generation
|
||||
self.logger.warning(f"Could not mark the untracked prompt: {error}")
|
||||
|
||||
def _record_use(self, prompt_id: int, times: int = 1) -> None:
|
||||
"""Count runs; a tracking failure must never block generation."""
|
||||
try:
|
||||
|
||||
+117
-77
@@ -117,85 +117,29 @@ class PromptManagerPair(PromptManagerFocused):
|
||||
)
|
||||
is_positive = "text" in used_fields
|
||||
pair_used = node_id in intrinsic_pair_nodes(prompt) if known else True
|
||||
try:
|
||||
pair_id = self.db.save_prompt_pair(
|
||||
text,
|
||||
negative_text,
|
||||
category=category.strip() or None,
|
||||
tags=self._parse_tags(tags),
|
||||
)
|
||||
pair = self.db.get_prompt_pair(pair_id)
|
||||
prompt_id = pair["positive_prompt_id"]
|
||||
resolved_positive = (
|
||||
resolve_text_input(prompt, node_id, "text") if known else None
|
||||
)
|
||||
resolved_negative = (
|
||||
resolve_text_input(prompt, node_id, "negative_text") if known else None
|
||||
)
|
||||
positive_seen = resolved_positive is not None and generate_prompt_hash(
|
||||
resolved_positive
|
||||
) == generate_prompt_hash(text)
|
||||
pair_seen = (
|
||||
positive_seen
|
||||
and resolved_negative is not None
|
||||
and generate_prompt_hash(resolved_negative)
|
||||
== generate_prompt_hash(negative_text)
|
||||
)
|
||||
positive_pending = getattr(self, "pending_first_use", PENDING_FIRST_USE)
|
||||
pair_pending = getattr(
|
||||
self, "pending_pair_first_use", PENDING_PAIR_FIRST_USE
|
||||
)
|
||||
counted = set()
|
||||
for field, component_text, resolved_text, component_id in (
|
||||
("text", text, resolved_positive, prompt_id),
|
||||
(
|
||||
"negative_text",
|
||||
if not text.strip():
|
||||
# Nothing to store: prepend/append are never the stored prompt (#342)
|
||||
self._skip_empty_prompt(prompt, unique_id)
|
||||
else:
|
||||
try:
|
||||
pair_id, prompt_id = self._save_and_track_pair(
|
||||
text,
|
||||
negative_text,
|
||||
resolved_negative,
|
||||
pair["negative_prompt_id"],
|
||||
),
|
||||
):
|
||||
component_hash = generate_prompt_hash(component_text)
|
||||
if (
|
||||
field not in used_fields
|
||||
or not component_id
|
||||
or component_hash in counted
|
||||
):
|
||||
continue
|
||||
counted.add(component_hash)
|
||||
seen_at_queue = (
|
||||
resolved_text is not None
|
||||
and generate_prompt_hash(resolved_text) == component_hash
|
||||
positive,
|
||||
category=category,
|
||||
tags=tags,
|
||||
prepend_text=prepend_text,
|
||||
append_text=append_text,
|
||||
prompt=prompt,
|
||||
unique_id=unique_id,
|
||||
used_fields=used_fields,
|
||||
is_positive=is_positive,
|
||||
pair_used=pair_used,
|
||||
)
|
||||
runs = positive_pending.consume(component_hash) + int(not seen_at_queue)
|
||||
if runs:
|
||||
self.db.record_prompt_use(component_id, times=runs)
|
||||
if pair_used:
|
||||
runs = pair_pending.consume(pair_run_key(text, negative_text)) + int(
|
||||
not pair_seen
|
||||
)
|
||||
if runs:
|
||||
self.db.record_prompt_pair_use(pair_id, times=runs)
|
||||
if is_positive:
|
||||
self.prompt_tracker.set_current_prompt(
|
||||
prompt_text=positive,
|
||||
additional_data={
|
||||
"prompt_id": prompt_id,
|
||||
"pair_id": pair_id if pair_used else None,
|
||||
"positive_text": positive,
|
||||
"negative_text": negative_text,
|
||||
"positive_storage_text": text.strip(),
|
||||
"negative_storage_text": negative_text.strip(),
|
||||
"prepend_text": prepend_text.strip(),
|
||||
"append_text": append_text.strip(),
|
||||
"final_text": positive,
|
||||
"category": category.strip() or None,
|
||||
"tags": self._parse_tags(tags) or [],
|
||||
},
|
||||
push_to_queue=not positive_seen or (pair_used and not pair_seen),
|
||||
)
|
||||
except Exception as error:
|
||||
self.logger.warning(f"Could not save or track paired prompt: {error}")
|
||||
except Exception as error:
|
||||
self.logger.warning(f"Could not save or track paired prompt: {error}")
|
||||
if is_positive:
|
||||
self._mark_untracked(prompt, unique_id)
|
||||
try:
|
||||
self.comfyui_integration.register_prompt(
|
||||
f"promptmanager_pair_{time.time_ns()}",
|
||||
@@ -223,6 +167,102 @@ class PromptManagerPair(PromptManagerFocused):
|
||||
)
|
||||
return result
|
||||
|
||||
def _save_and_track_pair(
|
||||
self,
|
||||
text: str,
|
||||
negative_text: str,
|
||||
positive: str,
|
||||
*,
|
||||
category: str,
|
||||
tags: str,
|
||||
prepend_text: str,
|
||||
append_text: str,
|
||||
prompt: Optional[Dict[str, Any]],
|
||||
unique_id: Optional[str],
|
||||
used_fields: set,
|
||||
is_positive: bool,
|
||||
pair_used: bool,
|
||||
) -> Tuple[Optional[int], Optional[int]]:
|
||||
"""Save the recipe, count its runs and register the positive for linking."""
|
||||
known = isinstance(prompt, dict) and unique_id is not None
|
||||
node_id = str(unique_id)
|
||||
pair_id = self.db.save_prompt_pair(
|
||||
text,
|
||||
negative_text,
|
||||
category=category.strip() or None,
|
||||
tags=self._parse_tags(tags),
|
||||
)
|
||||
pair = self.db.get_prompt_pair(pair_id)
|
||||
prompt_id = pair["positive_prompt_id"]
|
||||
resolved_positive = (
|
||||
resolve_text_input(prompt, node_id, "text") if known else None
|
||||
)
|
||||
resolved_negative = (
|
||||
resolve_text_input(prompt, node_id, "negative_text") if known else None
|
||||
)
|
||||
positive_seen = resolved_positive is not None and generate_prompt_hash(
|
||||
resolved_positive
|
||||
) == generate_prompt_hash(text)
|
||||
pair_seen = (
|
||||
positive_seen
|
||||
and resolved_negative is not None
|
||||
and generate_prompt_hash(resolved_negative)
|
||||
== generate_prompt_hash(negative_text)
|
||||
)
|
||||
positive_pending = getattr(self, "pending_first_use", PENDING_FIRST_USE)
|
||||
pair_pending = getattr(self, "pending_pair_first_use", PENDING_PAIR_FIRST_USE)
|
||||
counted = set()
|
||||
for field, component_text, resolved_text, component_id in (
|
||||
("text", text, resolved_positive, prompt_id),
|
||||
(
|
||||
"negative_text",
|
||||
negative_text,
|
||||
resolved_negative,
|
||||
pair["negative_prompt_id"],
|
||||
),
|
||||
):
|
||||
component_hash = generate_prompt_hash(component_text)
|
||||
if (
|
||||
field not in used_fields
|
||||
or not component_id
|
||||
or component_hash in counted
|
||||
):
|
||||
continue
|
||||
counted.add(component_hash)
|
||||
seen_at_queue = (
|
||||
resolved_text is not None
|
||||
and generate_prompt_hash(resolved_text) == component_hash
|
||||
)
|
||||
runs = positive_pending.consume(component_hash) + int(not seen_at_queue)
|
||||
if runs:
|
||||
self.db.record_prompt_use(component_id, times=runs)
|
||||
if pair_used:
|
||||
runs = pair_pending.consume(pair_run_key(text, negative_text)) + int(
|
||||
not pair_seen
|
||||
)
|
||||
if runs:
|
||||
self.db.record_prompt_pair_use(pair_id, times=runs)
|
||||
if is_positive:
|
||||
self.prompt_tracker.set_current_prompt(
|
||||
prompt_text=positive,
|
||||
additional_data={
|
||||
"prompt_id": prompt_id,
|
||||
"pair_id": pair_id if pair_used else None,
|
||||
"positive_text": positive,
|
||||
"negative_text": negative_text,
|
||||
"positive_storage_text": text.strip(),
|
||||
"negative_storage_text": negative_text.strip(),
|
||||
"prepend_text": prepend_text.strip(),
|
||||
"append_text": append_text.strip(),
|
||||
"final_text": positive,
|
||||
"category": category.strip() or None,
|
||||
"tags": self._parse_tags(tags) or [],
|
||||
},
|
||||
push_to_queue=not positive_seen or (pair_used and not pair_seen),
|
||||
**self._run_identity(prompt, unique_id),
|
||||
)
|
||||
return pair_id, prompt_id
|
||||
|
||||
@classmethod
|
||||
def IS_CHANGED(
|
||||
cls,
|
||||
|
||||
@@ -216,6 +216,9 @@ class PromptManagerText(PromptManagerBase, ComfyNodeABC):
|
||||
except Exception as e:
|
||||
# Log error but don't fail the processing
|
||||
self.logger.warning(f"Failed to save prompt to database: {e}")
|
||||
self._mark_untracked(prompt, unique_id)
|
||||
else:
|
||||
self._skip_empty_prompt(prompt, unique_id)
|
||||
|
||||
# Register with ComfyUI integration for standard metadata compatibility
|
||||
node_id = f"promptmanagertext_{int(time.time() * 1000)}"
|
||||
|
||||
@@ -19,6 +19,7 @@ Split into domain-specific mixins for maintainability:
|
||||
- MaintenanceRoutesMixin: reviewed maintenance plans (preview, apply)
|
||||
- WorkflowStorageRoutesMixin: store each workflow once, then Reclaim space
|
||||
- DuplicateFilesMixin: duplicate image files as find and removal jobs
|
||||
- LoraImportRoutesMixin: the reviewed, staged LoRA reference import job
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
@@ -56,6 +57,7 @@ from .thumbnail_auto import AutoThumbnailMixin
|
||||
from .system import SystemRoutesMixin
|
||||
from .library_location import LibraryLocationRoutesMixin
|
||||
from .maintenance import MaintenanceRoutesMixin
|
||||
from .backup_restore import BackupRestoreRoutesMixin
|
||||
from .workflow_storage import WorkflowStorageRoutesMixin
|
||||
from .duplicate_files import DuplicateFilesMixin
|
||||
from .maintenance_plans import PlanStore
|
||||
@@ -63,6 +65,7 @@ from .jobs import JobRegistry
|
||||
from .logging_routes import LoggingRoutesMixin
|
||||
from .autotag_routes import AutotagRoutesMixin
|
||||
from .lora_integration import LoraIntegrationMixin
|
||||
from .lora_import_routes import LoraImportRoutesMixin
|
||||
|
||||
try:
|
||||
from ...database.operations import PromptDatabase
|
||||
@@ -302,11 +305,13 @@ class PromptManagerAPI(
|
||||
SystemRoutesMixin,
|
||||
LibraryLocationRoutesMixin,
|
||||
MaintenanceRoutesMixin,
|
||||
BackupRestoreRoutesMixin,
|
||||
WorkflowStorageRoutesMixin,
|
||||
DuplicateFilesMixin,
|
||||
LoggingRoutesMixin,
|
||||
AutotagRoutesMixin,
|
||||
LoraIntegrationMixin,
|
||||
LoraImportRoutesMixin,
|
||||
):
|
||||
"""REST API handler for PromptManager operations and web interface.
|
||||
|
||||
@@ -376,7 +381,10 @@ class PromptManagerAPI(
|
||||
try:
|
||||
from .maintenance import sweep_safety_temps
|
||||
|
||||
from .backup_restore import restore_stage
|
||||
|
||||
swept = sweep_safety_temps(self.db.model.db_path)
|
||||
swept += restore_stage.sweep_restore_temps(self.db.model.db_path)
|
||||
if swept:
|
||||
self.logger.info(f"Removed {len(swept)} interrupted backup copies")
|
||||
except Exception as e:
|
||||
@@ -547,10 +555,12 @@ class PromptManagerAPI(
|
||||
self._register_system_routes(routes)
|
||||
self._register_library_location_routes(routes)
|
||||
self._register_maintenance_routes(routes)
|
||||
self._register_backup_restore_routes(routes)
|
||||
self._register_duplicate_routes(routes)
|
||||
self._register_logging_routes(routes)
|
||||
self._register_autotag_routes(routes)
|
||||
self._register_lora_routes(routes)
|
||||
self._register_lora_import_routes(routes)
|
||||
|
||||
# Register gzip compression middleware (once)
|
||||
global _gzip_registered
|
||||
|
||||
+130
-224
@@ -3,16 +3,15 @@
|
||||
import asyncio
|
||||
import contextlib
|
||||
from contextlib import closing
|
||||
import datetime
|
||||
import functools
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
@@ -27,7 +26,6 @@ from .jobs import (
|
||||
DUPLICATE_CLEANUP_JOB_KIND,
|
||||
DUPLICATES_JOB_KIND,
|
||||
MAINTENANCE_JOB_KIND,
|
||||
RESTORE_JOB_KIND,
|
||||
JobBusy,
|
||||
)
|
||||
|
||||
@@ -37,8 +35,12 @@ except ImportError:
|
||||
from utils.parallel import map_parallel
|
||||
|
||||
try:
|
||||
from ...database.operations import PromptDatabase
|
||||
from ...database.pair_ingestion import ingest_resolved_pair
|
||||
from ...database.pair_ingestion import blank_positive, ingest_resolved_pair
|
||||
from ...database.run_attribution import (
|
||||
host_output_directory,
|
||||
stored_run_prompt,
|
||||
stored_run_prompts,
|
||||
)
|
||||
from ...utils.generation_params import (
|
||||
PLACEHOLDERS,
|
||||
GenerationParamsError,
|
||||
@@ -48,8 +50,12 @@ try:
|
||||
from ...utils.prompt_pair_resolver import resolve_prompt_pair
|
||||
from ...utils.review_capture import reviewable
|
||||
except ImportError:
|
||||
from database.operations import PromptDatabase
|
||||
from database.pair_ingestion import ingest_resolved_pair
|
||||
from database.pair_ingestion import blank_positive, ingest_resolved_pair
|
||||
from database.run_attribution import (
|
||||
host_output_directory,
|
||||
stored_run_prompt,
|
||||
stored_run_prompts,
|
||||
)
|
||||
from utils.generation_params import (
|
||||
PLACEHOLDERS,
|
||||
GenerationParamsError,
|
||||
@@ -552,6 +558,24 @@ def _current_image_monitor():
|
||||
return getattr(_image_monitor_module(), "_monitor_instance", None)
|
||||
|
||||
|
||||
class _BatchHistory:
|
||||
"""A scan batch's recorded-history lookup, made on the first file needing it.
|
||||
|
||||
Most files link through their metadata and never ask; the first one that
|
||||
does triggers one lookup for the whole batch (``_recorded_runs_for``).
|
||||
"""
|
||||
|
||||
def __init__(self, api, media_files):
|
||||
self._api = api
|
||||
self._files = media_files
|
||||
self._found = None
|
||||
|
||||
def get(self, path):
|
||||
if self._found is None:
|
||||
self._found = self._api._recorded_runs_for(self._files)
|
||||
return self._found.get(path)
|
||||
|
||||
|
||||
class AdminRoutesMixin:
|
||||
"""Mixin providing admin, diagnostics, and maintenance API endpoints."""
|
||||
|
||||
@@ -734,9 +758,20 @@ class AdminRoutesMixin:
|
||||
|
||||
Only the files under ``output_dir`` count (paths are relative to it);
|
||||
a group needs two of them. A file that vanished since it was hashed
|
||||
is left out.
|
||||
is left out. Each ``url`` names the source ``output_dir`` is
|
||||
(``?root=``) when it is one, so View Full never opens another
|
||||
source's file at the same path.
|
||||
"""
|
||||
output_path = Path(os.path.abspath(output_dir))
|
||||
identity = os.path.normcase(os.path.realpath(str(output_path)))
|
||||
root_index = next(
|
||||
(
|
||||
index
|
||||
for index, root in enumerate(self._get_all_output_dirs())
|
||||
if os.path.normcase(os.path.realpath(str(root))) == identity
|
||||
),
|
||||
None,
|
||||
)
|
||||
base = os.path.normcase(str(output_path)).rstrip("\\/") + os.sep
|
||||
duplicates = []
|
||||
for group in groups:
|
||||
@@ -747,7 +782,10 @@ class AdminRoutesMixin:
|
||||
try:
|
||||
images.append(
|
||||
self._describe_media_file(
|
||||
Path(record["abs"]), output_path, file_hash=group["hash"]
|
||||
Path(record["abs"]),
|
||||
output_path,
|
||||
file_hash=group["hash"],
|
||||
root_index=root_index,
|
||||
)
|
||||
)
|
||||
except (OSError, ValueError) as e:
|
||||
@@ -847,9 +885,12 @@ class AdminRoutesMixin:
|
||||
)
|
||||
return groups
|
||||
|
||||
def _describe_media_file(self, media_path, output_path, file_hash=None):
|
||||
def _describe_media_file(
|
||||
self, media_path, output_path, file_hash=None, root_index=None
|
||||
):
|
||||
"""Describe one media file for the duplicates response (hashing it
|
||||
unless ``file_hash`` is given)."""
|
||||
unless ``file_hash`` is given). With ``root_index`` the ``url`` names
|
||||
that source and its path is URL-encoded."""
|
||||
stat = media_path.stat()
|
||||
rel_path = media_path.relative_to(output_path)
|
||||
extension = media_path.suffix.lower()
|
||||
@@ -862,13 +903,18 @@ class AdminRoutesMixin:
|
||||
thumbnail_settings.thumbnail_stem_rel(rel_path) + existing.suffix
|
||||
)
|
||||
thumbnail_url = thumbnail_url_for(thumb_rel, existing)
|
||||
if root_index is None:
|
||||
url = f"/prompt_manager/images/serve/{rel_path.as_posix()}"
|
||||
else:
|
||||
encoded = urllib.parse.quote(rel_path.as_posix(), safe="/")
|
||||
url = f"/prompt_manager/images/serve/{encoded}?root={root_index}"
|
||||
|
||||
return {
|
||||
"id": str(hash(str(media_path))),
|
||||
"filename": media_path.name,
|
||||
"path": str(rel_path),
|
||||
"relative_path": str(rel_path),
|
||||
"url": f"/prompt_manager/images/serve/{rel_path.as_posix()}",
|
||||
"url": url,
|
||||
"thumbnail_url": thumbnail_url,
|
||||
"size": stat.st_size,
|
||||
"modified_time": stat.st_mtime,
|
||||
@@ -1236,216 +1282,6 @@ class AdminRoutesMixin:
|
||||
"message": f"Found {len(issues)} consistency issues",
|
||||
}
|
||||
|
||||
async def backup_database(self, request):
|
||||
"""Download a consistent copy of the prompts database (WAL included)."""
|
||||
try:
|
||||
model = self.db.model
|
||||
db_path = model.db_path
|
||||
|
||||
if not os.path.exists(db_path):
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Database file not found"}, status=404
|
||||
)
|
||||
|
||||
fd, temp_path = tempfile.mkstemp(suffix=".db")
|
||||
os.close(fd)
|
||||
try:
|
||||
ok = await self._run_in_executor(model.backup_database, temp_path)
|
||||
if not ok:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Failed to create database backup"},
|
||||
status=500,
|
||||
)
|
||||
# Read once (the file is deleted right after) rather than
|
||||
# streaming so the temp file never outlives the request.
|
||||
with open(temp_path, "rb") as fh:
|
||||
file_data = fh.read()
|
||||
finally:
|
||||
if os.path.exists(temp_path):
|
||||
os.unlink(temp_path)
|
||||
|
||||
timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
filename = f"prompts_backup_{timestamp}.db"
|
||||
|
||||
return web.Response(
|
||||
body=file_data,
|
||||
content_type="application/octet-stream",
|
||||
headers={
|
||||
"Content-Disposition": f'attachment; filename="{filename}"',
|
||||
"Content-Length": str(len(file_data)),
|
||||
},
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Backup error: {e}", exc_info=True)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": f"Failed to backup database: {self._public_error(e)}",
|
||||
},
|
||||
status=500,
|
||||
)
|
||||
|
||||
async def restore_database(self, request):
|
||||
"""Restore the prompts database from an uploaded SQLite file.
|
||||
|
||||
The upload is streamed to a temp file. Free disk space decides how
|
||||
large it may be (``maintenance.restore_room``), under a 64 GiB ceiling
|
||||
(``restore_max_bytes``): a request whose Content-Length does not fit
|
||||
is refused before anything is written, and one without is refused as
|
||||
soon as its running size does not. The file is
|
||||
verified with PromptModel.verify_database_file and only then copied
|
||||
over the live database, which is backed up first. The whole request
|
||||
holds the registry (records): it is refused while a scan or another
|
||||
record-changing job runs, and those are refused while it runs.
|
||||
"""
|
||||
return await self._run_held(
|
||||
RESTORE_JOB_KIND,
|
||||
"Database restore (admin page)",
|
||||
"records",
|
||||
lambda: self._restore_database(request),
|
||||
)
|
||||
|
||||
def _restore_refusal(self, size, ceiling, room, *, known):
|
||||
"""``400 too-large`` past the ceiling, else ``507 insufficient-space``."""
|
||||
from .maintenance import restore_space_message
|
||||
|
||||
if size > ceiling:
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"code": "too-large",
|
||||
"error": (
|
||||
"File too large. Maximum size is "
|
||||
f"{ceiling // (1024 * 1024)}MB"
|
||||
),
|
||||
},
|
||||
status=400,
|
||||
)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"code": "insufficient-space",
|
||||
"error": restore_space_message(size, room, known=known),
|
||||
},
|
||||
status=507,
|
||||
)
|
||||
|
||||
async def _restore_database(self, request):
|
||||
from .maintenance import RESTORE_MAX_BYTES, restore_room
|
||||
|
||||
ceiling = getattr(self, "restore_max_bytes", RESTORE_MAX_BYTES)
|
||||
temp_path = None
|
||||
try:
|
||||
# Free disk space decides how large a backup may be (#313): the
|
||||
# upload, its staged copy and the swap must all fit.
|
||||
room = await self._run_in_executor(restore_room, self.db.model.db_path)
|
||||
max_bytes = min(ceiling, room)
|
||||
length = request.content_length
|
||||
if length is not None and length > max_bytes:
|
||||
return self._restore_refusal(length, ceiling, room, known=True)
|
||||
reader = await request.multipart()
|
||||
field = await reader.next()
|
||||
|
||||
if not field or field.name != "database_file":
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": (
|
||||
"No database file uploaded. "
|
||||
"Expected field name: database_file"
|
||||
),
|
||||
},
|
||||
status=400,
|
||||
)
|
||||
|
||||
fd, temp_path = tempfile.mkstemp(suffix=".db")
|
||||
os.close(fd)
|
||||
size = 0
|
||||
with open(temp_path, "wb") as out:
|
||||
while True:
|
||||
chunk = await field.read_chunk()
|
||||
if not chunk:
|
||||
break
|
||||
size += len(chunk)
|
||||
if size > max_bytes:
|
||||
return self._restore_refusal(size, ceiling, room, known=False)
|
||||
out.write(chunk)
|
||||
|
||||
if size == 0:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Uploaded file is empty"}, status=400
|
||||
)
|
||||
|
||||
model = self.db.model
|
||||
ok, reason = await self._run_in_executor(
|
||||
model.verify_database_file, temp_path
|
||||
)
|
||||
if ok:
|
||||
reason = await self._run_in_executor(_inspect_restore_upload, temp_path)
|
||||
ok = reason is None
|
||||
if not ok:
|
||||
return web.json_response(
|
||||
{"success": False, "error": reason}, status=400
|
||||
)
|
||||
|
||||
try:
|
||||
backup_path = await self._run_in_executor(
|
||||
model.restore_from_file, temp_path
|
||||
)
|
||||
except sqlite3.OperationalError as exc:
|
||||
if (
|
||||
getattr(exc, "_prompt_manager_restore_backup_created", False)
|
||||
is True
|
||||
):
|
||||
try:
|
||||
await self._run_in_executor(
|
||||
_prune_safety_backups, model.db_path
|
||||
)
|
||||
except Exception:
|
||||
self.logger.warning(
|
||||
"Could not prune safety backups after failed restore"
|
||||
)
|
||||
raise
|
||||
removed = await self._run_in_executor(_prune_safety_backups, model.db_path)
|
||||
if removed:
|
||||
self.logger.info(f"Removed {len(removed)} old safety backups")
|
||||
previous = self.db
|
||||
self.db = PromptDatabase(model.db_path)
|
||||
self._database_replaced() # a health report described the old one
|
||||
# Its connections would otherwise linger (a file lock on Windows)
|
||||
previous.close_all()
|
||||
prompt_count = self.db.model.get_database_info().get("total_prompts", 0)
|
||||
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"message": (
|
||||
"Database restored successfully. "
|
||||
f"Found {prompt_count} prompts."
|
||||
),
|
||||
"prompt_count": prompt_count,
|
||||
"backup_created": (
|
||||
self._public_path(backup_path) if backup_path else None
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Restore error: {e}", exc_info=True)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": f"Failed to restore database: {self._public_error(e)}",
|
||||
},
|
||||
status=500,
|
||||
)
|
||||
finally:
|
||||
if temp_path:
|
||||
for suffix in ("", "-wal", "-shm", "-journal"):
|
||||
if os.path.exists(temp_path + suffix):
|
||||
os.unlink(temp_path + suffix)
|
||||
|
||||
async def scan_images(self, request):
|
||||
"""Scan ComfyUI output images for prompt metadata and add them to the database.
|
||||
|
||||
@@ -1756,6 +1592,8 @@ class AdminRoutesMixin:
|
||||
if outcomes is not None:
|
||||
outcomes.append(_scan_outcome(media_file, roots, status, title))
|
||||
|
||||
batch_results = list(batch_results)
|
||||
history = _BatchHistory(self, [media for media, _meta in batch_results])
|
||||
for media_file, metadata in batch_results:
|
||||
if self._scan_job and self._scan_job.cancelled:
|
||||
break
|
||||
@@ -1769,6 +1607,7 @@ class AdminRoutesMixin:
|
||||
ambiguous=ambiguous,
|
||||
untraced=untraced,
|
||||
records=records,
|
||||
history=history,
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error processing {media_file.name}: {e}")
|
||||
@@ -1827,6 +1666,7 @@ class AdminRoutesMixin:
|
||||
ambiguous=None,
|
||||
untraced=None,
|
||||
records=None,
|
||||
history=None,
|
||||
):
|
||||
"""Enrich exact pairs; retain legacy positive-only import for absent pairs.
|
||||
|
||||
@@ -1844,7 +1684,9 @@ class AdminRoutesMixin:
|
||||
a dict.
|
||||
"""
|
||||
if not metadata:
|
||||
return None
|
||||
# No prompt metadata (--disable-metadata, bare savers): only the
|
||||
# recorded run can say which prompt made it (#342)
|
||||
return self._link_recorded_run(media_file, {}, detailed, history)
|
||||
canonical = dict(metadata)
|
||||
for key in ("prompt", "workflow"):
|
||||
if isinstance(canonical.get(key), str):
|
||||
@@ -1854,9 +1696,12 @@ class AdminRoutesMixin:
|
||||
# Preserve malformed metadata for conservative resolver rejection.
|
||||
pass
|
||||
if not canonical.get("prompt") and not canonical.get("parameters"):
|
||||
return None
|
||||
return self._link_recorded_run(media_file, canonical, detailed, history)
|
||||
_with_file_size(canonical, media_file)
|
||||
resolution = resolve_prompt_pair(canonical)
|
||||
if blank_positive(resolution):
|
||||
# Only prepend/append text: no stored prompt can own it (#342)
|
||||
resolution = {**resolution, "state": "unresolved"}
|
||||
# Ambiguous without a choice: unresolved unless something links it.
|
||||
unchoosable = resolution["state"] == "ambiguous" and not reviewable(resolution)
|
||||
if unchoosable:
|
||||
@@ -1885,6 +1730,11 @@ class AdminRoutesMixin:
|
||||
# Graphs without a recognized sampler keep the legacy extractor below.
|
||||
prompt_text = self._graph_prompt_text(metadata)
|
||||
if not prompt_text:
|
||||
linked = self._link_recorded_run(
|
||||
media_file, canonical, detailed, history
|
||||
)
|
||||
if linked is not None:
|
||||
return linked
|
||||
return "unresolved" if unchoosable else resolution["state"]
|
||||
elif resolution["state"] == "ambiguous":
|
||||
return "unresolved" if unchoosable else "ambiguous"
|
||||
@@ -1908,6 +1758,62 @@ class AdminRoutesMixin:
|
||||
return status
|
||||
return {"status": status, "title": self._scan_title(media_file, prompt_text)}
|
||||
|
||||
def _recorded_runs_for(self, media_files):
|
||||
"""``{path: prompt id or None}`` from recorded history for a scan batch.
|
||||
|
||||
One output lookup per chunk of names (``stored_run_prompts``), so a
|
||||
batch never queries history once per file. An empty dict on failure.
|
||||
"""
|
||||
try:
|
||||
return stored_run_prompts(
|
||||
self.db, [str(path) for path in media_files], host_output_directory()
|
||||
)
|
||||
except Exception:
|
||||
self.logger.warning(
|
||||
"History attribution failed for a scan batch", exc_info=True
|
||||
)
|
||||
return {}
|
||||
|
||||
def _link_recorded_run(self, media_file, canonical, detailed=False, history=None):
|
||||
"""Link a file recorded history attributes to one stored prompt (#342).
|
||||
|
||||
Returns the scan status (with ``detailed``, a stored or linked file's
|
||||
``{"status", "title"}``), or None when history names no single stored
|
||||
prompt for the file (``database.run_attribution``). Never creates a
|
||||
prompt. Works without any prompt metadata in the file. ``history`` is
|
||||
the batch's precomputed lookup; without it the file is looked up alone.
|
||||
"""
|
||||
if history is not None:
|
||||
prompt_id = history.get(str(media_file))
|
||||
else:
|
||||
try:
|
||||
prompt_id = stored_run_prompt(
|
||||
self.db, str(media_file), host_output_directory()
|
||||
)
|
||||
except Exception:
|
||||
self.logger.warning(
|
||||
f"History attribution failed for {media_file.name}",
|
||||
exc_info=True,
|
||||
)
|
||||
return None
|
||||
if prompt_id is None:
|
||||
return None
|
||||
try:
|
||||
image_id = self.db.link_image_to_prompt(
|
||||
prompt_id, str(media_file), metadata=canonical
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to link {media_file.name} from history: {e}")
|
||||
return "found"
|
||||
status = "linked" if image_id else "unchanged"
|
||||
if not detailed or status not in SCAN_TITLED_DETAILS:
|
||||
return status
|
||||
prompt = self.db.get_prompt_by_id(prompt_id) or {}
|
||||
return {
|
||||
"status": status,
|
||||
"title": self._scan_title(media_file, prompt.get("text") or ""),
|
||||
}
|
||||
|
||||
def _scan_title(self, media_file, text):
|
||||
"""The linked prompt's Library title (its stored title, else ``text``'s
|
||||
first line), as the Library and the inspector name it."""
|
||||
|
||||
@@ -0,0 +1,942 @@
|
||||
"""Database backup and restore routes (#249 System S8).
|
||||
|
||||
Legacy routes, responses unchanged:
|
||||
|
||||
- ``GET /prompt_manager/backup`` downloads a consistent copy of the library.
|
||||
While the copy is made it holds the registry as ``backup`` ("Database
|
||||
backup", ``guarded``): it is refused while a restore, maintenance apply or
|
||||
other record-changing job runs. The copy is written in the library's
|
||||
folder and made one standalone file; the hold ends there, so a slow or
|
||||
stalled download never keeps other work waiting. A copy that fails its
|
||||
integrity check is a 500 with no bytes. One that fails only a semantic
|
||||
check (``verify_database_file``) is still sent: a user can always get a
|
||||
backup. The reason is logged and sent as ``X-PromptManager-Verify``
|
||||
(``ok`` otherwise), and the status records it as not verified. The file
|
||||
is streamed, then closed and removed.
|
||||
- ``POST /prompt_manager/restore`` (multipart ``database_file``) restores in
|
||||
one request through the same stage and publish as the restore job, holding
|
||||
the registry as ``restore`` (``exclusive``).
|
||||
|
||||
Reviewed restore (new routes answer errors as ``{success:false, code, error}``):
|
||||
|
||||
- ``POST /prompt_manager/restore/preview`` (multipart ``database_file``, the
|
||||
same disk-space cap) stages the upload in the library's folder
|
||||
(``database/restore_stage.py``) and answers what it holds beside the live
|
||||
library: ``{token, expires_at, summary, schema:{migrated, library_state},
|
||||
integrity:"ok", documents:{repaired, cleared, images}, live, size,
|
||||
safety_backup:{destination_label}}``. Nothing live is written or backed
|
||||
up. At most two previews are held (``too-large``, 429).
|
||||
- ``DELETE /prompt_manager/restore/preview/{token}`` discards a stage.
|
||||
- ``POST /prompt_manager/restore/apply {token}`` starts the restore job (202
|
||||
``{job}``): kind ``restore``, "Database restore", ``exclusive`` (it runs
|
||||
alone, like Move library), not cancellable, indeterminate. The runner waits
|
||||
for request leases to end, rechecks the live fingerprint (``plan-changed``),
|
||||
publishes the exact staged bytes, swaps ``self.db``, prunes old safety
|
||||
backups and logs what was restored. Once the copy replaced the library,
|
||||
the restore is done: the later steps only log their failures.
|
||||
- ``GET /prompt_manager/backup/status``: the safety backups on disk
|
||||
(``<db>.backup_*``, newest first, names only) and this session's last
|
||||
verified download (Q9: no sidecar file).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import datetime
|
||||
import os
|
||||
import sqlite3
|
||||
import tempfile
|
||||
from contextlib import closing
|
||||
from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from . import admin as _admin
|
||||
from .job_routes import busy_response
|
||||
from .jobs import RESTORE_JOB_KIND, JobBusy
|
||||
from .maintenance_plans import PlanExpired, PlanUsed
|
||||
from .restore_plans import RestorePlanStore
|
||||
|
||||
try:
|
||||
from ...database import restore_stage
|
||||
from ...database.operations import PromptDatabase
|
||||
from ...database.safety_backup import (
|
||||
SafetyBackupFailed,
|
||||
fsync_file,
|
||||
make_standalone,
|
||||
remove_quietly,
|
||||
safety_temp,
|
||||
)
|
||||
except ImportError:
|
||||
from database import restore_stage
|
||||
from database.operations import PromptDatabase
|
||||
from database.safety_backup import (
|
||||
SafetyBackupFailed,
|
||||
fsync_file,
|
||||
make_standalone,
|
||||
remove_quietly,
|
||||
safety_temp,
|
||||
)
|
||||
|
||||
BACKUP_JOB_KIND = "backup"
|
||||
BACKUP_HOLD_TITLE = "Database backup"
|
||||
LEGACY_RESTORE_TITLE = "Database restore (admin page)"
|
||||
RESTORE_JOB_TITLE = "Database restore"
|
||||
RESTORE_GROUPS = ("errors",)
|
||||
MAX_TOKEN_LENGTH = 128
|
||||
APPLY_BODY_MAX = 1024
|
||||
STREAM_CHUNK = 1024 * 1024
|
||||
MAX_LISTED_BACKUPS = 10
|
||||
SIDECARS = ("", "-wal", "-shm", "-journal")
|
||||
REQUESTS_BUSY = (
|
||||
"Other requests are still reading the library; nothing was changed. "
|
||||
"Try again in a moment."
|
||||
)
|
||||
# S9 I1: the review goes back to the page that started the restore
|
||||
# (``scope.review_kept``), which opens it again with Restore and Discard; no
|
||||
# other place can start a kept review, so the line says where it is.
|
||||
REVIEW_KEPT = (
|
||||
"Other requests are still reading the library; nothing was changed. "
|
||||
"The review was reopened in the page that started the restore: restore it "
|
||||
"again there, or discard it."
|
||||
)
|
||||
VERIFY_HEADER = "X-PromptManager-Verify"
|
||||
SQLITE_HEADER = b"SQLite format 3\x00"
|
||||
PLAN_CHANGED = (
|
||||
"The library changed since the review; nothing was changed. "
|
||||
"Review the restore again."
|
||||
)
|
||||
NOT_RESTORED = "The restore failed; the library was not changed."
|
||||
|
||||
|
||||
def _failure(message, status, code=None, **extra):
|
||||
return web.json_response(
|
||||
{"success": False, "code": code, "error": message, **extra}, status=status
|
||||
)
|
||||
|
||||
|
||||
def _iso(timestamp):
|
||||
return datetime.datetime.fromtimestamp(timestamp, datetime.timezone.utc).isoformat()
|
||||
|
||||
|
||||
def _plural(count, word):
|
||||
return f"{count:,} {word}" + ("" if count == 1 else "s")
|
||||
|
||||
|
||||
def documents_line(documents):
|
||||
"""The repair report's sentence, or "" when nothing was damaged."""
|
||||
repaired, cleared = documents.get("repaired", 0), documents.get("cleared", 0)
|
||||
parts = []
|
||||
if repaired:
|
||||
parts.append(f"repaired {_plural(repaired, 'damaged workflow document')}")
|
||||
if cleared:
|
||||
images = documents.get("images", 0)
|
||||
parts.append(
|
||||
f"could not repair {_plural(cleared, 'workflow document')}: "
|
||||
f"{_plural(images, 'image')} {'shows' if images == 1 else 'show'} "
|
||||
"no workflow"
|
||||
)
|
||||
if not parts:
|
||||
return ""
|
||||
text = "; ".join(parts)
|
||||
return text[0].upper() + text[1:] + "."
|
||||
|
||||
|
||||
def restored_line(summary):
|
||||
return (
|
||||
f"Restored {_plural(summary['prompts'], 'prompt')}, "
|
||||
f"{_plural(summary['image_links'], 'image link')}, "
|
||||
f"{_plural(summary['pairs'], 'saved pair')} and "
|
||||
f"{_plural(summary['runs'], 'run')}"
|
||||
)
|
||||
|
||||
|
||||
def safety_backups(db_path, limit=MAX_LISTED_BACKUPS):
|
||||
"""``<db>.backup_*`` files beside the database, newest first (blocking)."""
|
||||
db_path = os.path.abspath(db_path)
|
||||
folder = os.path.dirname(db_path)
|
||||
prefix = os.path.basename(db_path) + ".backup_"
|
||||
found = []
|
||||
try:
|
||||
names = os.listdir(folder)
|
||||
except OSError:
|
||||
return []
|
||||
for name in names:
|
||||
if not name.startswith(prefix):
|
||||
continue
|
||||
try:
|
||||
stat = os.stat(os.path.join(folder, name))
|
||||
except OSError:
|
||||
continue
|
||||
if not os.path.isfile(os.path.join(folder, name)):
|
||||
continue
|
||||
found.append((stat.st_mtime, name, stat.st_size))
|
||||
found.sort(key=lambda item: (item[0], item[1]), reverse=True)
|
||||
return [
|
||||
{"name": name, "bytes": size, "modified_at": _iso(mtime)}
|
||||
for mtime, name, size in found[:limit]
|
||||
]
|
||||
|
||||
|
||||
def snapshot_corruption(path):
|
||||
"""Why the snapshot file is corrupt, or None (blocking).
|
||||
|
||||
Only physical damage refuses a download: not a SQLite file, or a failed
|
||||
``PRAGMA integrity_check``.
|
||||
"""
|
||||
try:
|
||||
with open(path, "rb") as handle:
|
||||
if handle.read(len(SQLITE_HEADER)) != SQLITE_HEADER:
|
||||
return "Not a SQLite database file"
|
||||
uri = Path(os.path.abspath(path)).as_uri() + "?mode=ro"
|
||||
with closing(sqlite3.connect(uri, uri=True)) as conn:
|
||||
row = conn.execute("PRAGMA integrity_check").fetchone()
|
||||
except (OSError, sqlite3.Error) as error:
|
||||
return f"Invalid SQLite database: {error}"
|
||||
verdict = row[0] if row else "no result"
|
||||
return None if verdict == "ok" else f"Integrity check failed: {verdict}"
|
||||
|
||||
|
||||
def write_backup_snapshot(model):
|
||||
"""A standalone copy of the library beside it: ``(path, warning)`` (blocking).
|
||||
|
||||
Raises SafetyBackupFailed with nothing left on disk when the copy cannot
|
||||
be written or is corrupt (``snapshot_corruption``). ``warning`` is the
|
||||
reason a semantic check (``verify_database_file``, workflow documents
|
||||
not read) refused the copy, or None. The caller removes the file.
|
||||
"""
|
||||
temp = safety_temp(model.db_path)
|
||||
try:
|
||||
if not model.backup_database(temp):
|
||||
raise SafetyBackupFailed("The backup could not be written")
|
||||
make_standalone(temp)
|
||||
corrupt = snapshot_corruption(temp)
|
||||
if corrupt is not None:
|
||||
raise SafetyBackupFailed("The backup is corrupt", corrupt)
|
||||
ok, reason = model.verify_database_file(temp, documents=False)
|
||||
fsync_file(temp)
|
||||
return temp, (None if ok else reason)
|
||||
except BaseException:
|
||||
remove_quietly(*(temp + suffix for suffix in SIDECARS))
|
||||
raise
|
||||
|
||||
|
||||
def _header_text(text):
|
||||
"""A reason as a one-line, latin-1 header value."""
|
||||
flat = " ".join(str(text).split())[:300]
|
||||
return flat.encode("latin-1", "replace").decode("latin-1")
|
||||
|
||||
|
||||
class BackupRestoreRoutesMixin:
|
||||
"""Mixin providing the backup and restore endpoints."""
|
||||
|
||||
def _register_backup_restore_routes(self, routes):
|
||||
@routes.post("/prompt_manager/restore/preview")
|
||||
async def restore_preview_route(request):
|
||||
return await self.preview_restore(request)
|
||||
|
||||
@routes.delete("/prompt_manager/restore/preview/{token}")
|
||||
async def restore_discard_route(request):
|
||||
return await self.discard_restore_preview(request)
|
||||
|
||||
@routes.post("/prompt_manager/restore/apply")
|
||||
async def restore_apply_route(request):
|
||||
return await self.apply_restore(request)
|
||||
|
||||
@routes.get("/prompt_manager/backup/status")
|
||||
async def backup_status_route(request):
|
||||
return await self.backup_status(request)
|
||||
|
||||
@property
|
||||
def _restore_plans(self):
|
||||
store = getattr(self, "_restore_plan_store", None)
|
||||
if store is None:
|
||||
store = self._restore_plan_store = RestorePlanStore()
|
||||
return store
|
||||
|
||||
# ── backup ────────────────────────────────────────────────────────
|
||||
|
||||
async def backup_database(self, request):
|
||||
"""Download a copy of the prompts database (WAL included)."""
|
||||
return await self._stream_backup(request)
|
||||
|
||||
async def _stream_backup(self, request):
|
||||
model = self.db.model
|
||||
if not os.path.exists(model.db_path):
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Database file not found"}, status=404
|
||||
)
|
||||
made = await self._run_held(
|
||||
BACKUP_JOB_KIND,
|
||||
BACKUP_HOLD_TITLE,
|
||||
"guarded",
|
||||
lambda: self._make_backup_snapshot(model),
|
||||
)
|
||||
if isinstance(made, web.StreamResponse):
|
||||
return made # refused, or the snapshot failed
|
||||
temp, warning = made
|
||||
return await self._send_snapshot(request, temp, warning)
|
||||
|
||||
async def _make_backup_snapshot(self, model):
|
||||
"""``(path, warning)``, or the error response (inside the hold)."""
|
||||
loop = asyncio.get_running_loop()
|
||||
future = loop.run_in_executor(None, write_backup_snapshot, model)
|
||||
try:
|
||||
temp, warning = await asyncio.shield(future)
|
||||
except asyncio.CancelledError:
|
||||
# The worker cannot be stopped: remove what it writes once it is
|
||||
# done (#335 re-review R4).
|
||||
future.add_done_callback(
|
||||
lambda done: loop.create_task(self._remove_finished_snapshot(done))
|
||||
)
|
||||
raise
|
||||
except SafetyBackupFailed as failed:
|
||||
self.logger.error(f"Backup failed: {failed} {failed.reason or ''}")
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Failed to create database backup"},
|
||||
status=500,
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.error(f"Backup error: {e}", exc_info=True)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": f"Failed to backup database: {self._public_error(e)}",
|
||||
},
|
||||
status=500,
|
||||
)
|
||||
if warning is not None:
|
||||
self.logger.warning(f"Backup sent with a failed check: {warning}")
|
||||
return temp, warning
|
||||
|
||||
async def _remove_finished_snapshot(self, future):
|
||||
"""Remove the snapshot a cancelled download's worker finished."""
|
||||
if future.cancelled() or future.exception() is not None:
|
||||
return # nothing was left on disk
|
||||
temp, _warning = future.result()
|
||||
await self._run_in_executor(
|
||||
remove_quietly, *(temp + suffix for suffix in SIDECARS)
|
||||
)
|
||||
|
||||
async def _send_snapshot(self, request, temp, warning):
|
||||
"""Stream the snapshot (outside the hold), then close and remove it."""
|
||||
timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
handle = None
|
||||
try:
|
||||
size = os.path.getsize(temp)
|
||||
handle = open(temp, "rb")
|
||||
response = web.StreamResponse(
|
||||
headers={
|
||||
"Content-Type": "application/octet-stream",
|
||||
"Content-Disposition": (
|
||||
f'attachment; filename="prompts_backup_{timestamp}.db"'
|
||||
),
|
||||
"Content-Length": str(size),
|
||||
VERIFY_HEADER: "ok" if warning is None else _header_text(warning),
|
||||
}
|
||||
)
|
||||
await response.prepare(request)
|
||||
while True:
|
||||
chunk = await self._run_in_executor(handle.read, STREAM_CHUNK)
|
||||
if not chunk:
|
||||
break
|
||||
await response.write(chunk)
|
||||
await response.write_eof()
|
||||
self._last_backup_download = {
|
||||
"at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
"bytes": size,
|
||||
"verified": warning is None,
|
||||
}
|
||||
return response
|
||||
finally:
|
||||
if handle is not None:
|
||||
handle.close() # before the unlink (Windows)
|
||||
await self._run_in_executor(
|
||||
remove_quietly, *(temp + suffix for suffix in SIDECARS)
|
||||
)
|
||||
|
||||
async def backup_status(self, request):
|
||||
try:
|
||||
backups = await self._run_in_executor(safety_backups, self.db.model.db_path)
|
||||
except Exception:
|
||||
self.logger.exception("Backup status failed")
|
||||
return _failure("Failed to read the backup status", 500)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"safety_backups": backups,
|
||||
"last_download": getattr(self, "_last_backup_download", None),
|
||||
},
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
|
||||
# ── the upload ────────────────────────────────────────────────────
|
||||
|
||||
def _restore_refusal(self, size, ceiling, room, *, known):
|
||||
"""``400 too-large`` past the ceiling, else ``507 insufficient-space``."""
|
||||
from .maintenance import restore_space_message
|
||||
|
||||
if size > ceiling:
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"code": "too-large",
|
||||
"error": (
|
||||
"File too large. Maximum size is "
|
||||
f"{ceiling // (1024 * 1024)}MB"
|
||||
),
|
||||
},
|
||||
status=400,
|
||||
)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"code": "insufficient-space",
|
||||
"error": restore_space_message(size, room, known=known),
|
||||
},
|
||||
status=507,
|
||||
)
|
||||
|
||||
async def _receive_restore_upload(self, request, uploads):
|
||||
"""Stream ``database_file`` into the library's folder.
|
||||
|
||||
Returns ``(path, None)`` or ``(None, refusal)``. Each temp path is
|
||||
appended to ``uploads`` as soon as it exists, so the caller removes
|
||||
it (and its sidecars) whatever happens.
|
||||
"""
|
||||
from .maintenance import RESTORE_MAX_BYTES, restore_room
|
||||
|
||||
ceiling = getattr(self, "restore_max_bytes", RESTORE_MAX_BYTES)
|
||||
db_path = self.db.model.db_path
|
||||
# Free disk space decides how large a backup may be (#313): the
|
||||
# upload, its staged copy and the swap must all fit.
|
||||
room = await self._run_in_executor(restore_room, db_path)
|
||||
max_bytes = min(ceiling, room)
|
||||
length = request.content_length
|
||||
if length is not None and length > max_bytes:
|
||||
return None, self._restore_refusal(length, ceiling, room, known=True)
|
||||
reader = await request.multipart()
|
||||
field = await reader.next()
|
||||
if not field or field.name != "database_file":
|
||||
return None, web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": (
|
||||
"No database file uploaded. "
|
||||
"Expected field name: database_file"
|
||||
),
|
||||
},
|
||||
status=400,
|
||||
)
|
||||
fd, temp_path = tempfile.mkstemp(
|
||||
prefix=restore_stage.UPLOAD_PREFIX,
|
||||
suffix=".tmp",
|
||||
dir=os.path.dirname(os.path.abspath(db_path)),
|
||||
)
|
||||
uploads.append(temp_path)
|
||||
size = 0
|
||||
with os.fdopen(fd, "wb") as out:
|
||||
while True:
|
||||
chunk = await field.read_chunk()
|
||||
if not chunk:
|
||||
break
|
||||
size += len(chunk)
|
||||
if size > max_bytes:
|
||||
return None, self._restore_refusal(size, ceiling, room, known=False)
|
||||
out.write(chunk)
|
||||
if size == 0:
|
||||
return None, web.json_response(
|
||||
{"success": False, "error": "Uploaded file is empty"}, status=400
|
||||
)
|
||||
return temp_path, None
|
||||
|
||||
async def _upload_problem(self, path):
|
||||
"""None when the upload may be staged, else a readable reason."""
|
||||
ok, reason = await self._run_in_executor(
|
||||
self.db.model.verify_database_file, path, documents=False
|
||||
)
|
||||
if ok:
|
||||
reason = await self._run_in_executor(_admin._inspect_restore_upload, path)
|
||||
ok = reason is None
|
||||
return None if ok else reason
|
||||
|
||||
async def _remove_uploads(self, uploads):
|
||||
for path in uploads:
|
||||
await self._run_in_executor(
|
||||
remove_quietly, *(path + suffix for suffix in SIDECARS)
|
||||
)
|
||||
|
||||
async def _prune_after(self, db_path, *, failed=False):
|
||||
"""Prune old safety backups; a failure is only logged (never raises).
|
||||
|
||||
It runs after a restore replaced the library, or after one failed:
|
||||
either way the restore's outcome stands.
|
||||
"""
|
||||
try:
|
||||
removed = await self._run_in_executor(_admin._prune_safety_backups, db_path)
|
||||
except Exception as error:
|
||||
when = "failed" if failed else "completed"
|
||||
self.logger.warning(
|
||||
f"Could not prune safety backups after a {when} restore: "
|
||||
f"{type(error).__name__}"
|
||||
)
|
||||
return
|
||||
if removed:
|
||||
self.logger.info(f"Removed {len(removed)} old safety backups")
|
||||
|
||||
async def _publish(self, model, staged):
|
||||
"""Publish ``staged``; on a page-copy failure prune and re-raise."""
|
||||
try:
|
||||
return await self._run_in_executor(model.publish_staged, staged)
|
||||
except sqlite3.OperationalError as exc:
|
||||
if getattr(exc, "_prompt_manager_restore_backup_created", False) is True:
|
||||
await self._prune_after(model.db_path, failed=True)
|
||||
raise
|
||||
|
||||
async def _after_publish(self, db_path):
|
||||
"""Everything after the copy replaced the library: best effort, logged.
|
||||
|
||||
The restore is done once the copy succeeded, so nothing here fails it
|
||||
(#335 review I1). The library is adopted as ``self.db`` (the previous
|
||||
object, which points at the same file, stays when a new one cannot
|
||||
be opened), the generation is bumped (so the health report and
|
||||
reviewed plans of the old library are dropped), held restore
|
||||
previews are dropped and the previous object's connections closed.
|
||||
"""
|
||||
await self._prune_after(db_path)
|
||||
previous = self.db
|
||||
try:
|
||||
self.db = PromptDatabase(db_path)
|
||||
except Exception:
|
||||
self.logger.exception("Could not reopen the restored library")
|
||||
try:
|
||||
self._database_replaced()
|
||||
except Exception:
|
||||
self.logger.exception("Could not drop the old library's reports")
|
||||
try:
|
||||
await self._discard_held(self._restore_plans.pop_all())
|
||||
except Exception:
|
||||
self.logger.exception("Could not remove the old restore previews")
|
||||
if previous is not self.db:
|
||||
try:
|
||||
# Its connections would otherwise linger (a lock on Windows).
|
||||
await self._run_in_executor(previous.close_all)
|
||||
except Exception:
|
||||
self.logger.exception("Could not close the previous library")
|
||||
|
||||
async def _discard_held(self, held):
|
||||
"""Remove the stage folders of dropped store entries (off the loop)."""
|
||||
for entry in held:
|
||||
try:
|
||||
await self._run_in_executor(entry.staged.discard)
|
||||
except Exception:
|
||||
self.logger.exception("Could not remove a restore stage")
|
||||
|
||||
async def _discard_popped(self, token):
|
||||
"""Drop ``token`` and remove its stage, only if the store still held it.
|
||||
|
||||
A stage another apply's job has taken is not this request's to remove
|
||||
(#335 re-review R2).
|
||||
"""
|
||||
popped = self._restore_plans.pop(token)
|
||||
if popped is not None:
|
||||
await self._discard_held([popped])
|
||||
|
||||
async def _drop_expired_restores(self):
|
||||
"""Drop expired previews from the store (on the loop), then their files."""
|
||||
await self._discard_held(self._restore_plans.pop_expired())
|
||||
|
||||
def _schedule_restore_expiry(self):
|
||||
"""Remove each preview's stage when it expires, without a request.
|
||||
|
||||
A loop timer (``call_later``), so a server that stops with previews
|
||||
held leaves nothing pending; the startup sweep removes the folders.
|
||||
"""
|
||||
store = self._restore_plans
|
||||
expires = store.next_expiry()
|
||||
if expires is None:
|
||||
return
|
||||
handle = getattr(self, "_restore_expiry_handle", None)
|
||||
if handle is not None:
|
||||
handle.cancel()
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
def fire():
|
||||
self._restore_expiry_handle = None
|
||||
task = loop.create_task(self._expire_restores())
|
||||
self._restore_expiry_task = task
|
||||
|
||||
delay = max(0.0, expires - store.now())
|
||||
self._restore_expiry_handle = loop.call_later(delay, fire)
|
||||
|
||||
async def _expire_restores(self):
|
||||
await self._drop_expired_restores()
|
||||
self._schedule_restore_expiry()
|
||||
|
||||
# ── legacy one-step restore ───────────────────────────────────────
|
||||
|
||||
async def restore_database(self, request):
|
||||
"""Restore the prompts database from an uploaded SQLite file.
|
||||
|
||||
The upload is streamed to a temp file in the library's folder. Free
|
||||
disk space decides how large it may be (``maintenance.restore_room``),
|
||||
under a 64 GiB ceiling (``restore_max_bytes``): a request whose
|
||||
Content-Length does not fit is refused before anything is written,
|
||||
and one without is refused as soon as its running size does not. The
|
||||
file is checked, staged, repaired and verified, then published over
|
||||
the live database, which is backed up first. The whole request holds
|
||||
the registry as ``restore`` (``exclusive``): it is refused while any
|
||||
job runs, and every job and request lease is refused while it runs.
|
||||
"""
|
||||
return await self._run_held(
|
||||
RESTORE_JOB_KIND,
|
||||
LEGACY_RESTORE_TITLE,
|
||||
"exclusive",
|
||||
lambda: self._restore_database(request),
|
||||
)
|
||||
|
||||
async def _restore_database(self, request):
|
||||
uploads = []
|
||||
try:
|
||||
temp_path, refused = await self._receive_restore_upload(request, uploads)
|
||||
if refused is not None:
|
||||
return refused
|
||||
reason = await self._upload_problem(temp_path)
|
||||
if reason is not None:
|
||||
return web.json_response(
|
||||
{"success": False, "error": reason}, status=400
|
||||
)
|
||||
if not await self._jobs.drain_leases():
|
||||
return web.json_response(
|
||||
{"success": False, "error": REQUESTS_BUSY}, status=409
|
||||
)
|
||||
model = self.db.model
|
||||
staged = await self._run_in_executor(model.stage_restore, temp_path)
|
||||
try:
|
||||
backup_path = await self._publish(model, staged)
|
||||
finally:
|
||||
await self._run_in_executor(staged.discard)
|
||||
# The library is replaced: from here on the restore succeeded.
|
||||
await self._after_publish(model.db_path)
|
||||
try:
|
||||
prompt_count = await self._run_in_executor(
|
||||
lambda: self.db.model.get_database_info().get("total_prompts", 0)
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception("Could not count the restored prompts")
|
||||
prompt_count = staged.summary.get("prompts", 0)
|
||||
message = f"Database restored successfully. Found {prompt_count} prompts."
|
||||
body = {
|
||||
"success": True,
|
||||
"message": message,
|
||||
"prompt_count": prompt_count,
|
||||
"backup_created": (
|
||||
self._public_path(backup_path) if backup_path else None
|
||||
),
|
||||
}
|
||||
repaired = documents_line(staged.documents)
|
||||
if repaired:
|
||||
body["message"] = f"{message} {repaired}"
|
||||
body["workflow_documents"] = dict(staged.documents)
|
||||
return web.json_response(body)
|
||||
except Exception as e:
|
||||
self.logger.error(f"Restore error: {e}", exc_info=True)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": f"Failed to restore database: {self._public_error(e)}",
|
||||
},
|
||||
status=500,
|
||||
)
|
||||
finally:
|
||||
await self._remove_uploads(uploads)
|
||||
|
||||
# ── reviewed restore ──────────────────────────────────────────────
|
||||
|
||||
def _live_fingerprint(self, db):
|
||||
return restore_stage.live_fingerprint(db.model.get_connection())
|
||||
|
||||
async def preview_restore(self, request):
|
||||
"""Stage an upload for review; nothing live is written (see module)."""
|
||||
await self._drop_expired_restores()
|
||||
if self._restore_plans.full():
|
||||
return _too_many_reviews()
|
||||
uploads = []
|
||||
try:
|
||||
db = self.db
|
||||
identity = self._database_identity(db)
|
||||
temp_path, refused = await self._receive_restore_upload(request, uploads)
|
||||
if refused is not None:
|
||||
return refused
|
||||
return await self._review_upload(db, identity, temp_path, uploads)
|
||||
except Exception:
|
||||
self.logger.exception("Restore preview failed")
|
||||
return _failure("Failed to review the restore", 500)
|
||||
finally:
|
||||
await self._remove_uploads(uploads)
|
||||
|
||||
async def _stage_for_review(self, db, temp_path, uploads):
|
||||
"""Stage ``temp_path`` in the executor; a cancel discards the result.
|
||||
|
||||
A cancelled handler (shutdown, a dropped client) cannot stop the
|
||||
worker thread: the stage it finishes is removed when it is done, and
|
||||
so is the upload it reads (``_orphan_stage_cleanup``).
|
||||
"""
|
||||
loop = asyncio.get_running_loop()
|
||||
future = loop.run_in_executor(None, db.model.stage_restore, temp_path)
|
||||
try:
|
||||
return await asyncio.shield(future)
|
||||
except asyncio.CancelledError:
|
||||
orphans = getattr(self, "_orphan_stages", None)
|
||||
if orphans is None:
|
||||
orphans = self._orphan_stages = set()
|
||||
entry = (future, tuple(uploads))
|
||||
orphans.add(entry)
|
||||
uploads.clear() # still being read: removed with the stage
|
||||
future.add_done_callback(
|
||||
lambda _f: loop.create_task(self._orphan_stage_cleanup())
|
||||
)
|
||||
raise
|
||||
|
||||
async def _orphan_stage_cleanup(self):
|
||||
"""Remove what cancelled previews staged, once their workers are done."""
|
||||
for entry in list(getattr(self, "_orphan_stages", ()) or ()):
|
||||
future, uploads = entry
|
||||
if not future.done():
|
||||
continue
|
||||
self._orphan_stages.discard(entry)
|
||||
if not future.cancelled() and future.exception() is None:
|
||||
await self._discard_held([_Orphan(future.result())])
|
||||
await self._remove_uploads(uploads)
|
||||
|
||||
async def _review_upload(self, db, identity, temp_path, uploads=None):
|
||||
"""Check, stage and compare an upload; the preview response.
|
||||
|
||||
Only the read of the live library takes a lease (#335 review M2):
|
||||
the upload and the staging never touch it, and a lease held for a
|
||||
long staging would make a restore's drain fail. A stage that is not
|
||||
handed to the store (a refusal, an error, a cancel) is removed.
|
||||
"""
|
||||
uploads = [] if uploads is None else uploads # the caller removes them
|
||||
reason = await self._upload_problem(temp_path)
|
||||
if reason is not None:
|
||||
return _failure(reason, 400, "invalid-backup")
|
||||
staged = None
|
||||
issued = False
|
||||
try:
|
||||
try:
|
||||
staged = await self._stage_for_review(db, temp_path, uploads)
|
||||
except ValueError as invalid:
|
||||
return _failure(str(invalid), 400, "invalid-backup")
|
||||
try:
|
||||
async with self._jobs.lease("Restore preview"):
|
||||
live, fingerprint = await self._run_in_executor(
|
||||
self._live_fingerprint, db
|
||||
)
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
store = self._restore_plans
|
||||
if identity != self._database_identity(self.db):
|
||||
return _failure(
|
||||
"The database was replaced while the restore was reviewed. "
|
||||
"Review again.",
|
||||
409,
|
||||
"storage-changed",
|
||||
)
|
||||
if store.full():
|
||||
return _too_many_reviews()
|
||||
token = store.issue(staged, fingerprint, identity, live)
|
||||
issued = True
|
||||
self._schedule_restore_expiry()
|
||||
finally:
|
||||
if staged is not None and not issued:
|
||||
await self._discard_held([_Orphan(staged)])
|
||||
held = store.entry(token)
|
||||
name = os.path.basename(db.model.db_path)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"token": token,
|
||||
"expires_at": _iso(held.expires_at),
|
||||
"summary": dict(staged.summary),
|
||||
"schema": dict(staged.schema),
|
||||
"integrity": "ok",
|
||||
"documents": dict(staged.documents),
|
||||
"live": dict(live),
|
||||
"size": staged.size,
|
||||
"safety_backup": {
|
||||
"destination_label": f"{name}.backup_<date and time>"
|
||||
},
|
||||
},
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
|
||||
async def discard_restore_preview(self, request):
|
||||
token = request.match_info.get("token", "")
|
||||
if not 0 < len(token) <= MAX_TOKEN_LENGTH:
|
||||
return _failure("Unknown restore review.", 410, "plan-expired")
|
||||
held = self._restore_plans.pop(token)
|
||||
if held is None:
|
||||
await self._drop_expired_restores()
|
||||
return _failure("Unknown restore review.", 410, "plan-expired")
|
||||
await self._discard_held([held])
|
||||
return web.json_response({"success": True})
|
||||
|
||||
def _peek_restore(self, token):
|
||||
try:
|
||||
return self._restore_plans.peek(token), None
|
||||
except PlanUsed as used:
|
||||
job = self._jobs.get(used.job_id)
|
||||
return None, _failure(
|
||||
"This restore was already started.",
|
||||
409,
|
||||
"plan-used",
|
||||
job={"id": used.job_id, "title": job.title if job else None},
|
||||
)
|
||||
except PlanExpired:
|
||||
return None, _failure(
|
||||
"This review expired. Review the restore again.",
|
||||
410,
|
||||
"plan-expired",
|
||||
)
|
||||
|
||||
async def apply_restore(self, request):
|
||||
data, error = await self._read_json_body(request, max_bytes=APPLY_BODY_MAX)
|
||||
if error is not None:
|
||||
if error.status == 413:
|
||||
return _failure("Request body too large", 413, "too-large")
|
||||
return _failure(
|
||||
"Request body must be a JSON object", 400, "invalid-request"
|
||||
)
|
||||
token = data.get("token")
|
||||
if set(data) != {"token"} or not isinstance(token, str):
|
||||
return _failure("token must be the review's token", 400, "invalid-request")
|
||||
if not 0 < len(token) <= MAX_TOKEN_LENGTH:
|
||||
return _failure("token must be the review's token", 400, "invalid-request")
|
||||
await self._drop_expired_restores()
|
||||
held, refused = self._peek_restore(token)
|
||||
if refused is not None:
|
||||
await self._drop_expired_restores() # one peek found expired
|
||||
return refused
|
||||
db = self.db
|
||||
if held.db_identity != self._database_identity(db):
|
||||
await self._discard_popped(token)
|
||||
return _failure(
|
||||
"The database was replaced since the review. Review again.",
|
||||
409,
|
||||
"storage-changed",
|
||||
)
|
||||
try:
|
||||
async with self._jobs.lease("Restore review check"):
|
||||
_live, fingerprint = await self._run_in_executor(
|
||||
self._live_fingerprint, db
|
||||
)
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
if fingerprint != held.fingerprint:
|
||||
# It can never match again: free the stage for a new review.
|
||||
await self._discard_popped(token)
|
||||
return _failure(PLAN_CHANGED, 409, "plan-changed")
|
||||
# The token may have been used, expired or discarded meanwhile.
|
||||
held, refused = self._peek_restore(token)
|
||||
if refused is not None:
|
||||
await self._drop_expired_restores()
|
||||
return refused
|
||||
if db is not self.db:
|
||||
return _failure(
|
||||
"The database was replaced since the review. Review again.",
|
||||
409,
|
||||
"storage-changed",
|
||||
)
|
||||
try:
|
||||
job = self._jobs.start(
|
||||
RESTORE_JOB_KIND,
|
||||
RESTORE_JOB_TITLE,
|
||||
# It replaces the whole database: it runs alone, beside no
|
||||
# job of any class, like Move library.
|
||||
admission="exclusive",
|
||||
cancellable=False,
|
||||
scope={},
|
||||
groups=RESTORE_GROUPS,
|
||||
runner=lambda job: self._run_restore(job, db, held, token),
|
||||
)
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy) # the token stays usable
|
||||
# Same loop turn as the peek and the start: the runner has not run.
|
||||
self._restore_plans.take(token, job.id)
|
||||
return web.json_response({"success": True, "job": job.summary()}, status=202)
|
||||
|
||||
async def _run_restore(self, job, db, held, token=None):
|
||||
from .maintenance import _fail_job
|
||||
|
||||
staged = held.staged
|
||||
keep_stage = False
|
||||
try:
|
||||
job.update(status_line="Waiting for other requests to finish")
|
||||
if not await self._jobs.drain_leases():
|
||||
# Nothing ran: the reviewed stage stays usable (#335 M2),
|
||||
# unless it expired meanwhile or its slot was reused.
|
||||
if token is not None and self._restore_plans.give_back(token, held):
|
||||
keep_stage = True
|
||||
self._schedule_restore_expiry()
|
||||
job.scope["review_kept"] = True
|
||||
return _fail_job(job, "busy", REVIEW_KEPT)
|
||||
return _fail_job(job, "busy", REQUESTS_BUSY)
|
||||
job.update(status_line="Checking the library")
|
||||
_live, fingerprint = await self._run_in_executor(self._live_fingerprint, db)
|
||||
if db is not self.db or fingerprint != held.fingerprint:
|
||||
return _fail_job(job, "plan-changed", PLAN_CHANGED)
|
||||
job.update(status_line="Restoring the database")
|
||||
job.log_line("Restoring the reviewed backup")
|
||||
model = db.model
|
||||
try:
|
||||
backup_path = await self._publish(model, staged)
|
||||
except restore_stage.StageChanged:
|
||||
return _fail_job(job, "plan-changed", PLAN_CHANGED)
|
||||
except SafetyBackupFailed:
|
||||
return _fail_job(
|
||||
job,
|
||||
"backup-failed",
|
||||
"The current library could not be backed up; nothing was "
|
||||
"changed.",
|
||||
)
|
||||
except sqlite3.OperationalError as error:
|
||||
kept = (
|
||||
" The safety backup was kept."
|
||||
if getattr(error, "_prompt_manager_restore_backup_created", False)
|
||||
else ""
|
||||
)
|
||||
self.logger.error(f"Restore failed: {error}")
|
||||
return _fail_job(job, "restore-failed", NOT_RESTORED + kept)
|
||||
except Exception:
|
||||
self.logger.exception("Restore job failed")
|
||||
return _fail_job(job, None, NOT_RESTORED)
|
||||
finally:
|
||||
if not keep_stage:
|
||||
await self._discard_held([_Orphan(staged)])
|
||||
# The copy replaced the library: the restore is done (#335 review I1).
|
||||
# Every later step only logs a failure.
|
||||
await self._after_publish(model.db_path)
|
||||
if backup_path:
|
||||
job.log_line(f"Safety backup {os.path.basename(backup_path)}")
|
||||
else:
|
||||
job.log_line("No safety backup: there was no library to keep")
|
||||
repaired = documents_line(staged.documents)
|
||||
if repaired:
|
||||
job.log_line(repaired)
|
||||
line = restored_line(staged.summary)
|
||||
job.log_line(line)
|
||||
job.update(progress=100, status_line=line)
|
||||
|
||||
|
||||
class _Orphan:
|
||||
"""A stage with no store entry, in the shape ``_discard_held`` takes."""
|
||||
|
||||
def __init__(self, staged):
|
||||
self.staged = staged
|
||||
|
||||
|
||||
def _too_many_reviews():
|
||||
return _failure(
|
||||
"Two restores are already being reviewed. Discard one first.",
|
||||
429,
|
||||
"too-large",
|
||||
)
|
||||
+100
@@ -28,6 +28,15 @@ ones. One job of a kind runs at a time, and each job has an admission class
|
||||
A short legacy-page operation takes a ``hold``: a running, non-cancellable
|
||||
job that lasts as long as the request, so the Activity page lists it and a
|
||||
workspace job is refused while it runs (and the other way round).
|
||||
|
||||
Short request-scoped work that reads or writes the library beside any
|
||||
ordinary job (a Needs review decision, a maintenance or restore preview)
|
||||
takes a ``lease`` instead: it is not listed (so quick requests never push
|
||||
real jobs out of the finished history, and two of them never refuse each
|
||||
other), but it is refused while an ``exclusive`` job runs, and an
|
||||
``exclusive`` job that replaces the database (a restore) awaits
|
||||
``drain_leases`` before it switches the file, so no lease's connection is
|
||||
open across the switch (#249 S8).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
@@ -48,6 +57,9 @@ MAX_OUTCOME_ROWS = 200 # rows listed per outcome group; counters stay exact
|
||||
MAX_LOG_LINES = 2000
|
||||
LOG_CAP_LINE = "[Log capped at 2,000 lines]"
|
||||
MAX_FINISHED_JOBS = 20 # finished jobs this session keeps
|
||||
# The longest an exclusive job waits for running leases to end (a ceiling a
|
||||
# hung request reaches, not a pacing timer: leases end with their request).
|
||||
LEASE_DRAIN_SECONDS = 60
|
||||
|
||||
# The metadata scan's outcome groups, in counter-tile order of the spec's
|
||||
# mapping, and the per-file scan status each group collects. Files without
|
||||
@@ -403,6 +415,74 @@ class JobRegistry:
|
||||
def __init__(self, logger_=None):
|
||||
self._logger = logger_ or logger
|
||||
self._jobs = {}
|
||||
self._leases = {} # title -> running count
|
||||
self._drain_waiters = []
|
||||
|
||||
# ── leases ────────────────────────────────────────────────────────
|
||||
|
||||
def lease(self, title):
|
||||
"""An async context manager for short request-scoped library work.
|
||||
|
||||
Entering raises JobBusy, naming the job, while an ``exclusive`` job
|
||||
runs; otherwise the lease is counted until the block exits. Leases
|
||||
are never listed and never refuse one another or any job.
|
||||
"""
|
||||
return _Lease(self, title)
|
||||
|
||||
def lease_blocker(self):
|
||||
"""The running ``exclusive`` job a new lease is refused for, or None."""
|
||||
for job in self.running():
|
||||
if job.admission == "exclusive":
|
||||
return job
|
||||
return None
|
||||
|
||||
@property
|
||||
def leases(self):
|
||||
"""Running leases by title (a copy)."""
|
||||
return dict(self._leases)
|
||||
|
||||
def _take_lease(self, title):
|
||||
blocker = self.lease_blocker()
|
||||
if blocker is not None:
|
||||
raise JobBusy(blocker)
|
||||
self._leases[title] = self._leases.get(title, 0) + 1
|
||||
|
||||
def _release_lease(self, title):
|
||||
count = self._leases.get(title, 0) - 1
|
||||
if count > 0:
|
||||
self._leases[title] = count
|
||||
else:
|
||||
self._leases.pop(title, None)
|
||||
if not self._leases:
|
||||
waiters, self._drain_waiters = self._drain_waiters, []
|
||||
for waiter in waiters:
|
||||
if not waiter.done():
|
||||
waiter.set_result(True)
|
||||
|
||||
async def drain_leases(self, timeout=None):
|
||||
"""Wait until no lease runs; False when ``timeout`` ended the wait.
|
||||
|
||||
For an ``exclusive`` job about to switch the database: once it runs
|
||||
no new lease is admitted, so the count only goes down.
|
||||
"""
|
||||
if not self._leases:
|
||||
return True
|
||||
waiter = asyncio.get_running_loop().create_future()
|
||||
self._drain_waiters.append(waiter)
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
asyncio.shield(waiter),
|
||||
LEASE_DRAIN_SECONDS if timeout is None else timeout,
|
||||
)
|
||||
return True
|
||||
except asyncio.TimeoutError:
|
||||
self._logger.warning(
|
||||
"Still running after the wait: " + ", ".join(sorted(self._leases))
|
||||
)
|
||||
return False
|
||||
finally:
|
||||
if waiter in self._drain_waiters:
|
||||
self._drain_waiters.remove(waiter)
|
||||
|
||||
def start(
|
||||
self,
|
||||
@@ -565,3 +645,23 @@ class _Hold:
|
||||
if self._job.running:
|
||||
self._job.finish("finished" if exc_type is None else "failed")
|
||||
return False
|
||||
|
||||
|
||||
class _Lease:
|
||||
"""The async context manager ``JobRegistry.lease`` returns."""
|
||||
|
||||
def __init__(self, registry, title):
|
||||
self._registry = registry
|
||||
self._title = title
|
||||
self._held = False
|
||||
|
||||
async def __aenter__(self):
|
||||
self._registry._take_lease(self._title)
|
||||
self._held = True
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
if self._held:
|
||||
self._held = False
|
||||
self._registry._release_lease(self._title)
|
||||
return False
|
||||
|
||||
@@ -0,0 +1,483 @@
|
||||
"""The staged LoRA reference import as a reviewed Activity job (#221, S10).
|
||||
|
||||
``POST /prompt_manager/lora/import/preview`` (body ``{}``) reviews an import:
|
||||
it lists the LoraManager metadata files and parses them, and counts the
|
||||
current imports, those a saved pair uses and the example images an import
|
||||
would download. It writes nothing and requests nothing. It answers a
|
||||
``Plan`` (the maintenance plan's shape, ``py/api/maintenance.py``) plus
|
||||
``{lora_files, with_metadata, current_imports, blocked_by_pairs,
|
||||
planned_downloads, network:{planned_downloads, api_key}}``. A plan that saved
|
||||
pairs block carries no token (nothing can be applied).
|
||||
|
||||
``POST /prompt_manager/lora/import/apply {token}`` starts the job (202
|
||||
``{job}``): kind ``lora_import``, titled "LoRA reference import", admission
|
||||
``records``, cancellable while it downloads only. Refused before it starts:
|
||||
``plan-changed`` (409: the metadata files changed since the review),
|
||||
``plan-expired``, ``plan-used``, ``storage-changed``, ``busy``,
|
||||
``insufficient-space`` (as maintenance). The job stages the records
|
||||
(downloads into the cache first, each counted in ``downloaded`` or
|
||||
``download_errors``), takes the safety backup the policy promises, then
|
||||
replaces the collection in one transaction bound to the reviewed imports. Any
|
||||
failure leaves the previous imports exactly as they were, and the job says so.
|
||||
|
||||
The legacy ``POST /lora/scan`` runs the same stages (``stage_lora_import``
|
||||
and ``replace_lora_imports``) under its SSE event shape.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sqlite3
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from .. import lora_import, lora_utils
|
||||
from .job_routes import busy_response
|
||||
from .jobs import LORA_IMPORT_JOB_KIND, JobBusy
|
||||
from .maintenance import (
|
||||
MAX_TOKEN_LENGTH,
|
||||
_failure,
|
||||
_fail_job,
|
||||
_invalid,
|
||||
_iso,
|
||||
disk_plan,
|
||||
short_space_message,
|
||||
)
|
||||
from .maintenance_plans import ScopeTooLarge
|
||||
|
||||
try:
|
||||
from ...database.category_import import EmptyReplacement
|
||||
from ...database.maintenance import PlanChanged
|
||||
from ...database.pairs import PairDependencyError
|
||||
except ImportError: # pragma: no cover - flat test imports
|
||||
from database.category_import import EmptyReplacement
|
||||
from database.maintenance import PlanChanged
|
||||
from database.pairs import PairDependencyError
|
||||
|
||||
LORA_OPERATION = "lora_import"
|
||||
# The review reads the library: a request lease, so it is refused while a
|
||||
# restore runs and a restore waits for it before switching the file (S8).
|
||||
LORA_PREVIEW_LEASE = "LoRA import review"
|
||||
CATEGORY = lora_import.LORA_CATEGORY
|
||||
SOURCE_CHANGED = (
|
||||
"The LoRA metadata files changed since the review; nothing was changed. "
|
||||
"Review the import again."
|
||||
)
|
||||
IMPORTS_CHANGED = (
|
||||
"The imported LoRA prompts changed since the review; the previous imports "
|
||||
"were kept. Review the import again."
|
||||
)
|
||||
BLOCKED = (
|
||||
"Saved pairs use some imported LoRA prompts, so the collection cannot be "
|
||||
"replaced; the previous imports were kept."
|
||||
)
|
||||
INTEGRATION_CHANGED = (
|
||||
"The LoraManager integration changed since the review; nothing was changed. "
|
||||
"Review the import again."
|
||||
)
|
||||
EMPTY = "No LoRA could be prepared for import; the previous imports were kept."
|
||||
FAILED = "The import failed; the previous LoRA imports were kept."
|
||||
CACHE_NOTE = "Images already downloaded stay in the image cache."
|
||||
CANCELLED = "Cancelled before anything was written; the previous imports were kept."
|
||||
DISABLED = "LoraManager integration is not enabled"
|
||||
NO_PATH = "LoraManager path not configured"
|
||||
|
||||
|
||||
def _noun(count, word, plural=None):
|
||||
return word if count == 1 else (plural or word + "s")
|
||||
|
||||
|
||||
def _count(count, word, plural=None):
|
||||
return f"{count:,} {_noun(count, word, plural)}"
|
||||
|
||||
|
||||
def _import_line(result, skipped):
|
||||
parts = [f"Imported {_count(result['imported'], 'LoRA')}"]
|
||||
if result["linked"] or result["duplicates"]:
|
||||
linked = result["linked"] + result["duplicates"]
|
||||
parts.append(f"{linked:,} linked to an existing prompt")
|
||||
if skipped:
|
||||
parts.append(f"{skipped:,} skipped")
|
||||
return " · ".join(parts)
|
||||
|
||||
|
||||
def _all_downloads_failed(errors):
|
||||
"""The warning when every planned download failed (#337)."""
|
||||
if errors == 1:
|
||||
return "Warning: the example image could not be downloaded"
|
||||
return f"Warning: none of the {errors:,} example images could be downloaded"
|
||||
|
||||
|
||||
def _with_downloads(line, downloaded, errors):
|
||||
"""``line`` with the failed downloads: counted when some failed, led by
|
||||
a warning when every planned download failed (#337). The job still
|
||||
finishes: the import itself succeeded and the downloads are best effort."""
|
||||
if not errors:
|
||||
return line
|
||||
if not downloaded:
|
||||
return f"{_all_downloads_failed(errors)} · {line}"
|
||||
failed = f"{_count(errors, 'example image')} could not be downloaded"
|
||||
return f"{line} · {failed}"
|
||||
|
||||
|
||||
class LoraImportRoutesMixin:
|
||||
"""Mixin providing the reviewed LoRA import (preview, apply, job)."""
|
||||
|
||||
def _register_lora_import_routes(self, routes):
|
||||
@routes.post("/prompt_manager/lora/import/preview")
|
||||
async def lora_import_preview_route(request):
|
||||
return await self.preview_lora_import(request)
|
||||
|
||||
@routes.post("/prompt_manager/lora/import/apply")
|
||||
async def lora_import_apply_route(request):
|
||||
return await self.apply_lora_import(request)
|
||||
|
||||
# ── shared stages ─────────────────────────────────────────────────
|
||||
|
||||
async def _lora_source(self):
|
||||
"""``(path, None)``: the configured, allowed LoraManager path, or
|
||||
``(None, (code, message))`` when the import cannot run."""
|
||||
from ..config import IntegrationConfig
|
||||
|
||||
if not IntegrationConfig.LORA_MANAGER_ENABLED:
|
||||
return None, ("lora-disabled", DISABLED)
|
||||
lm_path = IntegrationConfig.LORA_MANAGER_PATH
|
||||
if not lm_path:
|
||||
return None, ("lora-path", NO_PATH)
|
||||
allowed = await self._run_in_executor(
|
||||
lora_utils.resolve_lora_manager_path, lm_path
|
||||
)
|
||||
if allowed is None:
|
||||
from .lora_integration import _LORA_PATH_ERROR
|
||||
|
||||
return None, ("lora-path", _LORA_PATH_ERROR)
|
||||
return allowed, None
|
||||
|
||||
async def stage_lora_import(self, files, *, on_record=None, cancelled=None):
|
||||
"""Stage ``files`` off the loop: records, skips and download counts."""
|
||||
from ..config import IntegrationConfig
|
||||
|
||||
cache_dir = await self._run_in_executor(lora_utils.get_lora_image_cache_dir)
|
||||
await self._run_in_executor(lora_utils.sweep_stale_downloads, cache_dir)
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
def report(index, record, staged):
|
||||
if on_record is not None:
|
||||
loop.call_soon_threadsafe(on_record, index, record, staged)
|
||||
|
||||
return await self._run_in_executor(
|
||||
lambda: lora_import.stage(
|
||||
files,
|
||||
cache_dir,
|
||||
IntegrationConfig.CIVITAI_API_KEY,
|
||||
on_record=report,
|
||||
cancelled=cancelled or (lambda: False),
|
||||
logger=self.logger,
|
||||
)
|
||||
)
|
||||
|
||||
async def lora_files_unchanged(self, lm_path, expected):
|
||||
"""Whether the metadata files under ``lm_path`` still fingerprint as
|
||||
``expected`` (listed again, so an added file counts too)."""
|
||||
|
||||
def current():
|
||||
return lora_import.fingerprint(lora_import.discover(lm_path))
|
||||
|
||||
try:
|
||||
return await self._run_in_executor(current) == expected
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
async def replace_lora_imports(self, db, staged, *, expected_ids, allow_empty):
|
||||
"""Swap the collection for ``staged.records`` in one transaction."""
|
||||
return await self._run_in_executor(
|
||||
lambda: db.replace_category_prompts(
|
||||
CATEGORY,
|
||||
staged.records,
|
||||
expected_ids=expected_ids,
|
||||
allow_empty=allow_empty,
|
||||
)
|
||||
)
|
||||
|
||||
# ── preview ───────────────────────────────────────────────────────
|
||||
|
||||
async def preview_lora_import(self, request):
|
||||
try:
|
||||
async with self._jobs.lease(LORA_PREVIEW_LEASE):
|
||||
return await self._preview_lora_import(request)
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
|
||||
async def _preview_lora_import(self, request):
|
||||
data, error = await self._read_maintenance_body(request, ())
|
||||
if error is not None:
|
||||
return error
|
||||
lm_path, refused = await self._lora_source()
|
||||
if refused is not None:
|
||||
return _failure(refused[1], 400, refused[0])
|
||||
db = self.db
|
||||
identity = self._database_identity(db)
|
||||
try:
|
||||
cache_dir = await self._run_in_executor(
|
||||
lambda: lora_utils.get_lora_image_cache_dir(create=False)
|
||||
)
|
||||
found = await self._run_in_executor(lora_import.preview, lm_path, cache_dir)
|
||||
state = await self._run_in_executor(db.category_import_state, CATEGORY)
|
||||
except Exception:
|
||||
self.logger.exception("LoRA import preview failed")
|
||||
return _failure("The LoRA import could not be reviewed.", 500)
|
||||
if identity != self._database_identity(self.db):
|
||||
return _failure(
|
||||
"The database was replaced while it was reviewed. Review again.",
|
||||
409,
|
||||
"storage-changed",
|
||||
)
|
||||
return await self._lora_plan_response(db, identity, lm_path, found, state)
|
||||
|
||||
async def _lora_plan_response(self, db, identity, lm_path, found, state):
|
||||
from ..config import IntegrationConfig
|
||||
|
||||
current = len(state["ids"])
|
||||
blocked = state["paired"]
|
||||
policy = self._safety_backup_policy()
|
||||
try:
|
||||
disk, _short, size = await self._run_in_executor(
|
||||
disk_plan, db.model.db_path, backup=policy != "off", vacuum=False
|
||||
)
|
||||
except (OSError, sqlite3.Error):
|
||||
self.logger.exception("Could not measure the disk space for a plan")
|
||||
return _failure("The LoRA import could not be reviewed.", 500)
|
||||
token = None
|
||||
expires_at = None
|
||||
if not blocked:
|
||||
payload = {
|
||||
"path": lm_path,
|
||||
"fingerprint": found["fingerprint"],
|
||||
"expected_ids": state["ids"],
|
||||
"empty": found["with_metadata"] == 0,
|
||||
"backup": policy,
|
||||
}
|
||||
try:
|
||||
token = self._maintenance_plans.issue(
|
||||
LORA_OPERATION, found["fingerprint"], payload, identity
|
||||
)
|
||||
except ScopeTooLarge:
|
||||
return _failure(
|
||||
"This import is too large to review at once.", 413, "too-large"
|
||||
)
|
||||
expires_at = _iso(self._maintenance_plans.peek(token).expires_at)
|
||||
name = os.path.basename(db.model.db_path)
|
||||
downloads = found["planned_downloads"]
|
||||
reasons = (
|
||||
[f"{_count(blocked, 'imported prompt')} used by saved pairs"]
|
||||
if blocked
|
||||
else []
|
||||
)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"operation": LORA_OPERATION,
|
||||
"token": token,
|
||||
"expires_at": expires_at,
|
||||
"lora_files": found["lora_files"],
|
||||
"with_metadata": found["with_metadata"],
|
||||
"current_imports": current,
|
||||
"blocked_by_pairs": blocked,
|
||||
"planned_downloads": downloads,
|
||||
"scope": {
|
||||
"count": found["with_metadata"],
|
||||
"label": (
|
||||
f"Replace {_count(current, 'imported prompt')} with "
|
||||
f"{_count(found['with_metadata'], 'LoRA')}"
|
||||
),
|
||||
},
|
||||
"protected": {"count": blocked, "reasons": reasons},
|
||||
"backup": {
|
||||
"policy": policy,
|
||||
"destination_label": f"{name}.backup_<date and time>",
|
||||
"expected_bytes": size if policy != "off" else None,
|
||||
},
|
||||
"media": {
|
||||
"deletes_files": False,
|
||||
"label": "No files are removed; image links are replaced",
|
||||
},
|
||||
"network": {
|
||||
"planned_downloads": downloads,
|
||||
"api_key": bool(IntegrationConfig.CIVITAI_API_KEY),
|
||||
},
|
||||
"details": [{"label": n, "detail": ""} for n in found["names"]],
|
||||
"disk": disk,
|
||||
},
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
|
||||
# ── apply ─────────────────────────────────────────────────────────
|
||||
|
||||
async def apply_lora_import(self, request):
|
||||
data, error = await self._read_maintenance_body(request, ("token",))
|
||||
if error is not None:
|
||||
return error
|
||||
token = data.get("token")
|
||||
if not isinstance(token, str) or not 0 < len(token) <= MAX_TOKEN_LENGTH:
|
||||
return _invalid("token must be the plan's token")
|
||||
plan, refused = self._peek_plan(token)
|
||||
if refused is not None:
|
||||
return refused
|
||||
if plan.operation != LORA_OPERATION:
|
||||
return _failure(
|
||||
"This review expired. Review the plan again.", 410, "plan-expired"
|
||||
)
|
||||
store = self._maintenance_plans
|
||||
db = self.db
|
||||
if plan.db_identity != self._database_identity(db):
|
||||
store.discard(token)
|
||||
return _failure(
|
||||
"The database was replaced since the review. Review again.",
|
||||
409,
|
||||
"storage-changed",
|
||||
)
|
||||
lm_path = plan.payload["path"]
|
||||
# M5: the integration must still be on and at the reviewed install.
|
||||
current, _refused = await self._lora_source()
|
||||
if current != lm_path:
|
||||
store.discard(token)
|
||||
return _failure(INTEGRATION_CHANGED, 409, "plan-changed")
|
||||
try:
|
||||
files = await self._run_in_executor(lora_import.discover, lm_path)
|
||||
now = await self._run_in_executor(lora_import.fingerprint, files)
|
||||
backup = self._backup_promised(plan)
|
||||
_disk, short, _size = await self._run_in_executor(
|
||||
disk_plan, db.model.db_path, backup=backup, vacuum=False
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception("LoRA import could not start")
|
||||
return _failure(
|
||||
"The import could not start. Nothing was changed.", 500, "apply-failed"
|
||||
)
|
||||
if now != plan.fingerprint:
|
||||
store.discard(token)
|
||||
return _failure(SOURCE_CHANGED, 409, "plan-changed")
|
||||
if short is not None:
|
||||
return _failure(short_space_message(short), 507, "insufficient-space")
|
||||
plan, refused = self._peek_plan(token)
|
||||
if refused is not None:
|
||||
return refused
|
||||
try:
|
||||
job = self._jobs.start(
|
||||
LORA_IMPORT_JOB_KIND,
|
||||
lora_import.JOB_TITLE,
|
||||
admission="records",
|
||||
cancellable=True,
|
||||
scope={"operation": LORA_OPERATION, "lora_files": len(files)},
|
||||
groups=lora_import.GROUPS,
|
||||
runner=lambda job: self._run_lora_import(job, db, plan, files, backup),
|
||||
)
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
store.consume(token, LORA_OPERATION, job_id=job.id)
|
||||
return web.json_response({"success": True, "job": job.summary()}, status=202)
|
||||
|
||||
# ── the job ───────────────────────────────────────────────────────
|
||||
|
||||
async def _run_lora_import(self, job, db, plan, files, backup_on):
|
||||
total = len(files)
|
||||
job.update(total=total, status_line="Preparing LoRA records")
|
||||
job.log_line(f"Found {_count(total, 'LoRA metadata file')}")
|
||||
|
||||
def on_record(index, record, staged):
|
||||
job.update(
|
||||
processed=index + 1,
|
||||
progress=int(90 * (index + 1) / max(total, 1)),
|
||||
status_line=f"Prepared {index + 1:,} of {total:,}",
|
||||
)
|
||||
job.publish({"type": "progress", "progress": job.progress})
|
||||
|
||||
try:
|
||||
staged = await self.stage_lora_import(
|
||||
files, on_record=on_record, cancelled=lambda: job.cancelled
|
||||
)
|
||||
except lora_import.ImportCancelled:
|
||||
job.log_line(f"{CANCELLED} {CACHE_NOTE}")
|
||||
job.update(status_line=CANCELLED)
|
||||
job.finish("cancelled")
|
||||
return
|
||||
except Exception:
|
||||
self.logger.exception("LoRA import staging failed")
|
||||
return _fail_job(job, "import-failed", f"{FAILED} {CACHE_NOTE}")
|
||||
self._count_downloads(job, staged)
|
||||
# M4: what staging skipped stays on the job whatever happens next.
|
||||
job.record_counted(
|
||||
"skipped",
|
||||
len(staged.skipped),
|
||||
[{"path": "", "detail": name} for name in staged.skipped],
|
||||
)
|
||||
# Past this point nothing stops halfway: the replace is one transaction.
|
||||
# Published, so a view that offered Cancel during the downloads drops it.
|
||||
job.cancellable = False
|
||||
job.publish({"type": "progress", "progress": job.progress})
|
||||
if job.cancelled:
|
||||
job.log_line(f"{CANCELLED} {CACHE_NOTE}")
|
||||
job.update(status_line=CANCELLED)
|
||||
job.finish("cancelled")
|
||||
return
|
||||
ok, _backup = await self._safety_backup_step(job, db, backup_on)
|
||||
if not ok:
|
||||
return
|
||||
# I1: the files are listed and fingerprinted again right before the
|
||||
# replace, so a file changed, removed or added while the downloads ran
|
||||
# (LoraManager refreshing its CivitAI info) aborts with nothing deleted.
|
||||
if not await self.lora_files_unchanged(plan.payload["path"], plan.fingerprint):
|
||||
return _fail_job(job, "plan-changed", f"{SOURCE_CHANGED} {CACHE_NOTE}")
|
||||
job.update(status_line="Replacing the LoRA collection", progress=95)
|
||||
await self._replace_in_job(job, db, plan, staged)
|
||||
|
||||
def _count_downloads(self, job, staged):
|
||||
downloaded = staged.stats.get("downloaded", 0)
|
||||
errors = staged.stats.get("download_errors", 0)
|
||||
job.record_counted("downloaded", downloaded)
|
||||
rows = []
|
||||
if errors and not downloaded:
|
||||
warning = _all_downloads_failed(errors)
|
||||
rows = [{"path": "", "detail": f"{warning}; the log has the details"}]
|
||||
job.log_line(warning)
|
||||
job.record_counted("download_errors", errors, rows)
|
||||
if downloaded or errors:
|
||||
job.log_line(
|
||||
f"Downloaded {_count(downloaded, 'example image')}"
|
||||
+ (f"; {errors:,} could not be downloaded" if errors else "")
|
||||
)
|
||||
|
||||
async def _replace_in_job(self, job, db, plan, staged):
|
||||
try:
|
||||
result = await self.replace_lora_imports(
|
||||
db,
|
||||
staged,
|
||||
expected_ids=plan.payload["expected_ids"],
|
||||
allow_empty=plan.payload["empty"],
|
||||
)
|
||||
except PlanChanged:
|
||||
return _fail_job(job, "plan-changed", f"{IMPORTS_CHANGED} {CACHE_NOTE}")
|
||||
except PairDependencyError:
|
||||
return _fail_job(job, "blocked-by-pairs", f"{BLOCKED} {CACHE_NOTE}")
|
||||
except EmptyReplacement:
|
||||
return _fail_job(job, "nothing-to-import", f"{EMPTY} {CACHE_NOTE}")
|
||||
except Exception:
|
||||
self.logger.exception("LoRA import failed")
|
||||
return _fail_job(job, "import-failed", f"{FAILED} {CACHE_NOTE}")
|
||||
rows = {"imported": [], "linked": []}
|
||||
for row in result["rows"]:
|
||||
group = "imported" if row["outcome"] == "imported" else "linked"
|
||||
rows[group].append({"path": "", "detail": row["name"]})
|
||||
job.record_counted("imported", result["imported"], rows["imported"])
|
||||
job.record_counted(
|
||||
"linked", result["linked"] + result["duplicates"], rows["linked"]
|
||||
)
|
||||
line = _import_line(result, len(staged.skipped))
|
||||
if result["removed"]:
|
||||
job.log_line(f"Replaced {_count(result['removed'], 'previous import')}")
|
||||
job.log_line(line)
|
||||
line = _with_downloads(
|
||||
line,
|
||||
staged.stats.get("downloaded", 0),
|
||||
staged.stats.get("download_errors", 0),
|
||||
)
|
||||
job.update(progress=100, processed=job.total, status_line=line)
|
||||
+193
-136
@@ -1,7 +1,9 @@
|
||||
"""LoraManager integration API routes for PromptManager."""
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import threading
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
@@ -37,6 +39,21 @@ def _lora_settings_mutation(values):
|
||||
return mutate
|
||||
|
||||
|
||||
# The admin page's own words for the refusals it can meet (it has no review).
|
||||
_FILES_CHANGED = (
|
||||
"The LoRA metadata files changed during the scan; the previous imports "
|
||||
"were kept. Run the scan again."
|
||||
)
|
||||
_IMPORTS_CHANGED = (
|
||||
"The imported LoRA prompts changed during the scan; the previous imports "
|
||||
"were kept. Run the scan again."
|
||||
)
|
||||
|
||||
|
||||
class LoraFilesChanged(Exception):
|
||||
"""A metadata file changed, appeared or went while the legacy scan staged."""
|
||||
|
||||
|
||||
def _lora_error_frame(message):
|
||||
"""An import stream's error event, as the admin page's modal reads it.
|
||||
|
||||
@@ -46,6 +63,23 @@ def _lora_error_frame(message):
|
||||
return {"type": "error", "status": message, "progress": 0, "message": message}
|
||||
|
||||
|
||||
def _closed(request):
|
||||
"""A check (safe off the loop) for whether ``request``'s client left."""
|
||||
|
||||
def closed():
|
||||
transport = request.transport
|
||||
return transport is None or transport.is_closing()
|
||||
|
||||
return closed
|
||||
|
||||
|
||||
def _stopped(error) -> bool:
|
||||
"""Whether a legacy scan ended because its page closed the connection."""
|
||||
from .. import lora_import
|
||||
|
||||
return isinstance(error, (ConnectionError, lora_import.ImportCancelled))
|
||||
|
||||
|
||||
class LoraIntegrationMixin:
|
||||
"""Mixin providing LoraManager detection, scanning, and trigger word endpoints."""
|
||||
|
||||
@@ -108,7 +142,11 @@ class LoraIntegrationMixin:
|
||||
"""Get current LoraManager integration status."""
|
||||
try:
|
||||
from ..config import IntegrationConfig
|
||||
from ..lora_utils import detect_lora_manager, get_trigger_cache
|
||||
from ..lora_utils import (
|
||||
detect_lora_manager,
|
||||
get_trigger_cache,
|
||||
resolve_lora_manager_path,
|
||||
)
|
||||
|
||||
config = IntegrationConfig.get_config()["lora_manager"]
|
||||
cache = get_trigger_cache()
|
||||
@@ -117,6 +155,12 @@ class LoraIntegrationMixin:
|
||||
detected_path = await self._run_in_executor(
|
||||
detect_lora_manager, config.get("path", "")
|
||||
)
|
||||
# A detected install reached through a link outside custom_nodes is
|
||||
# found but cannot be enabled (S10 review N5): say which.
|
||||
usable = bool(detected_path) and (
|
||||
await self._run_in_executor(resolve_lora_manager_path, detected_path)
|
||||
is not None
|
||||
)
|
||||
|
||||
return web.json_response(
|
||||
{
|
||||
@@ -126,6 +170,7 @@ class LoraIntegrationMixin:
|
||||
"trigger_words_enabled": config["trigger_words_enabled"],
|
||||
"has_civitai_api_key": bool(config.get("civitai_api_key")),
|
||||
"detected": detected_path is not None,
|
||||
"detected_usable": usable,
|
||||
"detected_path": (
|
||||
self._public_path(detected_path) if detected_path else ""
|
||||
),
|
||||
@@ -168,6 +213,9 @@ class LoraIntegrationMixin:
|
||||
async def lora_enable(self, request):
|
||||
"""Enable or disable LoraManager integration and save it to config.json.
|
||||
|
||||
``expected_revision`` (optional, GET /settings' ``revision``) refuses
|
||||
a save over a config.json changed meanwhile (``settings-changed``).
|
||||
|
||||
The values are committed through the config store first (#220) and
|
||||
published at runtime only once config.json holds them; a save that
|
||||
cannot be written reports ``persisted: false`` and changes nothing.
|
||||
@@ -180,6 +228,11 @@ class LoraIntegrationMixin:
|
||||
for flag in _LORA_FLAGS:
|
||||
if flag in data and not isinstance(data[flag], bool):
|
||||
return self._bad_lora_request(f"{flag} must be true or false")
|
||||
# The System Save row sends the revision it read (#220); the
|
||||
# legacy page sends none and merges into what is on disk.
|
||||
expected_revision = data.get("expected_revision")
|
||||
if expected_revision is not None and not isinstance(expected_revision, str):
|
||||
return self._bad_lora_request("expected_revision must be a string")
|
||||
enabled = data.get("enabled", False)
|
||||
path = data.get("path", "")
|
||||
trigger_words = data.get("trigger_words_enabled", False)
|
||||
@@ -226,7 +279,7 @@ class LoraIntegrationMixin:
|
||||
_saved, revision = await self._run_in_executor(
|
||||
get_config_store().commit,
|
||||
_lora_settings_mutation(values),
|
||||
expected_revision=None,
|
||||
expected_revision=expected_revision,
|
||||
publish=publish,
|
||||
)
|
||||
except (ConfigUnreadable, RevisionConflict, OSError) as exc:
|
||||
@@ -308,43 +361,69 @@ class LoraIntegrationMixin:
|
||||
) as job:
|
||||
await response.prepare(request)
|
||||
try:
|
||||
await self._lora_scan_stream(allowed, send_progress)
|
||||
await self._lora_scan_stream(
|
||||
allowed, send_progress, disconnected=_closed(request)
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
# The server cancelled the handler when the page left.
|
||||
job.finish("cancelled")
|
||||
raise
|
||||
except Exception as e:
|
||||
self.logger.exception("LoRA scan failed")
|
||||
job.finish("failed")
|
||||
await send_progress(_lora_error_frame(self._public_error(e)))
|
||||
|
||||
await response.write_eof()
|
||||
# A page that left stopped the scan; nothing failed (R2).
|
||||
job.finish("cancelled" if _stopped(e) else "failed")
|
||||
await self._send_scan_failure(e, send_progress)
|
||||
with contextlib.suppress(ConnectionError, RuntimeError):
|
||||
await response.write_eof()
|
||||
return response
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"LoRA scan failed: {e}")
|
||||
return self._error_response(e)
|
||||
|
||||
async def _lora_scan_stream(self, lm_path, send_progress):
|
||||
"""Import every LoRA under ``lm_path``, reporting through ``send_progress``."""
|
||||
from ..config import IntegrationConfig
|
||||
from ..lora_utils import (
|
||||
download_civitai_images,
|
||||
find_lora_directories,
|
||||
get_example_prompt_from_metadata,
|
||||
get_lora_image_cache_dir,
|
||||
get_preview_images_from_metadata,
|
||||
get_trigger_words_from_metadata,
|
||||
get_model_name_from_metadata,
|
||||
read_lora_metadata,
|
||||
async def _send_scan_failure(self, error, send_progress):
|
||||
"""The error frame for a failed legacy scan; quiet when the page left.
|
||||
|
||||
Expected refusals (files or imports changed, saved pairs, nothing to
|
||||
import) are warnings in the admin page's own words; anything else is
|
||||
logged with its traceback.
|
||||
"""
|
||||
from .lora_import_routes import (
|
||||
EmptyReplacement,
|
||||
PairDependencyError,
|
||||
PlanChanged,
|
||||
)
|
||||
|
||||
await send_progress(
|
||||
{
|
||||
"type": "progress",
|
||||
"status": "Clearing previous lora-manager imports...",
|
||||
"progress": 0,
|
||||
}
|
||||
)
|
||||
if _stopped(error):
|
||||
self.logger.info("LoRA scan stopped: the page closed its connection")
|
||||
return
|
||||
if isinstance(error, PlanChanged):
|
||||
message = _IMPORTS_CHANGED
|
||||
elif isinstance(error, LoraFilesChanged):
|
||||
message = str(error)
|
||||
else:
|
||||
message = self._public_error(error)
|
||||
if isinstance(
|
||||
error,
|
||||
(PlanChanged, LoraFilesChanged, EmptyReplacement, PairDependencyError),
|
||||
):
|
||||
self.logger.warning(f"LoRA scan refused: {message}")
|
||||
else:
|
||||
self.logger.exception("LoRA scan failed")
|
||||
with contextlib.suppress(ConnectionError, RuntimeError):
|
||||
await send_progress(_lora_error_frame(message))
|
||||
|
||||
# Clear previous imports so reimport is always clean
|
||||
await self._run_in_executor(self.db.delete_prompts_by_category, "lora-manager")
|
||||
async def _lora_scan_stream(
|
||||
self, lm_path, send_progress, disconnected=lambda: False
|
||||
):
|
||||
"""Import every LoRA under ``lm_path``, reporting through ``send_progress``.
|
||||
|
||||
The staged path of the System import (#221, ``lora_import_routes``):
|
||||
discover, stage every record (downloads into the cache first), then
|
||||
replace the collection in one transaction. A failure at any stage
|
||||
raises with the previous imports untouched; an empty replacement of a
|
||||
non-empty collection is refused (it needs the reviewed import).
|
||||
"""
|
||||
from .. import lora_import
|
||||
|
||||
await send_progress(
|
||||
{
|
||||
@@ -353,19 +432,8 @@ class LoraIntegrationMixin:
|
||||
"progress": 2,
|
||||
}
|
||||
)
|
||||
|
||||
lora_dirs = await self._run_in_executor(find_lora_directories, lm_path)
|
||||
|
||||
# Collect all metadata files
|
||||
meta_files = []
|
||||
for d in lora_dirs:
|
||||
dir_path = Path(d)
|
||||
meta_files.extend(dir_path.rglob("*.metadata.json"))
|
||||
|
||||
total = len(meta_files)
|
||||
imported = 0
|
||||
skipped = 0
|
||||
|
||||
files = await self._run_in_executor(lora_import.discover, lm_path)
|
||||
total = len(files)
|
||||
await send_progress(
|
||||
{
|
||||
"type": "progress",
|
||||
@@ -375,116 +443,105 @@ class LoraIntegrationMixin:
|
||||
}
|
||||
)
|
||||
|
||||
cache_dir = get_lora_image_cache_dir()
|
||||
reviewed = await self._run_in_executor(lora_import.fingerprint, files)
|
||||
events = asyncio.Queue()
|
||||
state = await self._run_in_executor(
|
||||
self.db.category_import_state, lora_import.LORA_CATEGORY
|
||||
)
|
||||
# Records whose prompt already exists outside the collection are
|
||||
# linked, not imported (the running counts; the end ones are exact).
|
||||
known = set()
|
||||
|
||||
for i, meta_file in enumerate(meta_files):
|
||||
metadata = await self._run_in_executor(read_lora_metadata, meta_file)
|
||||
if not metadata:
|
||||
skipped += 1
|
||||
continue
|
||||
def on_record(index, record, _staged):
|
||||
events.put_nowait((index, record))
|
||||
|
||||
model_name = get_model_name_from_metadata(metadata)
|
||||
trigger_words = get_trigger_words_from_metadata(metadata)
|
||||
|
||||
# Collect all images: local previews + downloaded civitai examples
|
||||
preview_paths = await self._run_in_executor(
|
||||
get_preview_images_from_metadata, metadata, meta_file
|
||||
)
|
||||
civitai_paths = await self._run_in_executor(
|
||||
download_civitai_images,
|
||||
metadata,
|
||||
meta_file,
|
||||
cache_dir,
|
||||
IntegrationConfig.CIVITAI_API_KEY,
|
||||
)
|
||||
|
||||
# Merge, local first, dedup
|
||||
seen = set(preview_paths)
|
||||
all_images = list(preview_paths)
|
||||
for cp in civitai_paths:
|
||||
if cp not in seen:
|
||||
all_images.append(cp)
|
||||
seen.add(cp)
|
||||
|
||||
# Build prompt text: prefer example prompt, then model name
|
||||
example_prompt = get_example_prompt_from_metadata(metadata)
|
||||
prompt_text = example_prompt or model_name
|
||||
|
||||
# Build tags
|
||||
tags = ["lora-manager", f"lora:{model_name}"]
|
||||
tags.extend(trigger_words)
|
||||
|
||||
# Save to database via existing mechanism
|
||||
try:
|
||||
import hashlib
|
||||
|
||||
prompt_hash = hashlib.sha256(
|
||||
prompt_text.strip().lower().encode("utf-8")
|
||||
).hexdigest()
|
||||
counts = {"imported": 0}
|
||||
|
||||
async def report(index, record):
|
||||
if record is not None and record["hash"] not in known:
|
||||
known.add(record["hash"])
|
||||
existing = await self._run_in_executor(
|
||||
self.db.get_prompt_by_hash, prompt_hash
|
||||
self.db.existing_prompt_hashes,
|
||||
[record["hash"]],
|
||||
lora_import.LORA_CATEGORY,
|
||||
)
|
||||
|
||||
if existing:
|
||||
# Link all images
|
||||
for pp in all_images:
|
||||
await self._run_in_executor(
|
||||
self.db.link_image_to_prompt,
|
||||
existing["id"],
|
||||
pp,
|
||||
)
|
||||
skipped += 1
|
||||
else:
|
||||
prompt_id = await self._run_in_executor(
|
||||
self.db.save_prompt,
|
||||
prompt_text,
|
||||
"lora-manager", # category
|
||||
tags,
|
||||
None, # rating
|
||||
None, # notes
|
||||
prompt_hash,
|
||||
)
|
||||
|
||||
if prompt_id:
|
||||
for pp in all_images:
|
||||
await self._run_in_executor(
|
||||
self.db.link_image_to_prompt,
|
||||
prompt_id,
|
||||
pp,
|
||||
)
|
||||
imported += 1
|
||||
else:
|
||||
skipped += 1
|
||||
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Failed to import LoRA {model_name}: {e}")
|
||||
skipped += 1
|
||||
|
||||
# Progress update for every LoRA
|
||||
progress = int(5 + (90 * (i + 1) / max(total, 1)))
|
||||
img_count = len(all_images)
|
||||
status = f"{model_name}"
|
||||
if img_count:
|
||||
status += f" ({img_count} images)"
|
||||
if not existing:
|
||||
counts["imported"] += 1
|
||||
status = record["name"] if record else files[index].name
|
||||
if record and record["images"]:
|
||||
status += f" ({len(record['images'])} images)"
|
||||
await send_progress(
|
||||
{
|
||||
"type": "progress",
|
||||
"status": status,
|
||||
"progress": progress,
|
||||
"processed": i + 1,
|
||||
"imported": imported,
|
||||
"skipped": skipped,
|
||||
"progress": int(5 + (90 * (index + 1) / max(total, 1))),
|
||||
"processed": index + 1,
|
||||
"imported": counts["imported"],
|
||||
"skipped": index + 1 - counts["imported"],
|
||||
}
|
||||
)
|
||||
|
||||
# Each record's event is queued (through the loop) before staging
|
||||
# ends, so reporting until staging is done and then draining the
|
||||
# queue reports every record in order. I2: when the page goes away
|
||||
# (its connection closes, or a report cannot be sent) staging stops
|
||||
# before its next LoRA, and this returns only once it has stopped, so
|
||||
# the caller's records hold covers every download.
|
||||
stop = threading.Event()
|
||||
staging = asyncio.ensure_future(
|
||||
self.stage_lora_import(
|
||||
files,
|
||||
on_record=on_record,
|
||||
cancelled=lambda: stop.is_set() or disconnected(),
|
||||
)
|
||||
)
|
||||
try:
|
||||
while not staging.done():
|
||||
getter = asyncio.ensure_future(events.get())
|
||||
await asyncio.wait(
|
||||
{getter, staging}, return_when=asyncio.FIRST_COMPLETED
|
||||
)
|
||||
if getter.done():
|
||||
await report(*getter.result())
|
||||
else:
|
||||
getter.cancel()
|
||||
while not events.empty():
|
||||
await report(*events.get_nowait())
|
||||
finally:
|
||||
stop.set()
|
||||
# Staging must have stopped before the records hold is released,
|
||||
# even when the handler is cancelled again while it waits (R3).
|
||||
interrupted = False
|
||||
while not staging.done():
|
||||
try:
|
||||
await asyncio.shield(staging)
|
||||
except asyncio.CancelledError:
|
||||
interrupted = True
|
||||
except Exception:
|
||||
pass
|
||||
if interrupted:
|
||||
raise asyncio.CancelledError
|
||||
staged = await staging
|
||||
if not await self.lora_files_unchanged(lm_path, reviewed):
|
||||
raise LoraFilesChanged(_FILES_CHANGED)
|
||||
|
||||
await send_progress(
|
||||
{
|
||||
"type": "progress",
|
||||
"status": "Replacing previous lora-manager imports...",
|
||||
"progress": 96,
|
||||
}
|
||||
)
|
||||
result = await self.replace_lora_imports(
|
||||
self.db, staged, expected_ids=state["ids"], allow_empty=False
|
||||
)
|
||||
await send_progress(
|
||||
{
|
||||
"type": "complete",
|
||||
"progress": 100,
|
||||
"total": total,
|
||||
"imported": imported,
|
||||
"skipped": skipped,
|
||||
"imported": result["imported"],
|
||||
"skipped": total - result["imported"],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
+48
-171
@@ -37,7 +37,8 @@ runner runs); a refused start keeps it. The runner:
|
||||
|
||||
The job log names the backup by its file name only, and the job says "No
|
||||
safety backup" when none was taken. Apply errors: ``invalid-request`` (400),
|
||||
``plan-expired`` (410: unknown, expired or forgotten after a restart),
|
||||
``apply-failed`` (500: nothing started), ``plan-expired`` (410: unknown,
|
||||
expired or forgotten after a restart),
|
||||
``plan-used`` (409, with the job that applied it), ``storage-changed`` (409:
|
||||
a restore replaced the database since the review) and ``busy`` (409).
|
||||
"""
|
||||
@@ -47,8 +48,6 @@ import json
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import tempfile
|
||||
from contextlib import closing
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
@@ -58,15 +57,28 @@ from .jobs import MAINTENANCE_JOB_KIND, JobBusy
|
||||
from .maintenance_plans import PlanExpired, PlanUsed, ScopeTooLarge
|
||||
|
||||
try:
|
||||
from ...database import safety_backup as _backups
|
||||
from ...database.safety_backup import ( # noqa: F401 - re-exported
|
||||
SAFETY_TEMP_PREFIX,
|
||||
SafetyBackupFailed,
|
||||
sweep_safety_temps,
|
||||
)
|
||||
from ...database.workflow_storage import PRE_MIGRATION_LABEL, pre_migration_backups
|
||||
from ...database.health import readonly_uri
|
||||
from ...database.maintenance import MAINTENANCE_OPERATIONS, PlanChanged
|
||||
except ImportError:
|
||||
from database import safety_backup as _backups
|
||||
from database.safety_backup import ( # noqa: F401 - re-exported
|
||||
SAFETY_TEMP_PREFIX,
|
||||
SafetyBackupFailed,
|
||||
sweep_safety_temps,
|
||||
)
|
||||
from database.workflow_storage import PRE_MIGRATION_LABEL, pre_migration_backups
|
||||
from database.health import readonly_uri
|
||||
from database.maintenance import MAINTENANCE_OPERATIONS, PlanChanged
|
||||
|
||||
PREVIEW_BODY_MAX = 4096
|
||||
MAINTENANCE_PREVIEW_LEASE = "Maintenance preview"
|
||||
# Duplicate-file previews carry the reviewed keepers (py/api/duplicate_files.py).
|
||||
DUPLICATE_PREVIEW_BODY_MAX = 1024 * 1024
|
||||
DUPLICATE_FILES = "duplicate_files"
|
||||
@@ -95,18 +107,6 @@ PLAN_CHANGED = (
|
||||
APPLY_FAILED = "Maintenance failed; nothing was changed."
|
||||
NOTHING_TO_DO = "Every image record already stores its workflow once; nothing to do."
|
||||
NO_BACKUP = "No safety backup"
|
||||
_NAME_ATTEMPTS = 1000
|
||||
|
||||
|
||||
class SafetyBackupFailed(Exception):
|
||||
"""The safety backup could not be written, verified or put in place.
|
||||
|
||||
``reason`` is the verify reason when the copy did not verify.
|
||||
"""
|
||||
|
||||
def __init__(self, message, reason=None):
|
||||
super().__init__(message)
|
||||
self.reason = reason
|
||||
|
||||
|
||||
def backup_failed_message(error):
|
||||
@@ -117,10 +117,6 @@ def backup_failed_message(error):
|
||||
return BACKUP_FAILED
|
||||
|
||||
|
||||
def _now():
|
||||
return datetime.datetime.now()
|
||||
|
||||
|
||||
def _free_bytes(path):
|
||||
return shutil.disk_usage(path).free
|
||||
|
||||
@@ -243,29 +239,18 @@ def size_label(size):
|
||||
|
||||
|
||||
def restore_room(db_path):
|
||||
"""The largest restore upload the disks have room for (blocking).
|
||||
"""The largest restore upload the disk has room for (blocking).
|
||||
|
||||
The upload is written to a temp file and staged once more in the
|
||||
temporary folder (2x), then copied over the live database beside it,
|
||||
whose current contents are first kept as a safety backup (1x the upload
|
||||
for the WAL, plus the current database). Needs on one device add up,
|
||||
each with ``DISK_MARGIN_BYTES``.
|
||||
Everything a restore writes is in the library's folder (#318): the
|
||||
upload, its staged copy (``database/restore_stage.py``) and the WAL
|
||||
growth while the stage is copied over the live database (3x the upload),
|
||||
plus the safety backup of the current database (1x its size), with
|
||||
``DISK_MARGIN_BYTES`` to spare.
|
||||
"""
|
||||
db_path = os.path.abspath(db_path)
|
||||
folder = os.path.dirname(db_path)
|
||||
current = database_bytes(db_path) if os.path.exists(db_path) else 0
|
||||
devices = {}
|
||||
for path, per_upload, fixed in (
|
||||
(tempfile.gettempdir(), 2, 0),
|
||||
(folder, 1, current),
|
||||
):
|
||||
entry = devices.setdefault(_device(path), {"path": path, "per": 0, "fixed": 0})
|
||||
entry["per"] += per_upload
|
||||
entry["fixed"] += fixed
|
||||
room = min(
|
||||
(_free_bytes(e["path"]) - e["fixed"] - DISK_MARGIN_BYTES) // e["per"]
|
||||
for e in devices.values()
|
||||
)
|
||||
room = (_free_bytes(folder) - current - DISK_MARGIN_BYTES) // 3
|
||||
return max(0, room)
|
||||
|
||||
|
||||
@@ -305,142 +290,13 @@ def _changed_line(operation, count):
|
||||
return f"Removed {count:,} {noun}"
|
||||
|
||||
|
||||
def _remove_quietly(*paths):
|
||||
for path in paths:
|
||||
try:
|
||||
if path and os.path.exists(path):
|
||||
os.remove(path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _reserve_backup_name(db_path, label="backup"):
|
||||
"""Create and return an unused ``<db>.<label>_<stamp>[_n]`` (O_EXCL)."""
|
||||
for candidate in _backup_names(db_path, label):
|
||||
try:
|
||||
fd = os.open(candidate, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
|
||||
except FileExistsError:
|
||||
continue
|
||||
os.close(fd)
|
||||
return candidate
|
||||
raise SafetyBackupFailed("No free backup name")
|
||||
|
||||
|
||||
def _backup_names(db_path, label="backup"):
|
||||
base = f"{db_path}.{label}_{_now().strftime('%Y%m%d_%H%M%S_%f')}"
|
||||
for attempt in range(_NAME_ATTEMPTS):
|
||||
yield base if attempt == 0 else f"{base}_{attempt}"
|
||||
|
||||
|
||||
def _place_backup(temp, db_path, label="backup"):
|
||||
"""Give the verified ``temp`` its unique backup name; return that path.
|
||||
|
||||
``os.link`` is atomic and refuses an existing name, so no empty or half
|
||||
file ever carries a backup name, even across a crash. A filesystem
|
||||
without hard links falls back to reserving the name (``O_EXCL``) and
|
||||
``os.replace``. The temp file is removed by the caller.
|
||||
"""
|
||||
for candidate in _backup_names(db_path, label):
|
||||
try:
|
||||
os.link(temp, candidate)
|
||||
except FileExistsError:
|
||||
continue
|
||||
except (OSError, NotImplementedError):
|
||||
break # no hard links here
|
||||
else:
|
||||
return candidate
|
||||
else:
|
||||
raise SafetyBackupFailed("No free backup name")
|
||||
final = _reserve_backup_name(db_path, label)
|
||||
try:
|
||||
os.replace(temp, final)
|
||||
except OSError:
|
||||
_remove_quietly(final)
|
||||
raise
|
||||
return final
|
||||
|
||||
|
||||
SAFETY_TEMP_PREFIX = ".pm-safety-"
|
||||
|
||||
|
||||
def sweep_safety_temps(db_path):
|
||||
"""Remove the temp copies an interrupted safety backup left (blocking).
|
||||
|
||||
Called once at startup, before any job can take a backup. Returns the
|
||||
names removed.
|
||||
"""
|
||||
folder = os.path.dirname(os.path.abspath(db_path))
|
||||
removed = []
|
||||
try:
|
||||
names = os.listdir(folder)
|
||||
except OSError:
|
||||
return removed
|
||||
for name in names:
|
||||
if not (name.startswith(SAFETY_TEMP_PREFIX) and ".tmp" in name):
|
||||
continue
|
||||
try:
|
||||
os.remove(os.path.join(folder, name))
|
||||
removed.append(name)
|
||||
except OSError:
|
||||
continue
|
||||
return removed
|
||||
|
||||
|
||||
def _standalone(path):
|
||||
"""Make the copy one self-contained file (rollback journal, not WAL).
|
||||
|
||||
The backup API copies the live database's WAL flag; a WAL file opened
|
||||
read-only later (a verify, a restore preview) would leave ``-wal`` and
|
||||
``-shm`` sidecars beside it.
|
||||
"""
|
||||
try:
|
||||
with closing(sqlite3.connect(path)) as conn:
|
||||
conn.execute("PRAGMA journal_mode=DELETE").fetchone()
|
||||
except sqlite3.Error as exc:
|
||||
raise SafetyBackupFailed("The backup could not be finished") from exc
|
||||
|
||||
|
||||
def _fsync(path):
|
||||
with open(path, "rb+") as handle:
|
||||
os.fsync(handle.fileno())
|
||||
|
||||
|
||||
def take_safety_backup(model, label="backup"):
|
||||
"""Write and verify a safety backup; return its file name (blocking).
|
||||
|
||||
The copy is made in a temp file beside the database and only renamed
|
||||
onto its final, unique name ``<db>.<label>_<stamp>`` once it verified,
|
||||
so a backup file on disk is always a verified one. ``label`` "backup"
|
||||
joins the pruned pool; another label (the workflow storage migration's
|
||||
``pre-workflow-storage``) is kept until the user removes it. Raises
|
||||
SafetyBackupFailed, leaving no file.
|
||||
See ``database/safety_backup.py``: a backup file on disk is always a
|
||||
verified one. Raises SafetyBackupFailed, leaving no file.
|
||||
"""
|
||||
db_path = os.path.abspath(model.db_path)
|
||||
fd, temp = tempfile.mkstemp(
|
||||
dir=os.path.dirname(db_path), prefix=SAFETY_TEMP_PREFIX, suffix=".tmp"
|
||||
)
|
||||
os.close(fd)
|
||||
final = None
|
||||
try:
|
||||
if not model.backup_database(temp):
|
||||
raise SafetyBackupFailed("The backup could not be written")
|
||||
_standalone(temp)
|
||||
ok, reason = model.verify_database_file(temp)
|
||||
if not ok:
|
||||
raise SafetyBackupFailed("The backup did not verify", reason)
|
||||
_fsync(temp)
|
||||
if os.name != "nt":
|
||||
os.chmod(temp, 0o600)
|
||||
final = _place_backup(temp, db_path, label)
|
||||
return os.path.basename(final)
|
||||
except OSError as exc:
|
||||
_remove_quietly(final)
|
||||
raise SafetyBackupFailed("The backup could not be put in place") from exc
|
||||
except SafetyBackupFailed:
|
||||
_remove_quietly(final)
|
||||
raise
|
||||
finally:
|
||||
_remove_quietly(temp, temp + "-journal", temp + "-wal", temp + "-shm")
|
||||
return os.path.basename(_backups.take_safety_backup(model, label))
|
||||
|
||||
|
||||
def _failure(message, status, code=None, **extra):
|
||||
@@ -503,6 +359,19 @@ class MaintenanceRoutesMixin:
|
||||
return data, None
|
||||
|
||||
async def preview_maintenance(self, request):
|
||||
"""Review one operation under a registry lease (#249 S8).
|
||||
|
||||
The preview's read-only connections must not be open while a restore
|
||||
switches the database: it is refused (409 ``busy``) while an
|
||||
``exclusive`` job runs, and a restore waits for running previews.
|
||||
"""
|
||||
try:
|
||||
async with self._jobs.lease(MAINTENANCE_PREVIEW_LEASE):
|
||||
return await self._preview_maintenance(request)
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
|
||||
async def _preview_maintenance(self, request):
|
||||
data, error = await self._read_maintenance_body(
|
||||
request,
|
||||
("operation", "options"),
|
||||
@@ -644,7 +513,9 @@ class MaintenanceRoutesMixin:
|
||||
)
|
||||
except (OSError, sqlite3.Error):
|
||||
self.logger.exception("Could not measure the disk space for an apply")
|
||||
return _failure(APPLY_FAILED, 500)
|
||||
# Coded: the workspace tells the server's own refusal (nothing
|
||||
# started) from a proxy's 5xx (the job may have started).
|
||||
return _failure(APPLY_FAILED, 500, "apply-failed")
|
||||
if short is not None:
|
||||
# Refused before any job starts: nothing fails halfway for space.
|
||||
return _failure(short_space_message(short), 507, "insufficient-space")
|
||||
@@ -808,7 +679,13 @@ def backup_note(backup):
|
||||
|
||||
|
||||
def _fail_job(job, code, message):
|
||||
"""End ``job`` failed with ``message`` (one error row) and its code."""
|
||||
"""End ``job`` failed with ``message`` (one error row) and its code.
|
||||
|
||||
The code is also the job's ``scope["failure"]``, so Activity can offer
|
||||
Review again for a plan the library overtook.
|
||||
"""
|
||||
if code:
|
||||
job.scope["failure"] = code
|
||||
job.record("errors", {"path": "", "detail": message}, f"! {message}")
|
||||
job.update(status_line=message)
|
||||
job.publish({"type": "error", "code": code, "message": message})
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
"""Reviewed restores: staged uploads held under single-use tokens (#249 S8).
|
||||
|
||||
A restore preview stages the upload (``database/restore_stage.py``) and
|
||||
keeps it here under a ``secrets.token_urlsafe(32)`` token with the live
|
||||
library's fingerprint and the database identity it was reviewed against.
|
||||
``take`` hands the stage to the restore job exactly once; a second apply of
|
||||
the token names that job (``PlanUsed``). An expired, discarded or unknown
|
||||
token raises ``PlanExpired``; its stage folder is removed. A restart forgets
|
||||
every token (the startup sweep removes the folders).
|
||||
|
||||
At most ``max_stages`` stages are held. Bytes are bounded by free disk space
|
||||
(the upload is refused when the disk has no room, and held stages use that
|
||||
disk), not by a fixed budget. A stage being applied belongs to its job and is
|
||||
never evicted or counted.
|
||||
|
||||
The store is only read and changed on the event loop (it has no lock). Its
|
||||
entries own stage folders, whose removal is blocking: every method that
|
||||
drops an entry returns it (or keeps it in ``take_doomed``), and the caller
|
||||
removes the folder in the executor (``HeldRestore.staged.discard``).
|
||||
"""
|
||||
|
||||
import secrets
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
from .maintenance_plans import PlanExpired, PlanUsed
|
||||
|
||||
STAGE_TTL_SECONDS = 900
|
||||
MAX_STAGES = 2
|
||||
_USED_KEPT = 32 # applied tokens remembered for plan-used answers
|
||||
|
||||
|
||||
class TooManyStages(Exception):
|
||||
"""Every stage slot is held by a reviewed restore."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class HeldRestore:
|
||||
staged: Any # database.restore_stage.StagedRestore
|
||||
fingerprint: str
|
||||
db_identity: Any
|
||||
live: Dict[str, int]
|
||||
expires_at: float
|
||||
|
||||
|
||||
class RestorePlanStore:
|
||||
def __init__(
|
||||
self,
|
||||
clock: Optional[Callable[[], float]] = None,
|
||||
ttl: float = STAGE_TTL_SECONDS,
|
||||
max_stages: int = MAX_STAGES,
|
||||
):
|
||||
self._ttl = ttl
|
||||
self._max = max_stages
|
||||
self._clock = clock or time.time
|
||||
self._held: Dict[str, HeldRestore] = {}
|
||||
self._used: Dict[str, str] = {}
|
||||
self._doomed: List[HeldRestore] = []
|
||||
|
||||
@property
|
||||
def ttl(self) -> float:
|
||||
return self._ttl
|
||||
|
||||
def now(self) -> float:
|
||||
return self._clock()
|
||||
|
||||
def __len__(self):
|
||||
return len(self._held)
|
||||
|
||||
def full(self) -> bool:
|
||||
return len(self._held) >= self._max
|
||||
|
||||
def issue(self, staged, fingerprint, db_identity, live) -> str:
|
||||
"""Hold ``staged`` and return its token; TooManyStages when full."""
|
||||
if self.full():
|
||||
raise TooManyStages()
|
||||
token = secrets.token_urlsafe(32)
|
||||
self._held[token] = HeldRestore(
|
||||
staged=staged,
|
||||
fingerprint=fingerprint,
|
||||
db_identity=db_identity,
|
||||
live=dict(live),
|
||||
expires_at=self._clock() + self._ttl,
|
||||
)
|
||||
return token
|
||||
|
||||
def peek(self, token) -> HeldRestore:
|
||||
if token in self._used:
|
||||
raise PlanUsed(self._used[token])
|
||||
held = self._held.get(token)
|
||||
if held is None:
|
||||
raise PlanExpired()
|
||||
if held.expires_at <= self._clock():
|
||||
self._doomed.append(self._held.pop(token))
|
||||
raise PlanExpired()
|
||||
return held
|
||||
|
||||
def take(self, token, job_id) -> HeldRestore:
|
||||
"""Hand the held stage to the job applying it (single use)."""
|
||||
held = self.peek(token)
|
||||
del self._held[token]
|
||||
self._used[token] = job_id
|
||||
while len(self._used) > _USED_KEPT:
|
||||
self._used.pop(next(iter(self._used)))
|
||||
return held
|
||||
|
||||
def give_back(self, token, held) -> bool:
|
||||
"""Hold a taken stage again under its token (its job did not apply it).
|
||||
|
||||
False, holding nothing, when the review expired meanwhile or its slot
|
||||
was reused (the store is full): the caller then discards the stage.
|
||||
"""
|
||||
if held.expires_at <= self._clock() or self.full():
|
||||
return False
|
||||
self._used.pop(token, None)
|
||||
self._held[token] = held
|
||||
return True
|
||||
|
||||
def entry(self, token) -> Optional[HeldRestore]:
|
||||
"""The held entry, expired or not (for the response that issued it)."""
|
||||
return self._held.get(token)
|
||||
|
||||
def pop(self, token) -> Optional[HeldRestore]:
|
||||
"""Drop the token; its entry (to discard) or None when it was not held."""
|
||||
return self._held.pop(token, None)
|
||||
|
||||
def pop_expired(self) -> List[HeldRestore]:
|
||||
"""Drop every expired entry; them, with any ``peek`` found expired."""
|
||||
now = self._clock()
|
||||
expired = [t for t, held in self._held.items() if held.expires_at <= now]
|
||||
return [self._held.pop(token) for token in expired] + self.take_doomed()
|
||||
|
||||
def pop_all(self) -> List[HeldRestore]:
|
||||
"""Drop every entry (the library was replaced); them, to discard."""
|
||||
held = list(self._held.values()) + self.take_doomed()
|
||||
self._held.clear()
|
||||
return held
|
||||
|
||||
def take_doomed(self) -> List[HeldRestore]:
|
||||
doomed, self._doomed = self._doomed, []
|
||||
return doomed
|
||||
|
||||
def next_expiry(self) -> Optional[float]:
|
||||
"""When the soonest held entry expires (clock time), or None."""
|
||||
return min((held.expires_at for held in self._held.values()), default=None)
|
||||
+19
-1
@@ -31,6 +31,8 @@ from aiohttp import web
|
||||
from .annotations import _HEX64, _TOKEN, _UUID, _capture_error, _error
|
||||
from .images import media_thumbnail_url
|
||||
from .inspection_service import VIDEO_EXTENSIONS
|
||||
from .job_routes import busy_response
|
||||
from .jobs import JobBusy
|
||||
from .prompts import bad_request
|
||||
|
||||
try:
|
||||
@@ -239,6 +241,9 @@ def _side_label(titles: dict, side, empty: str) -> str:
|
||||
return display_title(titles.get(stored, ()), text) or empty
|
||||
|
||||
|
||||
REVIEW_DECISION_LEASE = "Needs review decision"
|
||||
|
||||
|
||||
class ReviewRoutesMixin:
|
||||
"""The Needs review queue over E1's schema and E2's recorded items."""
|
||||
|
||||
@@ -446,7 +451,20 @@ class ReviewRoutesMixin:
|
||||
# -- decision --------------------------------------------------------------
|
||||
|
||||
async def decide_review_item(self, request: web.Request) -> web.Response:
|
||||
"""POST one explicit decision on one pending item."""
|
||||
"""POST one explicit decision on one pending item.
|
||||
|
||||
The decision runs under a registry lease (#249 S8): it is refused
|
||||
(409 ``busy``) while a restore or another ``exclusive`` job runs, and
|
||||
a restore waits for decisions already running before it switches the
|
||||
database. Decisions never refuse one another or a scan.
|
||||
"""
|
||||
try:
|
||||
async with self._jobs.lease(REVIEW_DECISION_LEASE):
|
||||
return await self._decide_review_item(request)
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
|
||||
async def _decide_review_item(self, request: web.Request) -> web.Response:
|
||||
data, error = await self._read_json_body(request, max_bytes=MAX_BODY_BYTES)
|
||||
if error is not None:
|
||||
return error
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
"""The staged LoRA reference import (#221, #249 System S10).
|
||||
|
||||
The import reads LoraManager's ``*.metadata.json`` files and turns each LoRA
|
||||
into one prompt of the ``lora-manager`` category, tagged with its name and
|
||||
trigger words and linked to its preview images. It runs in three stages, and
|
||||
only the last writes to the database:
|
||||
|
||||
1. **discover**: list the metadata files (``discover``) and fingerprint them
|
||||
(path, size and modification time; ``fingerprint``). ``preview`` adds the
|
||||
counts a review shows. Nothing is written and nothing is requested.
|
||||
2. **stage**: read each file and build its record; CivitAI example images are
|
||||
downloaded into the image cache first (best effort, counted). A cancelled
|
||||
or failed stage leaves the database untouched; downloaded images stay in
|
||||
the cache.
|
||||
3. **replace**: the files are listed and fingerprinted once more (a file
|
||||
changed, removed or added while staging aborts the import), then
|
||||
``PromptDatabase.replace_category_prompts`` swaps the old collection for
|
||||
the staged records in one transaction (or changes nothing).
|
||||
|
||||
Both the System job (``POST /lora/import/apply``) and the legacy ``POST
|
||||
/lora/scan`` run these stages (``py/api/lora_integration.py``).
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Callable, Dict, List, Optional
|
||||
|
||||
from . import lora_utils
|
||||
|
||||
try:
|
||||
from ..utils.hashing import generate_prompt_hash
|
||||
except ImportError: # pragma: no cover - flat test imports
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
LORA_CATEGORY = "lora-manager"
|
||||
JOB_TITLE = "LoRA reference import"
|
||||
# The job's outcome groups, in its counter-tile order.
|
||||
GROUPS = ("imported", "linked", "skipped", "downloaded", "download_errors", "errors")
|
||||
METADATA_SUFFIX = "*.metadata.json"
|
||||
|
||||
|
||||
class ImportCancelled(Exception):
|
||||
"""The import was cancelled while staging; nothing was written."""
|
||||
|
||||
|
||||
def discover(lora_manager_path: str) -> List[Path]:
|
||||
"""Every LoRA metadata file, sorted and without repeats.
|
||||
|
||||
Raises whatever listing a folder raised: a discovery that cannot see
|
||||
every folder must not replace the collection with part of it.
|
||||
"""
|
||||
seen = set()
|
||||
files = []
|
||||
for folder in lora_utils.find_lora_directories(lora_manager_path):
|
||||
for path in Path(folder).rglob(METADATA_SUFFIX):
|
||||
key = str(path)
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
files.append(path)
|
||||
files.sort(key=str)
|
||||
return files
|
||||
|
||||
|
||||
def fingerprint(files: List[Path]) -> str:
|
||||
"""What a review saw of ``files``: each path, size and modification time."""
|
||||
digest = hashlib.sha256()
|
||||
for path in files:
|
||||
try:
|
||||
stat = path.stat()
|
||||
entry = f"{path}\0{stat.st_size}\0{stat.st_mtime_ns}\n"
|
||||
except OSError:
|
||||
entry = f"{path}\0missing\n"
|
||||
digest.update(entry.encode("utf-8", "surrogatepass"))
|
||||
return "sha256:" + digest.hexdigest()
|
||||
|
||||
|
||||
def preview(lora_manager_path: str, cache_dir: Path) -> Dict:
|
||||
"""The review's counts (blocking; reads only, requests nothing).
|
||||
|
||||
``{lora_files, with_metadata, planned_downloads, fingerprint, names}``:
|
||||
the metadata files found, those that parse, the example images an import
|
||||
would download (not yet in the cache) and the first LoRA names.
|
||||
"""
|
||||
files = discover(lora_manager_path)
|
||||
with_metadata = 0
|
||||
planned = 0
|
||||
names = []
|
||||
for path in files:
|
||||
metadata = lora_utils.read_lora_metadata(path)
|
||||
if not metadata:
|
||||
continue
|
||||
with_metadata += 1
|
||||
planned += lora_utils.planned_civitai_downloads(metadata, path, cache_dir)
|
||||
if len(names) < 50:
|
||||
names.append(lora_utils.get_model_name_from_metadata(metadata))
|
||||
return {
|
||||
"lora_files": len(files),
|
||||
"with_metadata": with_metadata,
|
||||
"planned_downloads": planned,
|
||||
"fingerprint": fingerprint(files),
|
||||
"names": names,
|
||||
}
|
||||
|
||||
|
||||
def build_record(metadata: Dict, path: Path, images: List[str]) -> Dict:
|
||||
"""The prompt a LoRA becomes: its example prompt (else its name), tags, images."""
|
||||
model_name = lora_utils.get_model_name_from_metadata(metadata)
|
||||
text = lora_utils.get_example_prompt_from_metadata(metadata) or model_name
|
||||
tags = ["lora-manager", f"lora:{model_name}"]
|
||||
tags.extend(lora_utils.get_trigger_words_from_metadata(metadata))
|
||||
return {
|
||||
"name": model_name,
|
||||
"text": text,
|
||||
"hash": generate_prompt_hash(text),
|
||||
"tags": list(dict.fromkeys(tags)),
|
||||
"images": images,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class StagedImport:
|
||||
"""The records an import will write, and what staging counted."""
|
||||
|
||||
total: int
|
||||
records: List[Dict] = field(default_factory=list)
|
||||
skipped: List[str] = field(default_factory=list)
|
||||
stats: Dict[str, int] = field(
|
||||
default_factory=lambda: {"downloaded": 0, "download_errors": 0}
|
||||
)
|
||||
|
||||
|
||||
def stage(
|
||||
files: List[Path],
|
||||
cache_dir: Path,
|
||||
api_key: str = "",
|
||||
*,
|
||||
on_record: Optional[Callable[[int, Dict, "StagedImport"], None]] = None,
|
||||
cancelled: Callable[[], bool] = lambda: False,
|
||||
logger=None,
|
||||
) -> StagedImport:
|
||||
"""Read each file and build its record, downloading its example images.
|
||||
|
||||
Blocking. A file that cannot be read, or whose record fails to build, is
|
||||
skipped by its name. ``cancelled()`` is asked before each file (raises
|
||||
ImportCancelled); ``on_record(index, record_or_None, staged)`` follows
|
||||
each one.
|
||||
"""
|
||||
staged = StagedImport(total=len(files))
|
||||
for index, path in enumerate(files):
|
||||
if cancelled():
|
||||
raise ImportCancelled()
|
||||
record = None
|
||||
metadata = lora_utils.read_lora_metadata(path)
|
||||
if not metadata:
|
||||
staged.skipped.append(path.name)
|
||||
else:
|
||||
try:
|
||||
local = lora_utils.get_preview_images_from_metadata(metadata, path)
|
||||
remote = lora_utils.download_civitai_images(
|
||||
metadata, path, cache_dir, api_key, stats=staged.stats
|
||||
)
|
||||
images = list(dict.fromkeys([*local, *remote]))
|
||||
record = build_record(metadata, path, images)
|
||||
staged.records.append(record)
|
||||
except Exception as error: # one LoRA never stops the import
|
||||
name = lora_utils.get_model_name_from_metadata(metadata)
|
||||
if logger is not None:
|
||||
logger.warning(f"Failed to prepare LoRA {name}: {error}")
|
||||
staged.skipped.append(name)
|
||||
if on_record is not None:
|
||||
on_record(index, record, staged)
|
||||
return staged
|
||||
+315
-64
@@ -8,14 +8,16 @@ return empty results rather than raising.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from urllib.parse import urlsplit
|
||||
from urllib.parse import urljoin, urlsplit
|
||||
|
||||
try:
|
||||
from ..utils.logging_config import get_logger
|
||||
@@ -82,23 +84,46 @@ def detect_lora_manager(custom_path: str = "") -> Optional[str]:
|
||||
if p.is_dir() and _looks_like_lora_manager(p):
|
||||
return str(p.resolve())
|
||||
|
||||
# 2. Auto-detect via custom_nodes (case-insensitive scan)
|
||||
# 2. Auto-detect in every custom_nodes folder (case-insensitive scan):
|
||||
# ComfyUI's own first, then the extra ones it registered (#144).
|
||||
folders = []
|
||||
root = find_comfyui_root()
|
||||
if root:
|
||||
custom_nodes = root / "custom_nodes"
|
||||
if custom_nodes.is_dir():
|
||||
for entry in custom_nodes.iterdir():
|
||||
if (
|
||||
entry.is_dir()
|
||||
and "lora" in entry.name.lower()
|
||||
and "manager" in entry.name.lower()
|
||||
and _looks_like_lora_manager(entry)
|
||||
):
|
||||
return str(entry.resolve())
|
||||
folders.append(root / "custom_nodes")
|
||||
folders.extend(custom_nodes_directories())
|
||||
seen = set()
|
||||
for custom_nodes in folders:
|
||||
key = os.path.normcase(os.path.abspath(str(custom_nodes)))
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
found = _lora_manager_in(Path(custom_nodes))
|
||||
if found:
|
||||
return found
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _lora_manager_in(custom_nodes: Path) -> Optional[str]:
|
||||
"""The LoraManager install directly under ``custom_nodes``, or None."""
|
||||
try:
|
||||
entries = sorted(custom_nodes.iterdir()) if custom_nodes.is_dir() else []
|
||||
except OSError:
|
||||
return None
|
||||
for entry in entries:
|
||||
try:
|
||||
if (
|
||||
entry.is_dir()
|
||||
and "lora" in entry.name.lower()
|
||||
and "manager" in entry.name.lower()
|
||||
and _looks_like_lora_manager(entry)
|
||||
):
|
||||
return str(entry.resolve())
|
||||
except OSError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _looks_like_lora_manager(path: Path) -> bool:
|
||||
"""Heuristic: does this directory look like a LoraManager install?"""
|
||||
# Must have __init__.py (ComfyUI extension) or README.md
|
||||
@@ -396,19 +421,40 @@ def get_preview_image_from_metadata(
|
||||
|
||||
_THUMB_MAX_SIZE = 512
|
||||
|
||||
# CivitAI's own domains (#340: civitai.red and civitai.green serve the same
|
||||
# content as civitai.com).
|
||||
CIVITAI_DOMAINS = ("civitai.com", "civitai.red", "civitai.green")
|
||||
# Only these hosts may ever receive the CivitAI API key, and only over HTTPS.
|
||||
CIVITAI_HOSTS = frozenset(
|
||||
{"civitai.com", "www.civitai.com", "api.civitai.com", "image.civitai.com"}
|
||||
f"{prefix}{domain}"
|
||||
for domain in CIVITAI_DOMAINS
|
||||
for prefix in ("", "www.", "api.", "image.")
|
||||
)
|
||||
MAX_CIVITAI_DOWNLOAD_BYTES = 50 * 1024 * 1024
|
||||
_DOWNLOAD_CHUNK_BYTES = 64 * 1024
|
||||
|
||||
|
||||
# A redirect is followed (#337: image.civitai.com now answers with a redirect
|
||||
# to blobs-b2.civitai.com) only over HTTPS to one of CIVITAI_DOMAINS or a
|
||||
# subdomain of one, at most this many hops, and never with the API key.
|
||||
MAX_CIVITAI_REDIRECTS = 3
|
||||
_REDIRECT_CODES = frozenset({301, 302, 303, 307, 308})
|
||||
_USER_AGENT = "ComfyUI-PromptManager/1.0"
|
||||
|
||||
|
||||
class _DownloadFailed(Exception):
|
||||
"""A download failed for the short, loggable ``reason`` it carries."""
|
||||
|
||||
def __init__(self, reason: str):
|
||||
super().__init__(reason)
|
||||
self.reason = reason
|
||||
|
||||
|
||||
class _RefuseRedirects(urllib.request.HTTPRedirectHandler):
|
||||
"""Never follow a redirect: urllib would re-send the Bearer header to it."""
|
||||
"""Never follow a redirect inside urllib: it would re-send the Bearer
|
||||
header. A 3xx surfaces as ``HTTPError``; ``_download_one`` decides."""
|
||||
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
logger.warning(f"Refusing redirect from {req.full_url} to {newurl}")
|
||||
return None
|
||||
|
||||
|
||||
@@ -425,6 +471,84 @@ def _open_url(req: urllib.request.Request, timeout: float = 10):
|
||||
return _opener.open(req, timeout=timeout)
|
||||
|
||||
|
||||
def _loggable(url: str) -> str:
|
||||
"""``url`` without its query, fragment or credentials (signed URLs)."""
|
||||
try:
|
||||
parts = urlsplit(url)
|
||||
host = parts.hostname or ""
|
||||
except ValueError:
|
||||
return "<unparsable URL>"
|
||||
return f"{parts.scheme}://{host}{parts.path}"
|
||||
|
||||
|
||||
def _on_civitai_domain(hostname: str) -> bool:
|
||||
"""True when ``hostname`` is one of CIVITAI_DOMAINS or a subdomain of one."""
|
||||
return any(
|
||||
hostname == domain or hostname.endswith(f".{domain}")
|
||||
for domain in CIVITAI_DOMAINS
|
||||
)
|
||||
|
||||
|
||||
def is_civitai_redirect_target(url: str) -> bool:
|
||||
"""True when a redirect to ``url`` may be followed: HTTPS on one of
|
||||
CIVITAI_DOMAINS or a subdomain of one, matched on the parsed host name,
|
||||
default port and no credentials in the URL."""
|
||||
try:
|
||||
parts = urlsplit(url)
|
||||
port = parts.port
|
||||
except ValueError:
|
||||
return False
|
||||
hostname = (parts.hostname or "").lower()
|
||||
return (
|
||||
parts.scheme == "https"
|
||||
and _on_civitai_domain(hostname)
|
||||
and parts.username is None
|
||||
and parts.password is None
|
||||
and port in (None, 443)
|
||||
)
|
||||
|
||||
|
||||
def _redirect_target(url: str, error) -> Optional[str]:
|
||||
"""The URL a 3xx ``error`` for ``url`` points to, or None when absent."""
|
||||
if error.code not in _REDIRECT_CODES:
|
||||
return None
|
||||
location = (error.headers.get("Location") or "").strip()
|
||||
if not location:
|
||||
return None
|
||||
return urljoin(url, location)
|
||||
|
||||
|
||||
def _open_following_civitai_redirects(url: str, api_key: str):
|
||||
"""Open ``url``, following only allowed CivitAI redirects (keyless).
|
||||
|
||||
The first request carries the key; every redirected one carries the
|
||||
User-Agent alone. Returns the open response; raises ``_DownloadFailed``
|
||||
when a redirect is refused or the hop limit is reached. Other HTTP
|
||||
errors propagate.
|
||||
"""
|
||||
headers = {"User-Agent": _USER_AGENT}
|
||||
if api_key:
|
||||
headers["Authorization"] = f"Bearer {api_key}"
|
||||
current = url
|
||||
for hop in range(MAX_CIVITAI_REDIRECTS + 1):
|
||||
req = urllib.request.Request(current, headers=headers)
|
||||
try:
|
||||
return _open_url(req, timeout=10)
|
||||
except urllib.error.HTTPError as error:
|
||||
with error:
|
||||
target = _redirect_target(current, error)
|
||||
if target is None:
|
||||
raise
|
||||
if hop == MAX_CIVITAI_REDIRECTS:
|
||||
raise _DownloadFailed(f"more than {MAX_CIVITAI_REDIRECTS} redirects")
|
||||
if not is_civitai_redirect_target(target):
|
||||
raise _DownloadFailed(f"redirect refused to {_loggable(target)}")
|
||||
logger.debug(f"Following redirect to {_loggable(target)}")
|
||||
current = target
|
||||
headers = {"User-Agent": _USER_AGENT} # never the key across a redirect
|
||||
raise AssertionError("unreachable") # pragma: no cover - loop ends above
|
||||
|
||||
|
||||
def is_civitai_url(url: str) -> bool:
|
||||
"""True when ``url`` is an HTTPS URL on an allow-listed CivitAI host."""
|
||||
try:
|
||||
@@ -465,73 +589,160 @@ def _read_capped(resp, cap: int) -> Optional[bytes]:
|
||||
return b"".join(chunks)
|
||||
|
||||
|
||||
def _network_reason(error: BaseException) -> str:
|
||||
"""A short reason for a failed request: a timeout or a network error."""
|
||||
cause = getattr(error, "reason", error) # URLError wraps the socket error
|
||||
if isinstance(cause, TimeoutError):
|
||||
return "timed out"
|
||||
return f"network error: {cause}"
|
||||
|
||||
|
||||
def _fetch_civitai_body(url: str, api_key: str) -> bytes:
|
||||
"""The body behind ``url`` (redirects as allowed, size capped), or
|
||||
``_DownloadFailed`` with the reason it could not be fetched."""
|
||||
try:
|
||||
with _open_following_civitai_redirects(url, api_key) as resp:
|
||||
raw = _read_capped(resp, MAX_CIVITAI_DOWNLOAD_BYTES)
|
||||
except urllib.error.HTTPError as error:
|
||||
error.close()
|
||||
raise _DownloadFailed(f"HTTP {error.code}") from None
|
||||
except (urllib.error.URLError, OSError, http.client.HTTPException) as error:
|
||||
raise _DownloadFailed(_network_reason(error)) from None
|
||||
if raw is None:
|
||||
cap = MAX_CIVITAI_DOWNLOAD_BYTES
|
||||
size = f"{cap // (1024 * 1024)} MB" if cap >= 1024 * 1024 else f"{cap} byte"
|
||||
raise _DownloadFailed(f"larger than the {size} cap")
|
||||
return raw
|
||||
|
||||
|
||||
def _thumbnail(raw: bytes):
|
||||
"""``raw`` decoded and shrunk to an RGB thumbnail, or ``_DownloadFailed``."""
|
||||
from io import BytesIO
|
||||
|
||||
from PIL import Image
|
||||
|
||||
try:
|
||||
with Image.open(BytesIO(raw)) as img:
|
||||
img.thumbnail((_THUMB_MAX_SIZE, _THUMB_MAX_SIZE), Image.LANCZOS)
|
||||
return img.convert("RGB")
|
||||
except Exception as error: # PIL raises several types for bad data
|
||||
raise _DownloadFailed(
|
||||
f"image could not be decoded ({type(error).__name__})"
|
||||
) from None
|
||||
|
||||
|
||||
def _download_one(url: str, local_path: Path, api_key: str) -> Optional[str]:
|
||||
"""Download a single CivitAI image, resize to thumbnail, save as JPEG.
|
||||
|
||||
Refuses any URL that is not HTTPS on an allow-listed CivitAI host so the
|
||||
API key is never sent elsewhere, and refuses bodies larger than
|
||||
MAX_CIVITAI_DOWNLOAD_BYTES.
|
||||
API key is never sent elsewhere, follows a redirect only as
|
||||
``_open_following_civitai_redirects`` allows (without the key), and
|
||||
refuses bodies larger than MAX_CIVITAI_DOWNLOAD_BYTES. A failure logs
|
||||
one warning with a short reason and the URL without query or
|
||||
credentials (#340), and returns None.
|
||||
"""
|
||||
if not is_civitai_url(url):
|
||||
logger.warning(f"Refusing non-CivitAI download URL: {url}")
|
||||
return None
|
||||
try:
|
||||
headers = {"User-Agent": "ComfyUI-PromptManager/1.0"}
|
||||
if api_key:
|
||||
headers["Authorization"] = f"Bearer {api_key}"
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
with _open_url(req, timeout=10) as resp:
|
||||
raw = _read_capped(resp, MAX_CIVITAI_DOWNLOAD_BYTES)
|
||||
if raw is None:
|
||||
logger.warning(f"Refusing oversized download: {url}")
|
||||
return None
|
||||
|
||||
# Resize to thumbnail to save disk space
|
||||
from io import BytesIO
|
||||
|
||||
from PIL import Image
|
||||
|
||||
img = Image.open(BytesIO(raw))
|
||||
img.thumbnail((_THUMB_MAX_SIZE, _THUMB_MAX_SIZE), Image.LANCZOS)
|
||||
img = img.convert("RGB")
|
||||
img.save(str(local_path), "JPEG", quality=85)
|
||||
|
||||
if not is_civitai_url(url):
|
||||
raise _DownloadFailed("not an allowed CivitAI URL")
|
||||
image = _thumbnail(_fetch_civitai_body(url, api_key))
|
||||
try:
|
||||
_save_atomically(image, local_path)
|
||||
except OSError as error:
|
||||
raise _DownloadFailed(
|
||||
f"image could not be saved ({error.strerror or type(error).__name__})"
|
||||
) from None
|
||||
except _DownloadFailed as failure:
|
||||
reason = failure.reason
|
||||
except Exception as error: # best effort: an import never fails on one image
|
||||
reason = f"unexpected {type(error).__name__}"
|
||||
else:
|
||||
return str(local_path.resolve())
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to download {url}: {e}")
|
||||
return None
|
||||
logger.warning(f"CivitAI image download failed: {_loggable(url)}: {reason}")
|
||||
return None
|
||||
|
||||
|
||||
def download_civitai_images(
|
||||
metadata: Dict, metadata_path: Path, cache_dir: Path, api_key: str = ""
|
||||
) -> List[str]:
|
||||
"""Download civitai example images to a local cache directory.
|
||||
_PART_PREFIX = ".pm-"
|
||||
_PART_SUFFIX = ".part"
|
||||
# A temp download older than this was left by a killed process, not a live one
|
||||
# (each download has a 10 s timeout).
|
||||
STALE_PART_SECONDS = 3600
|
||||
|
||||
Uses thumbnail URLs (512px) instead of full-size originals, and
|
||||
downloads in parallel (up to 8 concurrent) for speed.
|
||||
|
||||
Args:
|
||||
api_key: CivitAI API key for authenticated downloads (NSFW content).
|
||||
def _save_atomically(image, local_path: Path) -> None:
|
||||
"""Write ``image`` as JPEG so ``local_path`` only ever names a whole file.
|
||||
|
||||
Returns:
|
||||
List of absolute paths to downloaded image files.
|
||||
The image goes to a ``.pm-*.part`` file in the same folder, which is
|
||||
closed before ``os.replace`` puts it in place (Windows cannot replace an
|
||||
open file); a failure removes the temp file. A reader (another import's
|
||||
cache check) never sees a half-written image. The temp file is opened
|
||||
like ``img.save`` opens a file, so the image keeps the umask's mode.
|
||||
"""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import secrets
|
||||
|
||||
while True:
|
||||
temp = local_path.parent / (
|
||||
f"{_PART_PREFIX}{secrets.token_hex(8)}{_PART_SUFFIX}"
|
||||
)
|
||||
try:
|
||||
handle = open(temp, "xb")
|
||||
except FileExistsError:
|
||||
continue
|
||||
break
|
||||
try:
|
||||
with handle:
|
||||
image.save(handle, "JPEG", quality=85)
|
||||
os.replace(temp, local_path)
|
||||
except BaseException:
|
||||
try:
|
||||
os.remove(temp)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def sweep_stale_downloads(cache_dir: Path) -> None:
|
||||
"""Remove ``.pm-*.part`` files a killed download left in the image cache.
|
||||
|
||||
Only that exact pattern, only in ``cache_dir`` and its LoRA folders, and
|
||||
only once older than ``STALE_PART_SECONDS``; a file that cannot be read
|
||||
or removed is left for a later import.
|
||||
"""
|
||||
import time
|
||||
|
||||
cutoff = time.time() - STALE_PART_SECONDS
|
||||
try:
|
||||
folders = [cache_dir] + [p for p in cache_dir.iterdir() if p.is_dir()]
|
||||
except OSError:
|
||||
return
|
||||
for folder in folders:
|
||||
try:
|
||||
candidates = list(folder.glob(f"{_PART_PREFIX}*{_PART_SUFFIX}"))
|
||||
except OSError:
|
||||
continue
|
||||
for path in candidates:
|
||||
try:
|
||||
if path.is_file() and path.stat().st_mtime < cutoff:
|
||||
os.remove(path)
|
||||
except OSError:
|
||||
logger.debug(f"Could not remove stale download {path}")
|
||||
|
||||
|
||||
def _civitai_cache_plan(
|
||||
metadata: Dict, metadata_path: Path, cache_dir: Path
|
||||
) -> Tuple[Path, List[str], List[Tuple[str, Path]]]:
|
||||
"""``(lora_cache, cached, tasks)`` for a LoRA's CivitAI example images.
|
||||
|
||||
``cached`` are the images already in the cache, ``tasks`` the
|
||||
``(url, local_path)`` downloads still needed. Reads only; creates nothing.
|
||||
"""
|
||||
civitai = _get_civitai(metadata)
|
||||
images = civitai.get("images", []) or []
|
||||
if not images:
|
||||
return []
|
||||
|
||||
file_name = metadata.get("file_name", "")
|
||||
if not file_name:
|
||||
file_name = metadata_path.name.replace(".metadata.json", "")
|
||||
lora_stem = Path(file_name).stem
|
||||
lora_cache = cache_dir / Path(file_name).stem
|
||||
|
||||
lora_cache = cache_dir / lora_stem
|
||||
lora_cache.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Build download tasks
|
||||
cached = []
|
||||
tasks = [] # (url, local_path)
|
||||
for img in images:
|
||||
@@ -548,9 +759,42 @@ def download_civitai_images(
|
||||
cached.append(str(local_path.resolve()))
|
||||
else:
|
||||
tasks.append((url, local_path))
|
||||
return lora_cache, cached, tasks
|
||||
|
||||
|
||||
def planned_civitai_downloads(
|
||||
metadata: Dict, metadata_path: Path, cache_dir: Path
|
||||
) -> int:
|
||||
"""How many example images an import would download (no request, no write)."""
|
||||
return len(_civitai_cache_plan(metadata, metadata_path, cache_dir)[2])
|
||||
|
||||
|
||||
def download_civitai_images(
|
||||
metadata: Dict,
|
||||
metadata_path: Path,
|
||||
cache_dir: Path,
|
||||
api_key: str = "",
|
||||
stats: Optional[Dict[str, int]] = None,
|
||||
) -> List[str]:
|
||||
"""Download civitai example images to a local cache directory.
|
||||
|
||||
Uses thumbnail URLs (512px) instead of full-size originals, and
|
||||
downloads in parallel (up to 8 concurrent) for speed.
|
||||
|
||||
Args:
|
||||
api_key: CivitAI API key for authenticated downloads (NSFW content).
|
||||
stats: When given, ``downloaded`` and ``download_errors`` are
|
||||
increased by the downloads that succeeded and failed.
|
||||
|
||||
Returns:
|
||||
List of absolute paths to downloaded image files.
|
||||
"""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
lora_cache, cached, tasks = _civitai_cache_plan(metadata, metadata_path, cache_dir)
|
||||
if not tasks:
|
||||
return cached
|
||||
lora_cache.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Download in parallel
|
||||
downloaded = []
|
||||
@@ -563,15 +807,21 @@ def download_civitai_images(
|
||||
if result:
|
||||
downloaded.append(result)
|
||||
|
||||
if stats is not None:
|
||||
stats["downloaded"] = stats.get("downloaded", 0) + len(downloaded)
|
||||
stats["download_errors"] = (
|
||||
stats.get("download_errors", 0) + len(tasks) - len(downloaded)
|
||||
)
|
||||
return cached + downloaded
|
||||
|
||||
|
||||
def get_lora_image_cache_dir() -> Path:
|
||||
def get_lora_image_cache_dir(create: bool = True) -> Path:
|
||||
"""Get the directory used to cache downloaded LoRA example images.
|
||||
|
||||
``<data folder>/data/lora_images`` (``utils.library_location``): the node
|
||||
folder for a library that was not moved, else ComfyUI's
|
||||
``user/__promptmanager``.
|
||||
``user/__promptmanager``. ``create=False`` only names it (a preview
|
||||
writes nothing).
|
||||
"""
|
||||
try:
|
||||
from ..utils import library_location
|
||||
@@ -582,7 +832,8 @@ def get_lora_image_cache_dir() -> Path:
|
||||
cache = Path(
|
||||
library_location.cache_dir("lora_images", extension_root=str(ext_root))
|
||||
)
|
||||
cache.mkdir(parents=True, exist_ok=True)
|
||||
if create:
|
||||
cache.mkdir(parents=True, exist_ok=True)
|
||||
return cache
|
||||
|
||||
|
||||
|
||||
+1
-1
@@ -7,7 +7,7 @@ requires-python = ">=3.10"
|
||||
dependencies = ["watchdog>=2.1.0", "Pillow>=8.0.0"]
|
||||
|
||||
[build-system]
|
||||
requires = ["setuptools>=84.0.0"]
|
||||
requires = ["setuptools>=84.2.1"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project.urls]
|
||||
|
||||
@@ -149,3 +149,49 @@ test('Classic test button reports success and failure and releases expired notif
|
||||
await page.clock.fastForward(3001);
|
||||
await expect(page.locator('.pm-classic-notification')).toHaveCount(0);
|
||||
});
|
||||
|
||||
test('Classic shows linked text, keeps the typed text through save and refuses to overwrite it (#343)',async({page})=>{
|
||||
await page.route('**/prompt_manager/recent?**',route=>route.fulfill({json:{success:true,results:[{text:'Saved prompt',tags:[]}]}}));
|
||||
const note=page.locator('.pm-classic').getByText('Text comes from a link. Your typed text is kept for when you disconnect it.',{exact:true});
|
||||
await expect(note).toBeHidden();
|
||||
await page.evaluate(()=>{
|
||||
const widget=node.widgets.find(item=>item.name==='text');widget.value=widget.inputEl.value='Typed classic';
|
||||
node.inputs=[{name:'text',type:'STRING',link:3,widget:{name:'text'}}];node.onConnectionsChange();
|
||||
});
|
||||
await expect(note).toBeVisible();
|
||||
// The link drives the run; the saved workflow keeps what the user typed, not a stale or empty value.
|
||||
expect(await page.evaluate(()=>node.serialize().widgets_values[0])).toBe('Typed classic');
|
||||
await page.getByRole('button',{name:'📋 Recent',exact:true}).click();
|
||||
await page.getByRole('button',{name:/Saved prompt/}).click();
|
||||
expect(await page.evaluate(()=>node.widgets.find(item=>item.name==='text').value)).toBe('Typed classic');
|
||||
await expect(page.getByRole('alert').filter({hasText:'Text comes from a link. Disconnect it to use a saved prompt.'})).toBeVisible();
|
||||
await page.evaluate(()=>{node.inputs[0].link=null;node.onConnectionsChange();});
|
||||
await expect(note).toBeHidden();
|
||||
await page.getByRole('button',{name:/Saved prompt/}).click();
|
||||
expect(await page.evaluate(()=>node.widgets.find(item=>item.name==='text').value)).toBe('Saved prompt');
|
||||
// A workflow load re-evaluates the link state.
|
||||
await page.evaluate(()=>{node.inputs[0].link=8;node.configure({});});
|
||||
await expect(note).toBeVisible();
|
||||
});
|
||||
|
||||
test('PromptManagerText shows linked text in its legacy controls and keeps the typed text (#343)',async({page})=>{
|
||||
await page.evaluate(async()=>{
|
||||
class TextNode {
|
||||
constructor(){this.widgets=['text','category','tags','search_text'].map(name=>({name,value:'',inputEl:document.createElement('textarea')}));this.widgets_values=[];this.inputs=[{name:'text',type:'STRING',link:null,widget:{name:'text'}}];}
|
||||
addDOMWidget(name,type,element){element.id='legacy-text-node';document.body.append(element);return {element};}
|
||||
setSize(){} setDirtyCanvas(){} serialize(){return {properties:{...this.properties}};}
|
||||
}
|
||||
await fixtureApp.extensions[0].beforeRegisterNodeDef(TextNode,{name:'PromptManagerText'},fixtureApp);
|
||||
window.textNode=new TextNode();textNode.onNodeCreated();
|
||||
const widget=textNode.widgets[0];widget.value=widget.inputEl.value='Typed text node';
|
||||
});
|
||||
const note=page.locator('#legacy-text-node').getByText('Text comes from a link. Your typed text is kept for when you disconnect it.',{exact:true});
|
||||
await expect(note).toBeHidden();
|
||||
await page.evaluate(()=>{textNode.inputs[0].link=5;textNode.onConnectionsChange();});
|
||||
await expect(note).toBeVisible();
|
||||
expect(await page.evaluate(()=>textNode.serialize().widgets_values[0])).toBe('Typed text node');
|
||||
await page.evaluate(()=>{textNode.usePrompt({text:'Replacement'},0);});
|
||||
expect(await page.evaluate(()=>textNode.widgets[0].value)).toBe('Typed text node');
|
||||
await page.evaluate(()=>{textNode.inputs[0].link=null;textNode.onConnectionsChange();textNode.onRemoved();});
|
||||
await expect(note).toBeHidden();
|
||||
});
|
||||
|
||||
@@ -26,7 +26,8 @@ test.beforeEach(async({page,browserName})=>{
|
||||
window.fixtureApp={registerExtension(extension){this.extension=extension;},ui:{settings:new EventTarget()},graph:{setDirtyCanvas(){}},canvas:{}};
|
||||
await import('/extensions/promptmanager/prompt_manager.js');
|
||||
class Node {
|
||||
constructor(){this.widgets=['text','category','tags','search_text'].map(name=>({name,type:'text',value:'',options:{},inputEl:document.createElement('textarea')}));this.inputs=['clip','prepend_text','append_text'].map(name=>({name,link:null}));this.properties={};this.size=[550,700];}
|
||||
constructor(){this.widgets=['text','category','tags','search_text'].map(name=>({name,type:'text',value:'',options:{},inputEl:document.createElement('textarea')}));this.inputs=[...['clip','prepend_text','append_text'].map(name=>({name,type:name==='clip'?'CLIP':'STRING',link:null})),...['text','category','tags','search_text'].map(name=>({name,type:'STRING',link:null,widget:{name}}))];this.properties={};this.size=[550,700];}
|
||||
findInputByType(type){const index=this.inputs.findIndex(input=>input.type===type&&input.link==null);return index<0?undefined:{index,slot:this.inputs[index]};}
|
||||
addWidget(type,name,value,callback,options){const widget={type,name,value,callback,options};this.widgets.push(widget);return widget;}
|
||||
addDOMWidget(name,type,element){const host=document.createElement('div');host.style.height='650px';host.className='fixture-dom-widget';host.append(element);document.body.append(host);return {element};}
|
||||
setSize(size){this.size=size;}setDirtyCanvas(){}
|
||||
@@ -246,13 +247,39 @@ test(`${variant} restores workflow values and disclosures and protects connected
|
||||
await page.evaluate(()=>{const saved=node.serialize();node.configure(saved);});
|
||||
await expect(page.getByLabel('Category',{exact:true})).toBeVisible();
|
||||
await expect(page.getByLabel('Category',{exact:true})).toHaveValue('Saved category');
|
||||
await page.evaluate(()=>{node.inputs.push({name:'text',link:9});node.onConnectionsChange();});
|
||||
await page.evaluate(()=>{node.inputs.find(input=>input.name==='text').link=9;node.onConnectionsChange();});
|
||||
await expect(page.getByRole('textbox',{name:'Main prompt',exact:true})).toBeHidden();
|
||||
await expect(page.getByText('Text is supplied by an upstream connection.',{exact:true})).toBeVisible();
|
||||
await expect(page.getByText('Main prompt comes from a link. Your typed text is kept for when you disconnect it.',{exact:true})).toBeVisible();
|
||||
await page.evaluate(()=>{node.inputs.find(input=>input.name==='text').link=null;node.onConnectionsChange();});
|
||||
await expect(page.getByRole('textbox',{name:'Main prompt',exact:true})).toHaveValue('Saved draft');
|
||||
});
|
||||
|
||||
test(`${variant} offers text as a link socket, keeps typed text while linked and through a reload (#343)`,async({page})=>{
|
||||
const sockets=()=>page.evaluate(()=>Object.fromEntries(node.inputs.map(input=>[input.name,input.widget?.name??null])));
|
||||
// The hidden text widget no longer owns the input, so the host draws it as a socket; the other widget inputs are unchanged.
|
||||
expect(await sockets()).toEqual({clip:null,prepend_text:null,append_text:null,text:null,category:'category',tags:'tags',search_text:'search_text'});
|
||||
expect(await page.evaluate(()=>node.findInputByType('STRING').slot.name)).toBe('text');
|
||||
expect(await page.evaluate(()=>node.findInputByType('CLIP').slot.name)).toBe('clip');
|
||||
const main=page.getByRole('textbox',{name:'Main prompt',exact:true});
|
||||
await main.fill('Typed main');
|
||||
await page.evaluate(()=>{node.inputs.find(input=>input.name==='text').link=21;node.onConnectionsChange();});
|
||||
await expect(main).toBeHidden();
|
||||
const note=page.getByText('Main prompt comes from a link. Your typed text is kept for when you disconnect it.',{exact:true});
|
||||
await expect(note).toBeVisible();
|
||||
await expect(page.locator('[data-composed-preview]')).toHaveText('[text supplied by connection]');
|
||||
// A second STRING link dropped on the editor falls back to the frontend's own choice.
|
||||
expect(await page.evaluate(()=>node.findInputByType('STRING').slot.name)).toBe('prepend_text');
|
||||
// An older saved workflow brings the widget binding back with its inputs: configure exposes the socket again.
|
||||
const saved=await page.evaluate(()=>{const saved=node.serialize();node.inputs.find(input=>input.name==='text').widget={name:'text'};node.configure(saved);return saved;});
|
||||
expect(saved.widgets_values[0]).toBe('Typed main');
|
||||
expect((await sockets()).text).toBeNull();
|
||||
await expect(note).toBeVisible();
|
||||
await page.evaluate(()=>{node.inputs.find(input=>input.name==='text').link=null;node.onConnectionsChange();});
|
||||
await expect(main).toBeVisible();
|
||||
await expect(main).toHaveValue('Typed main');
|
||||
await expect(note).toBeHidden();
|
||||
});
|
||||
|
||||
test(`${variant} searches quoted tags with Enter and explicitly loads metadata`, async ({page}) => {
|
||||
let requested;
|
||||
await page.route('**/prompt_manager/search?**', async route => {
|
||||
|
||||
@@ -17,7 +17,8 @@ test.beforeEach(async({page,browserName})=>{
|
||||
window.fixtureApp={registerExtension(extension){this.extension=extension;},ui:{settings},graph:{_nodes:[],setDirtyCanvas(){}},canvas:{},graphToPrompt:async()=>({})};
|
||||
await import('/extensions/promptmanager/prompt_manager.js');
|
||||
class Node {
|
||||
constructor(){this.id=1;this.type='PromptManagerPair';this.widgets=['text','negative_text','category','tags','search_text'].map(name=>({name,type:'text',value:'',options:{},inputEl:document.createElement('textarea')}));this.inputs=['clip','prepend_text','append_text'].map(name=>({name,link:null}));this.properties={};this.size=[1100,700];}
|
||||
constructor(){this.id=1;this.type='PromptManagerPair';this.widgets=['text','negative_text','category','tags','search_text'].map(name=>({name,type:'text',value:'',options:{},inputEl:document.createElement('textarea')}));this.inputs=[...['clip','prepend_text','append_text'].map(name=>({name,type:name==='clip'?'CLIP':'STRING',link:null})),...['text','negative_text','category','tags','search_text'].map(name=>({name,type:'STRING',link:null,widget:{name}}))];this.properties={};this.size=[1100,700];}
|
||||
findInputByType(type){const index=this.inputs.findIndex(input=>input.type===type&&input.link==null);return index<0?undefined:{index,slot:this.inputs[index]};}
|
||||
addWidget(type,name,value,callback,options){const widget={type,name,value,callback,options};this.widgets.push(widget);return widget;}
|
||||
addDOMWidget(name,type,element){const host=document.createElement('div');host.style.cssText='height:700px;width:1100px;max-width:100%';host.className='fixture-dom-widget';host.append(element);document.body.append(host);return {element};}
|
||||
setSize(size){this.size=size;}setDirtyCanvas(){}
|
||||
@@ -123,6 +124,46 @@ test('a superseded library response never replaces the latest pair selection',as
|
||||
await expect(page.locator('.pm-editor-result')).not.toContainText('Stale positive');
|
||||
});
|
||||
|
||||
test('paired editor offers positive and negative link sockets and keeps both typed texts (#343)',async({page})=>{
|
||||
await draft(page,'Typed positive','Typed negative');
|
||||
expect(await page.evaluate(()=>node.inputs.filter(input=>!input.widget).map(input=>input.name))).toEqual(['clip','prepend_text','append_text','text','negative_text']);
|
||||
const target=()=>page.evaluate(()=>node.findInputByType('STRING').slot.name);
|
||||
expect(await target()).toBe('text');
|
||||
const positive=page.getByRole('textbox',{name:'Positive prompt',exact:true}),negative=page.getByRole('textbox',{name:'Negative prompt',exact:true});
|
||||
const positiveNote=page.getByText('Positive prompt comes from a link. Your typed text is kept for when you disconnect it.',{exact:true});
|
||||
const negativeNote=page.getByText('Negative prompt comes from a link. Your typed text is kept for when you disconnect it.',{exact:true});
|
||||
const link=(name,value)=>page.evaluate(({name,value})=>{node.inputs.find(input=>input.name===name).link=value;node.onConnectionsChange();},{name,value});
|
||||
for(const [linked,next] of [[['text'],'negative_text'],[['negative_text'],'text'],[['text','negative_text'],'prepend_text'],[[],'text']]){
|
||||
for(const name of ['text','negative_text'])await link(name,linked.includes(name)?30:null);
|
||||
expect(await target()).toBe(next);
|
||||
await expect(positive).toBeVisible({visible:!linked.includes('text')});
|
||||
await expect(negative).toBeVisible({visible:!linked.includes('negative_text')});
|
||||
await expect(positiveNote).toBeVisible({visible:linked.includes('text')});
|
||||
await expect(negativeNote).toBeVisible({visible:linked.includes('negative_text')});
|
||||
const saved=await page.evaluate(()=>{const saved=node.serialize();for(const input of node.inputs)if(['text','negative_text'].includes(input.name))input.widget={name:input.name};node.configure(saved);return saved.widgets_values.slice(0,2);});
|
||||
expect(saved).toEqual(['Typed positive','Typed negative']);
|
||||
expect(await page.evaluate(()=>node.inputs.filter(input=>input.widget&&['text','negative_text'].includes(input.name)).length)).toBe(0);
|
||||
}
|
||||
await expect(positive).toHaveValue('Typed positive');await expect(negative).toHaveValue('Typed negative');
|
||||
});
|
||||
|
||||
test('a link dropped on either prompt box, or its note, fills that box (#343 review M1)',async({page})=>{
|
||||
const keys=await page.evaluate(()=>[...document.querySelectorAll('.pm-pair .lg-node-widget [data-slot-key]')].map(key=>key.dataset.slotKey));
|
||||
// Nodes 2.0 resolves a drop through the nearest .lg-node-widget and its slot key: positive box and note, negative box and note.
|
||||
expect(keys).toEqual(['1-in-3','1-in-3','1-in-4','1-in-4']);
|
||||
const at=(name,connecting=true)=>page.evaluate(({name,connecting})=>{
|
||||
fixtureApp.canvas={linkConnector:{state:{connectingTo:connecting?'input':undefined}},ds:{convertOffsetToCanvas:point=>point},canvas:{getBoundingClientRect:()=>({left:0,top:0})}};
|
||||
const box=[...document.querySelectorAll('.pm-pair .lg-node-widget')].find(element=>element.getClientRects().length&&element.textContent.includes(name)).getBoundingClientRect();
|
||||
return node.getInputOnPos([box.left+box.width/2,box.top+box.height/2])?.name??null;
|
||||
},{name,connecting});
|
||||
expect(await at('Negative prompt')).toBe('negative_text');
|
||||
expect(await at('Positive prompt')).toBe('text');
|
||||
expect(await at('Negative prompt',false)).toBeNull();
|
||||
await page.evaluate(()=>{node.inputs.find(input=>input.name==='negative_text').link=4;node.onConnectionsChange();});
|
||||
expect(await at('Negative prompt comes from a link')).toBe('negative_text');
|
||||
expect(await page.evaluate(()=>{fixtureApp.canvas={};return node.getInputOnPos([1,1])??null;})).toBeNull();
|
||||
});
|
||||
|
||||
test('queue hook synchronizes both editors, preserves links and forwards branch execution arguments',async({page})=>{
|
||||
await draft(page);
|
||||
const args=await page.evaluate(async()=>{
|
||||
|
||||
@@ -23,6 +23,9 @@ from tests.browser.request_hold import register_request_holds
|
||||
from tests.browser import slow_reset_watch
|
||||
from tests.browser.workspace_activity_fixture import register_activity_commands
|
||||
from tests.browser.workspace_system_fixture import register_system_commands
|
||||
from tests.browser.workspace_integrations_fixture import (
|
||||
register_integrations_commands,
|
||||
)
|
||||
from tests.browser.workspace_concept_fixture import (
|
||||
MAX_EXTRA_IMAGES as CONCEPT_MAX_EXTRA_IMAGES,
|
||||
MAX_SEEDLESS_IMAGES as CONCEPT_MAX_SEEDLESS_IMAGES,
|
||||
@@ -203,6 +206,7 @@ def create_app(root: Path) -> web.Application:
|
||||
reveal_stub = register_reveal_stub(routes, api, root)
|
||||
register_activity_commands(app, routes, api)
|
||||
register_system_commands(app, routes, api, root)
|
||||
register_integrations_commands(app, routes, api, root)
|
||||
request_holds = register_request_holds(app, routes)
|
||||
slow_reset_watch.install(app, api)
|
||||
|
||||
|
||||
@@ -0,0 +1,329 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { SIZES, THEMES, seedConcept, openConcept, openProduction, diffInventory, mutate } from './concept-inventory.mjs';
|
||||
|
||||
// #249 System S9 Task 25: System › Backup & export (app.js renderSystem 691–693), its "Restore preview" (`restore`,
|
||||
// 1472–1476) and its "Import & inspect" (`import`, 1483–1487) against the immutable concept, on the concept-shaped
|
||||
// fixture. Each concept measurement is mapped only by the recorded deviations (D103, D104, D105; the Import dialog's
|
||||
// D15 and D55), each commented where it applies; after that the difference must be empty, and a mutated label, row,
|
||||
// danger mark or token must be noticed.
|
||||
test.use({ locale: 'en-US', timezoneId: 'UTC' });
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
await seedConcept(request);
|
||||
await expect(await request.post('/__test__/system/reset')).toBeOK();
|
||||
await expect(await request.post('/__test__/jobs/restart')).toBeOK();
|
||||
page.conceptErrors = [];
|
||||
page.on('pageerror', error => page.conceptErrors.push(error.message));
|
||||
});
|
||||
|
||||
test.afterEach(async ({ page, request, browserName }) => {
|
||||
await saveCoverage(page, browserName);
|
||||
await request.post('/__test__/jobs/restart');
|
||||
await request.post('/__test__/system/reset');
|
||||
expect(page.conceptErrors).toEqual([]);
|
||||
});
|
||||
|
||||
const PREVIEW_URL = '/prompt_manager/restore/preview';
|
||||
|
||||
/** Runs in the page: the section's surfaces under `mainSelector`, their text, controls, geometry and tokens. */
|
||||
function measureBackups(mainSelector) {
|
||||
const layout = document.querySelector(`${mainSelector} .settings-layout`);
|
||||
const css = (node, ...keys) => node ? Object.fromEntries(keys.map(key => [key, getComputedStyle(node)[key]])) : null;
|
||||
const name = node => [node.localName, ...[...node.classList].sort()].join('.');
|
||||
const text = node => node?.textContent.trim() ?? null;
|
||||
const button = node => ({ text: text(node), className: node.className,
|
||||
...css(node, 'backgroundColor', 'color', 'fontSize', 'paddingTop', 'paddingLeft', 'borderTopColor') });
|
||||
const base = layout.getBoundingClientRect();
|
||||
const box = node => {
|
||||
const rect = node.getBoundingClientRect();
|
||||
return { left: Math.round(rect.left - base.left), top: Math.round(rect.top - base.top), width: Math.round(rect.width) };
|
||||
};
|
||||
const control = row => {
|
||||
const select = row.querySelector(':scope > select');
|
||||
if (select) return { kind: 'select', id: select.id, options: [...select.options].map(option => [option.textContent, option.disabled]),
|
||||
selected: select.selectedOptions[0]?.textContent ?? null, disabled: select.disabled };
|
||||
const toggle = row.querySelector(':scope > * input[type="checkbox"]');
|
||||
return toggle ? { kind: 'switch', id: toggle.id, wrapper: name(toggle.parentElement), checked: toggle.checked,
|
||||
disabled: toggle.disabled, ...css(toggle.parentElement, 'width', 'height') } : null;
|
||||
};
|
||||
const columns = layout.querySelector(':scope > .two-col');
|
||||
const surfaces = [...columns.querySelectorAll(':scope > section.surface')];
|
||||
const policy = layout.querySelector(':scope > section.surface');
|
||||
const chip = policy.querySelector(':scope > h2 > .chip');
|
||||
const row = policy.querySelector('.setting-row');
|
||||
const status = surfaces[0].querySelector(':scope > p.small');
|
||||
const warning = policy.querySelector(':scope > .warning-note');
|
||||
return {
|
||||
layout: [...layout.children].map(name),
|
||||
columns: { ...css(columns, 'display', 'gridTemplateColumns', 'columnGap'), box: box(columns) },
|
||||
surfaces: surfaces.map(surface => ({ tree: [...surface.children].map(name), heading: text(surface.querySelector(':scope > h2')),
|
||||
intro: text(surface.querySelector(':scope > p:not(.small)')), box: box(surface),
|
||||
actions: [...surface.querySelectorAll(':scope > .row-actions')].map(group => ({ tree: [...group.children].map(name),
|
||||
...css(group, 'display', 'columnGap', 'flexWrap', 'marginTop'),
|
||||
buttons: [...group.children].map(button) })) })),
|
||||
status: { text: text(status), ...css(status, 'fontSize', 'color', 'marginTop') },
|
||||
surface: css(surfaces[0], 'paddingTop', 'paddingLeft', 'backgroundColor', 'borderTopWidth', 'borderTopColor', 'borderTopLeftRadius'),
|
||||
h2: css(surfaces[0].querySelector('h2'), 'fontSize', 'fontWeight', 'color', 'marginBottom'),
|
||||
intro: css(surfaces[0].querySelector(':scope > p'), 'fontSize', 'color', 'lineHeight'),
|
||||
// Its place below the columns (the concept's `.surface + .surface` 18px), whatever the columns' height.
|
||||
policy: { tree: [...policy.children].map(name), heading: text(policy.querySelector(':scope > h2')),
|
||||
box: { left: box(policy).left, width: box(policy).width,
|
||||
gap: Math.round(policy.getBoundingClientRect().top - columns.getBoundingClientRect().bottom) },
|
||||
chip: { text: text(chip), className: chip.className, ...css(chip, 'color', 'fontSize', 'borderTopColor', 'backgroundColor') },
|
||||
rows: [...policy.querySelectorAll(':scope > .setting-row')].map(item => ({ tree: [...item.children].map(name),
|
||||
text: [...item.firstElementChild.children].map(node => [name(node), text(node), node.localName === 'label' ? node.htmlFor : null]),
|
||||
control: control(item) })),
|
||||
warning: { text: text(warning), ...css(warning, 'fontSize', 'color', 'marginTop', 'marginBottom') } },
|
||||
row: css(row, 'display', 'alignItems', 'justifyContent', 'columnGap', 'paddingTop', 'paddingBottom', 'borderTopWidth', 'borderTopColor',
|
||||
'marginTop'),
|
||||
label: css(row.querySelector('label'), 'fontSize', 'fontWeight', 'color'),
|
||||
description: css(row.querySelector('p'), 'fontSize', 'color', 'marginTop', 'maxWidth'),
|
||||
select: css(policy.querySelector('.setting-row > select'), 'fontSize', 'minWidth', 'maxWidth', 'backgroundColor', 'color',
|
||||
'borderTopColor'),
|
||||
pageScroll: document.documentElement.scrollWidth > document.documentElement.clientWidth,
|
||||
};
|
||||
}
|
||||
|
||||
const SCHEDULE_REASON = 'Daily and Weekly are not available yet: back up with Back up whenever you choose.';
|
||||
const SAFETY_DESCRIPTION = 'Keep a verified copy of the database on disk before maintenance, cleanups and LoRA imports change ' +
|
||||
'records. A restore always keeps one.';
|
||||
|
||||
/**
|
||||
* D103: "Back up" (the concept's "Simulate backup"), the status line from /backup/status, "Export JSON" (its "Export
|
||||
* sample JSON") and the two secondary exports in a second `.row-actions.spaced` with the concept's plain buttons.
|
||||
* D104: the toggle's line says what the policy does, the schedule's reason replaces the prototype's proposal line,
|
||||
* Daily and Weekly are disabled, and the warning note is truthful. The fixture has no backup yet.
|
||||
*/
|
||||
function mapBackupsConcept(reference) {
|
||||
const [database, portable] = reference.surfaces;
|
||||
const [backup, restore] = database.actions[0].buttons;
|
||||
const [exportButton, importButton] = portable.actions[0].buttons;
|
||||
const secondary = ['Export all prompts JSON', 'Export saved pairs JSON'].map(label => ({ ...exportButton, text: label }));
|
||||
return {
|
||||
...reference,
|
||||
surfaces: [
|
||||
{ ...database, actions: [{ ...database.actions[0], buttons: [{ ...backup, text: 'Back up' }, restore] }] },
|
||||
{ ...portable, tree: [...portable.tree, 'div.row-actions.spaced'],
|
||||
actions: [{ ...portable.actions[0], buttons: [{ ...exportButton, text: 'Export JSON' }, importButton] },
|
||||
{ ...portable.actions[0], tree: ['button', 'button'], buttons: secondary }] },
|
||||
],
|
||||
status: { ...reference.status, text: 'No backup recorded.' },
|
||||
policy: { ...reference.policy,
|
||||
rows: reference.policy.rows.map(item => (item.control.kind !== 'select'
|
||||
// D104 (M2): what the safety-backup policy really does.
|
||||
? { ...item, text: item.text.map(([node, value, target]) => [node, node === 'p' ? SAFETY_DESCRIPTION : value, target]) }
|
||||
: { ...item,
|
||||
text: item.text.map(([node, value, target]) => [node, node === 'p' ? SCHEDULE_REASON : value, target]),
|
||||
control: { ...item.control, options: item.control.options.map(([label]) => [label, label !== 'Manual only']) } })),
|
||||
warning: { ...reference.policy.warning, text: 'Restores are staged, migrated and validated before replacing live data.' } },
|
||||
};
|
||||
}
|
||||
|
||||
async function conceptBackups(page, theme) {
|
||||
await openConcept(page, 'system', theme);
|
||||
await page.locator('#main .section-nav').getByRole('button', { name: 'Backup & export', exact: true }).click();
|
||||
await expect(page.locator('#main .settings-layout > .two-col > section.surface')).toHaveCount(2);
|
||||
await page.mouse.move(0, 0);
|
||||
return page.evaluate(measureBackups, '#main');
|
||||
}
|
||||
|
||||
async function productionBackups(page, theme, status = 'No backup recorded.') {
|
||||
await openProduction(page, 'system/backups', theme);
|
||||
await expect(page.locator('#workspace-main .settings-layout > .two-col > section.surface')).toHaveCount(2);
|
||||
// The status line and the policy row are their own reads: compare once both answered.
|
||||
await expect(page.locator('#workspace-main #backup-status')).toHaveText(status);
|
||||
await expect(page.locator('#workspace-main #backup-before')).toBeEnabled();
|
||||
await page.mouse.move(0, 0);
|
||||
return page.evaluate(measureBackups, '#workspace-main');
|
||||
}
|
||||
|
||||
for (const theme of THEMES) for (const [width, height] of SIZES) {
|
||||
test(`backup & export matches the concept ${theme} ${width}x${height}`, async ({ page }, testInfo) => {
|
||||
await page.setViewportSize({ width, height });
|
||||
const concept = await conceptBackups(page, theme);
|
||||
await page.screenshot({ path: testInfo.outputPath(`concept-backups-${theme}-${width}.png`), fullPage: true });
|
||||
const production = await productionBackups(page, theme);
|
||||
await page.screenshot({ path: testInfo.outputPath(`production-backups-${theme}-${width}.png`), fullPage: true });
|
||||
expect(diffInventory(mapBackupsConcept(concept), production)).toEqual([]);
|
||||
expect(production.pageScroll).toBe(false);
|
||||
});
|
||||
}
|
||||
|
||||
/** Runs in the page: the open dialog's title, body tree, grid, note, backup box and actions with the concept's tokens. */
|
||||
function measureDialog() {
|
||||
const dialog = document.querySelector('dialog#modal[open]');
|
||||
const css = (node, ...keys) => node ? Object.fromEntries(keys.map(key => [key, getComputedStyle(node)[key]])) : null;
|
||||
const name = node => [node.localName, ...[...node.classList].sort()].join('.');
|
||||
const text = node => node?.textContent.trim() ?? null;
|
||||
const body = dialog.querySelector('#modal-body');
|
||||
const grid = body.querySelector(':scope > dl.meta-grid');
|
||||
const note = body.querySelector(':scope > p.small');
|
||||
const actions = body.querySelector(':scope > .dialog-actions');
|
||||
const list = body.querySelector(':scope > .operation-list');
|
||||
return {
|
||||
title: text(dialog.querySelector('#modal-title')),
|
||||
tree: [...body.children].map(name),
|
||||
grid: grid && { rows: [...grid.children].map(row => [text(row.querySelector('dt')), text(row.querySelector('dd'))]),
|
||||
...css(grid, 'display', 'gridTemplateColumns', 'columnGap', 'rowGap'),
|
||||
dt: css(grid.querySelector('dt'), 'fontSize', 'color', 'textTransform'), dd: css(grid.querySelector('dd'), 'fontSize', 'color') },
|
||||
note: note && { text: text(note), ...css(note, 'fontSize', 'color') },
|
||||
backup: body.querySelector(':scope > .notice') && { tree: [...body.querySelector(':scope > .notice').children].map(name),
|
||||
text: [...body.querySelectorAll(':scope > .notice > div:first-child > *')].map(node => [node.localName, text(node)]),
|
||||
buttons: [...body.querySelectorAll(':scope > .notice button')].map(text) },
|
||||
list: list && { ...css(list, 'display', 'rowGap'),
|
||||
rows: [...list.children].map(row => ({ tree: [...row.children].map(name), title: text(row.querySelector('h3')),
|
||||
description: text(row.querySelector('p')), buttons: [...row.querySelectorAll('button')].map(text),
|
||||
...css(row, 'display', 'alignItems', 'paddingTop', 'borderTopColor') })) },
|
||||
actions: actions && { tree: [...actions.children].map(name), ...css(actions, 'display', 'justifyContent', 'columnGap', 'marginTop'),
|
||||
buttons: [...actions.children].map(item => ({ text: text(item), className: item.className,
|
||||
...css(item, 'backgroundColor', 'color', 'fontSize', 'paddingTop', 'paddingLeft', 'borderTopColor') })) },
|
||||
};
|
||||
}
|
||||
|
||||
const RESTORE_BACKUP_BOX = Object.freeze({ tree: ['div', 'div.row-actions'], text: [['h3', 'Back up the database first (recommended)'],
|
||||
['p', 'Downloads a copy of the prompts database before the restore replaces it.']], buttons: ['Back up now'] });
|
||||
|
||||
/**
|
||||
* D105: the server's review in the concept's "Restore preview": no sample note; the four rows with the review's values,
|
||||
* then "In the backup" and "In the library now"; the hold line in the concept's dialog `p.small` (its Import dialog's
|
||||
* intro, `small`); the D55 backup box before the actions; "Restore" (the concept's "Simulate restore", still danger).
|
||||
*/
|
||||
function mapRestoreConcept(reference, small, rows) {
|
||||
const [cancel, restore] = reference.actions.buttons;
|
||||
return {
|
||||
...reference,
|
||||
tree: ['dl.meta-grid', 'p.small', 'div.notice', 'div.dialog-actions'],
|
||||
grid: { ...reference.grid, rows },
|
||||
note: { ...small, text: 'Restore replaces the whole library with this backup and runs alone in Activity. ' +
|
||||
'Nothing changes until you restore. This review holds for 15 minutes.' },
|
||||
backup: RESTORE_BACKUP_BOX,
|
||||
actions: { ...reference.actions, buttons: [cancel, { ...restore, text: 'Restore' }] },
|
||||
};
|
||||
}
|
||||
|
||||
/** D15 and D55, as on Images: Rescan for "Scan an output folder", then "Scan for metadata"; no portable recipe import. */
|
||||
function mapImportConcept(reference) {
|
||||
return { ...reference,
|
||||
note: { ...reference.note, text: 'Image inspection and output scanning have different effects. Choose the workflow you want.' },
|
||||
list: { ...reference.list, rows: reference.list.rows.filter(row => row.title !== 'Import portable recipes').flatMap(row =>
|
||||
row.title === 'Scan an output folder'
|
||||
? [{ ...row, description: 'Find files added or removed in the output folders.', buttons: ['Rescan'] },
|
||||
{ ...row, title: 'Scan for metadata' }] : [row]) } };
|
||||
}
|
||||
|
||||
async function conceptDialog(page, theme, action, title) {
|
||||
await conceptBackups(page, theme);
|
||||
await page.locator(`#main .settings-layout [data-action="${action}"]`).click();
|
||||
await expect(page.locator('dialog#modal[open] #modal-title')).toHaveText(title);
|
||||
await page.mouse.move(0, 0);
|
||||
return page.evaluate(measureDialog);
|
||||
}
|
||||
|
||||
/** Production's restore review of a fresh backup of this server's library; the review's own answer too. */
|
||||
async function productionRestore(page, request, theme) {
|
||||
const copy = await request.get('/prompt_manager/backup');
|
||||
await expect(copy).toBeOK();
|
||||
// That copy is this session's download now.
|
||||
await productionBackups(page, theme, /^Downloaded backup · .+ · verified\.$/);
|
||||
const chooser = page.waitForEvent('filechooser');
|
||||
const answer = page.waitForResponse(response => response.url().endsWith(PREVIEW_URL));
|
||||
await page.locator('#workspace-main').getByRole('button', { name: 'Review restore…', exact: true }).click();
|
||||
await (await chooser).setFiles({ name: 'prompts_backup_20261025_093000.db', mimeType: 'application/octet-stream',
|
||||
buffer: await copy.body() });
|
||||
const preview = await (await answer).json();
|
||||
await expect(page.locator('dialog#modal[open] #modal-title')).toHaveText('Restore preview');
|
||||
await page.mouse.move(0, 0);
|
||||
return { preview, production: await page.evaluate(measureDialog) };
|
||||
}
|
||||
|
||||
const counts = summary => `${summary.prompts} ${summary.prompts === 1 ? 'prompt' : 'prompts'} · ${summary.image_links} ` +
|
||||
`${summary.image_links === 1 ? 'image link' : 'image links'} · ${summary.pairs} ${summary.pairs === 1 ? 'saved pair' : 'saved pairs'}` +
|
||||
` · ${summary.runs} ${summary.runs === 1 ? 'run' : 'runs'}`;
|
||||
|
||||
function restoreRows(preview, snapshot) {
|
||||
return [['Snapshot', snapshot], ['Integrity', 'Verified'], ['Migration', 'Current schema'],
|
||||
['Safety backup', `Created before replacing (${preview.safety_backup.destination_label})`],
|
||||
['In the backup', counts(preview.summary)], ['In the library now', counts(preview.live)]];
|
||||
}
|
||||
|
||||
for (const theme of THEMES) for (const [width, height] of SIZES) {
|
||||
test(`restore preview and import dialogs match the concept ${theme} ${width}x${height}`, async ({ page, request }, testInfo) => {
|
||||
await page.setViewportSize({ width, height });
|
||||
// The concept's own dialog `p.small` (its Import dialog's intro) is the reference for the review's note.
|
||||
const conceptImport = await conceptDialog(page, theme, 'import', 'Import & inspect');
|
||||
await page.screenshot({ path: testInfo.outputPath(`concept-import-${theme}-${width}.png`) });
|
||||
const conceptRestore = await conceptDialog(page, theme, 'restore', 'Restore preview');
|
||||
expect(conceptRestore.tree).toEqual(['p.demo-note', 'dl.meta-grid', 'div.dialog-actions']);
|
||||
await page.screenshot({ path: testInfo.outputPath(`concept-restore-${theme}-${width}.png`) });
|
||||
const { preview, production } = await productionRestore(page, request, theme);
|
||||
await page.screenshot({ path: testInfo.outputPath(`production-restore-${theme}-${width}.png`) });
|
||||
const snapshot = production.grid.rows[0][1];
|
||||
expect(snapshot).toMatch(/^prompts_backup_20261025_093000.db · (\d+ KB|\d+\.\d MB) · .+$/);
|
||||
const { text: _intro, ...small } = conceptImport.note;
|
||||
expect(diffInventory(mapRestoreConcept(conceptRestore, small, restoreRows(preview, snapshot)), production)).toEqual([]);
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(page.locator('dialog#modal[open]')).toHaveCount(0);
|
||||
await page.locator('#workspace-main').getByRole('button', { name: 'Import / inspect file', exact: true }).click();
|
||||
await expect(page.locator('dialog#modal[open] #modal-title')).toHaveText('Import & inspect');
|
||||
await page.mouse.move(0, 0);
|
||||
await page.screenshot({ path: testInfo.outputPath(`production-import-${theme}-${width}.png`) });
|
||||
expect(diffInventory(mapImportConcept(conceptImport), await page.evaluate(measureDialog))).toEqual([]);
|
||||
});
|
||||
}
|
||||
|
||||
test('backup & export comparison detects mutations', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const concept = await conceptBackups(page, 'dark');
|
||||
const expected = mapBackupsConcept(concept);
|
||||
await productionBackups(page, 'dark');
|
||||
const measure = () => page.evaluate(measureBackups, '#workspace-main');
|
||||
const paths = async () => diffInventory(expected, await measure()).map(item => item.path);
|
||||
expect(await paths()).toEqual([]);
|
||||
// Unmapped, the section differs exactly on the D103 and D104 paths (the added row's own paths, then the rest).
|
||||
const unmapped = diffInventory(concept, await measure()).map(item => item.path).sort();
|
||||
const added = 'surfaces.1.actions.1.';
|
||||
expect(unmapped.some(path => path.startsWith(added))).toBe(true);
|
||||
expect(unmapped.filter(path => !path.startsWith(added))).toEqual([
|
||||
'policy.rows.0.text.1.1', 'policy.rows.1.control.options.1.1', 'policy.rows.1.control.options.2.1', 'policy.rows.1.text.1.1',
|
||||
'policy.warning.text',
|
||||
'status.text', 'surfaces.0.actions.0.buttons.0.text', 'surfaces.1.actions.0.buttons.0.text', 'surfaces.1.actions.length',
|
||||
'surfaces.1.tree.3', 'surfaces.1.tree.length']);
|
||||
// A renamed label, a removed row, a lost danger mark and a swapped token are noticed.
|
||||
await page.evaluate(() => { document.querySelector('#workspace-main .setting-row label').textContent = 'Safety backup'; });
|
||||
expect(await paths()).toEqual(['policy.rows.0.text.0.1']);
|
||||
await page.evaluate(() => { document.querySelector('#workspace-main .setting-row label').textContent = 'Create a safety backup before maintenance'; });
|
||||
const undoRow = await mutate(page, { remove: ['#workspace-main .settings-layout > section.surface > .setting-row:last-of-type'] });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['policy.rows.length', 'policy.tree.length']));
|
||||
await undoRow();
|
||||
await page.evaluate(() => { document.querySelector('#workspace-main button.danger').classList.remove('danger'); });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['surfaces.0.actions.0.buttons.1.className']));
|
||||
await page.evaluate(() => { document.querySelector('#workspace-main .two-col .row-actions > button:nth-child(2)').classList.add('danger'); });
|
||||
const undoToken = await mutate(page, { css: '#workspace-main .settings-layout { --line: rgb(255, 0, 0); }' });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['row.borderTopColor']));
|
||||
await undoToken();
|
||||
const undoChip = await mutate(page, { css: '#workspace-main .chip.blue { color: rgb(255, 0, 0); }' });
|
||||
expect(await paths()).toEqual(['policy.chip.color']);
|
||||
await undoChip();
|
||||
expect(await paths()).toEqual([]);
|
||||
|
||||
// The restore review: a renamed term, a lost danger mark and the grid's token are noticed.
|
||||
const conceptImport = await conceptDialog(page, 'dark', 'import', 'Import & inspect');
|
||||
const conceptRestore = await conceptDialog(page, 'dark', 'restore', 'Restore preview');
|
||||
const { preview, production } = await productionRestore(page, request, 'dark');
|
||||
const { text: _intro, ...small } = conceptImport.note;
|
||||
const expectedRestore = mapRestoreConcept(conceptRestore, small, restoreRows(preview, production.grid.rows[0][1]));
|
||||
const restorePaths = async () => diffInventory(expectedRestore, await page.evaluate(measureDialog)).map(item => item.path);
|
||||
expect(await restorePaths()).toEqual([]);
|
||||
await page.evaluate(() => { document.querySelector('dialog#modal[open] dl.meta-grid dt').textContent = 'Backup'; });
|
||||
expect(await restorePaths()).toEqual(['grid.rows.0.0']);
|
||||
await page.evaluate(() => { document.querySelector('dialog#modal[open] dl.meta-grid dt').textContent = 'Snapshot'; });
|
||||
await page.evaluate(() => { document.querySelector('dialog#modal[open] .dialog-actions .danger').classList.remove('danger'); });
|
||||
expect(await restorePaths()).toEqual(expect.arrayContaining(['actions.buttons.1.className']));
|
||||
await page.evaluate(() => { document.querySelector('dialog#modal[open] .dialog-actions button:last-child').classList.add('danger'); });
|
||||
const undoGrid = await mutate(page, { css: 'dialog#modal dl.meta-grid { column-gap: 3px; }' });
|
||||
expect(await restorePaths()).toEqual(expect.arrayContaining(['grid.columnGap']));
|
||||
await undoGrid();
|
||||
expect(await restorePaths()).toEqual([]);
|
||||
});
|
||||
@@ -172,8 +172,8 @@ function mapOverviewConcept(reference, overview) {
|
||||
// D93: the real pending count; Review metadata opens Needs review (the concept's go-review).
|
||||
'3 ambiguous image links': row => ({ ...row,
|
||||
title: `${number(overview.needs_review.pending)} ambiguous image link${overview.needs_review.pending === 1 ? '' : 's'}` }),
|
||||
// D93: no backup has been recorded (verified backups arrive with S8).
|
||||
'Backup status': row => ({ ...row, description: 'No retained backup record.' }),
|
||||
// D93 (S9): what GET /backup/status reports; the fixture has no backup yet.
|
||||
'Backup status': row => ({ ...row, description: 'No backup recorded.' }),
|
||||
// D93: there is no paging, so the shared settings are grid and density.
|
||||
'One display configuration': row => ({ ...row, description: 'Grid and density settings are shared across views.' }),
|
||||
// D93: the real sources, not samples.
|
||||
@@ -219,6 +219,9 @@ async function productionOverview(page, theme) {
|
||||
await expect(page.locator('#workspace-main .summary-metrics')).toBeVisible();
|
||||
// The thumbnail row is its own read (I2): compare once it answered.
|
||||
await expect(page.locator('#workspace-main .operation h3').filter({ hasText: 'thumbnails' })).toHaveText('No missing thumbnails');
|
||||
// So is the backup row (S9).
|
||||
await expect(page.locator('#workspace-main .operation').filter({ has: page.locator('h3', { hasText: 'Backup status' }) })
|
||||
.locator('p')).toHaveText('No backup recorded.');
|
||||
await page.mouse.move(0, 0);
|
||||
return page.evaluate(measureSystem, '#workspace-main');
|
||||
}
|
||||
|
||||
@@ -0,0 +1,343 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { SIZES, THEMES, seedConcept, openConcept, openProduction, diffInventory, mutate } from './concept-inventory.mjs';
|
||||
|
||||
// #249 System S10 Task 28: System › Integrations (app.js renderSystem 682–684) and the LoRA import review against the
|
||||
// immutable concept, on the concept-shaped fixture with a fake LoraManager install. Each concept measurement is mapped
|
||||
// only by the recorded deviations (D106, D107, D114), each commented where it applies; after that the difference must
|
||||
// be empty, and a mutated label, option, row or token must be noticed. The concept starts its sample import without a
|
||||
// review, so the import review is compared with the concept's own record-changing review, `maintenancePlan`
|
||||
// (953–958), whose shell, grid, note and actions it shares.
|
||||
test.use({ locale: 'en-US', timezoneId: 'UTC' });
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
await seedConcept(request);
|
||||
await expect(await request.post('/__test__/system/reset')).toBeOK();
|
||||
await expect(await request.post('/__test__/integrations/reset')).toBeOK();
|
||||
await expect(await request.post('/__test__/jobs/restart')).toBeOK();
|
||||
page.conceptErrors = [];
|
||||
page.on('pageerror', error => page.conceptErrors.push(error.message));
|
||||
});
|
||||
|
||||
test.afterEach(async ({ page, request, browserName }) => {
|
||||
await saveCoverage(page, browserName);
|
||||
await request.post('/__test__/jobs/restart');
|
||||
await request.post('/__test__/integrations/reset');
|
||||
await request.post('/__test__/system/reset');
|
||||
expect(page.conceptErrors).toEqual([]);
|
||||
});
|
||||
|
||||
const PREVIEW_URL = '/prompt_manager/lora/import/preview';
|
||||
|
||||
/** Runs in the page: Integrations' surfaces under `mainSelector`, every row's text and control, geometry and tokens. */
|
||||
function measureIntegrations(mainSelector) {
|
||||
const layout = document.querySelector(`${mainSelector} .settings-layout`);
|
||||
const css = (node, ...keys) => node ? Object.fromEntries(keys.map(key => [key, getComputedStyle(node)[key]])) : null;
|
||||
const name = node => [node.localName, ...[...node.classList].sort()].join('.');
|
||||
const text = node => node?.textContent.trim() ?? null;
|
||||
const control = row => {
|
||||
const select = row.querySelector(':scope > select');
|
||||
if (select) return { kind: 'select', id: select.id, options: [...select.options].map(option => [option.textContent, option.disabled]),
|
||||
selected: select.selectedOptions[0]?.textContent ?? null, disabled: select.disabled };
|
||||
const input = row.querySelector(':scope > input');
|
||||
if (input) return { kind: `input.${input.type}`, id: input.id, value: input.value, readOnly: input.readOnly };
|
||||
const toggle = row.querySelector(':scope > * input[type="checkbox"]');
|
||||
return toggle ? { kind: 'switch', id: toggle.id, wrapper: name(toggle.parentElement), checked: toggle.checked,
|
||||
disabled: toggle.disabled, ...css(toggle.parentElement, 'width', 'height') } : null;
|
||||
};
|
||||
const surfaces = [...layout.querySelectorAll(':scope > section.surface')];
|
||||
const firstRow = surfaces[0].querySelector('.setting-row');
|
||||
const note = layout.querySelector('.setting-row p.small');
|
||||
return {
|
||||
layout: [...layout.children].map(name),
|
||||
surfaces: surfaces.map(surface => ({ tree: [...surface.children].map(name), heading: text(surface.querySelector(':scope > h2')),
|
||||
intro: text(surface.querySelector(':scope > p')),
|
||||
rows: [...surface.querySelectorAll(':scope > .setting-row')].map(row => ({ tree: [...row.children].map(name),
|
||||
text: [...row.firstElementChild.children].map(node => [name(node), text(node), node.localName === 'label' ? node.htmlFor : null]),
|
||||
control: control(row) })),
|
||||
operations: [...surface.querySelectorAll(':scope > .operation')].map(row => ({ tree: [...row.children].map(name),
|
||||
title: text(row.querySelector('h3')), description: text(row.querySelector('p')),
|
||||
actions: [...row.querySelectorAll('.row-actions > button')].map(item => [text(item), item.className]) })) })),
|
||||
surface: css(surfaces[1], 'paddingTop', 'paddingLeft', 'backgroundColor', 'borderTopWidth', 'borderTopColor', 'borderTopLeftRadius',
|
||||
'marginTop'),
|
||||
row: css(firstRow, 'display', 'alignItems', 'justifyContent', 'columnGap', 'paddingTop', 'paddingBottom', 'borderTopWidth',
|
||||
'borderTopColor', 'marginTop'),
|
||||
label: css(firstRow.querySelector('label'), 'fontSize', 'fontWeight', 'color'),
|
||||
description: css(firstRow.querySelector('p'), 'fontSize', 'color', 'marginTop', 'maxWidth'),
|
||||
select: css(firstRow.querySelector('select'), 'fontSize', 'minWidth', 'maxWidth', 'backgroundColor', 'color', 'borderTopColor'),
|
||||
password: css(layout.querySelector('.setting-row input[type="password"]'), 'fontSize', 'minWidth', 'maxWidth', 'backgroundColor',
|
||||
'color', 'borderTopColor'),
|
||||
// The concept's `.small` text (its Save row status); production's row notes use the same class.
|
||||
small: css(layout.querySelector('#settings-status'), 'fontSize', 'color'),
|
||||
note: note && css(note, 'fontSize', 'color'),
|
||||
// Whether each select is as wide as the browser lays out a copy holding only its chosen option, i.e. it shows
|
||||
// that option whole (the browser's own select sizing, arrow included).
|
||||
selectsFit: [...layout.querySelectorAll('.setting-row > select')].map(select => {
|
||||
const copy = select.cloneNode(false);
|
||||
copy.removeAttribute('id');
|
||||
copy.append(new Option(select.selectedOptions[0]?.textContent ?? ''));
|
||||
Object.assign(copy.style, { position: 'absolute', visibility: 'hidden', width: 'max-content', minWidth: '0',
|
||||
maxWidth: 'none', flex: 'none' });
|
||||
select.parentElement.append(copy);
|
||||
const needed = copy.getBoundingClientRect().width;
|
||||
copy.remove();
|
||||
return select.getBoundingClientRect().width + 0.5 >= needed;
|
||||
}),
|
||||
pageScroll: document.documentElement.scrollWidth > document.documentElement.clientWidth,
|
||||
};
|
||||
}
|
||||
|
||||
const SEND_REASON = 'Sending always asks for the ComfyUI window and node, so “Ask each time” is the only option.';
|
||||
const RENDERER_REASON = 'Prompt Manager nodes always follow ComfyUI’s own renderer setting.';
|
||||
const RENDERER_NONE = 'No ComfyUI window is open in this browser, so its renderer is not known.';
|
||||
const RETURN_REASON = 'Not available: sending applies the text in ComfyUI and keeps this page open.';
|
||||
const PATH_DESCRIPTION = 'Found automatically in ComfyUI’s custom_nodes folders; read-only, as on the existing settings page.';
|
||||
|
||||
/**
|
||||
* The concept's Integrations with the recorded deviations applied, for the fixture's LoraManager (detected, disabled,
|
||||
* no key, trigger words off and not loaded).
|
||||
*/
|
||||
function mapIntegrationsConcept(reference) {
|
||||
const [comfy, lora] = reference.surfaces;
|
||||
const small = text => ['p.small', text, null];
|
||||
const firstOnly = row => ({ ...row, control: { ...row.control, options: row.control.options.map(([label], index) => [label, index > 0]) } });
|
||||
const [send, renderer, back] = comfy.rows;
|
||||
const [enable, triggers, key] = lora.rows;
|
||||
return {
|
||||
...reference,
|
||||
surfaces: [
|
||||
{ ...comfy, rows: [
|
||||
// D106: only "Ask each time" can be chosen, and the row says why.
|
||||
{ ...firstOnly(send), text: [...send.text, small(SEND_REASON)] },
|
||||
// D106: only "Follow host renderer" can be chosen; the reason, then the detected renderer.
|
||||
{ ...firstOnly(renderer), text: [...renderer.text, small(RENDERER_REASON), small(RENDERER_NONE)] },
|
||||
// D106: Return to canvas is not available: off, disabled and saying why.
|
||||
{ ...back, text: [...back.text, small(RETURN_REASON)], control: { ...back.control, checked: false, disabled: true } },
|
||||
] },
|
||||
{ ...lora,
|
||||
// D106: the LoraManager path row (legacy, read-only) follows the integration toggle.
|
||||
tree: [lora.tree[0], lora.tree[1], 'div.setting-row', 'div.setting-row', 'div.setting-row', 'div.setting-row', 'div.operation'],
|
||||
rows: [
|
||||
// The real detection replaces the concept's sample line; the integration is off in the fixture.
|
||||
{ ...enable, text: [enable.text[0], ['p', 'Detected: custom_nodes/ComfyUI-Lora-Manager.', null]],
|
||||
control: { ...enable.control, checked: false } },
|
||||
{ tree: ['div', 'input'],
|
||||
text: [['label', 'LoraManager path', 'lora-path'], ['p', PATH_DESCRIPTION, null]],
|
||||
control: { kind: 'input.text', id: 'lora-path', value: 'custom_nodes/ComfyUI-Lora-Manager', readOnly: true } },
|
||||
// The trigger cache is a separate fact from the setting: its own line.
|
||||
{ ...triggers, text: [...triggers.text, small('Trigger words are not loaded.')], control: { ...triggers.control, checked: false } },
|
||||
// D114: no prototype warning; the saved-key line under it.
|
||||
// (No key is saved in the fixture, so "Clear saved key" is not offered.)
|
||||
{ ...key, text: [key.text[0], ['p', 'Credentials stay in the server configuration.', null], small('No key saved.')],
|
||||
control: { ...key.control, value: '' } },
|
||||
],
|
||||
// D107: the import is reviewed first ("Review import"; the concept's button only simulated a job).
|
||||
operations: lora.operations.map(operation => ({ ...operation, actions: [['Review import', '']] })) },
|
||||
],
|
||||
// D106: both selects show their whole chosen option at every width (at 390 the concept cuts "Follow host
|
||||
// renderer"; production lets the two selects grow to 160px and never shrink for the reason lines).
|
||||
selectsFit: reference.selectsFit.map(() => true),
|
||||
select: reference.select.maxWidth === '120px' ? { ...reference.select, maxWidth: '160px' } : reference.select,
|
||||
// The D106/D114 row notes are `p.small` inside the row text, so they read as its description (`.setting-row p`).
|
||||
note: { fontSize: reference.description.fontSize, color: reference.description.color },
|
||||
};
|
||||
}
|
||||
|
||||
async function conceptIntegrations(page, theme) {
|
||||
await openConcept(page, 'system', theme);
|
||||
await page.locator('#main .section-nav').getByRole('button', { name: 'Integrations', exact: true }).click();
|
||||
await expect(page.locator('#main .settings-layout > section.surface')).toHaveCount(2);
|
||||
await page.mouse.move(0, 0);
|
||||
return page.evaluate(measureIntegrations, '#main');
|
||||
}
|
||||
|
||||
async function productionIntegrations(page, theme) {
|
||||
await openProduction(page, 'system/integrations', theme);
|
||||
// Compare once the LoRA state and the renderer check answered.
|
||||
await expect(page.locator('#workspace-main #lora-path')).toHaveValue('custom_nodes/ComfyUI-Lora-Manager');
|
||||
await expect(page.locator('#workspace-main #renderer-status')).toHaveText(RENDERER_NONE);
|
||||
await page.mouse.move(0, 0);
|
||||
return page.evaluate(measureIntegrations, '#workspace-main');
|
||||
}
|
||||
|
||||
const fakeLoraManager = async (request, data = {}) =>
|
||||
expect(await request.post('/__test__/integrations/lora-manager', { data })).toBeOK();
|
||||
|
||||
for (const theme of THEMES) for (const [width, height] of SIZES) {
|
||||
test(`integrations matches the concept ${theme} ${width}x${height}`, async ({ page, request }, testInfo) => {
|
||||
await page.setViewportSize({ width, height });
|
||||
await fakeLoraManager(request);
|
||||
const concept = await conceptIntegrations(page, theme);
|
||||
await page.screenshot({ path: testInfo.outputPath(`concept-integrations-${theme}-${width}.png`), fullPage: true });
|
||||
const production = await productionIntegrations(page, theme);
|
||||
await page.screenshot({ path: testInfo.outputPath(`production-integrations-${theme}-${width}.png`), fullPage: true });
|
||||
expect(diffInventory(mapIntegrationsConcept(concept), production)).toEqual([]);
|
||||
expect(production.pageScroll).toBe(false);
|
||||
});
|
||||
}
|
||||
|
||||
// D106 below the concept's smallest size: on a 320px phone both selects still show their whole option, without page
|
||||
// scroll (S10 re-review N9).
|
||||
for (const theme of THEMES) test(`integrations selects fit at 320 wide ${theme}`, async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 320, height: 700 });
|
||||
await fakeLoraManager(request);
|
||||
const production = await productionIntegrations(page, theme);
|
||||
expect(production.selectsFit).toEqual([true, true]);
|
||||
expect(production.pageScroll).toBe(false);
|
||||
});
|
||||
|
||||
test('integrations comparison detects mutations', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
await fakeLoraManager(request);
|
||||
const concept = await conceptIntegrations(page, 'dark');
|
||||
const expected = mapIntegrationsConcept(concept);
|
||||
const production = await productionIntegrations(page, 'dark');
|
||||
expect(diffInventory(expected, production)).toEqual([]);
|
||||
// Unmapped, the section differs only where D106, D107 and D114 are recorded.
|
||||
expect(diffInventory(concept, production).map(item => item.path.split('.').slice(0, 4).join('.'))
|
||||
.filter((path, index, all) => all.indexOf(path) === index).sort()).toEqual([
|
||||
'note', 'note.color', 'note.fontSize', 'surfaces.0.rows.0', 'surfaces.0.rows.1', 'surfaces.0.rows.2', 'surfaces.1.operations.0',
|
||||
'surfaces.1.rows.0', 'surfaces.1.rows.1', 'surfaces.1.rows.2', 'surfaces.1.rows.3', 'surfaces.1.rows.length',
|
||||
'surfaces.1.tree.5', 'surfaces.1.tree.6', 'surfaces.1.tree.length'].sort());
|
||||
const measure = () => page.evaluate(measureIntegrations, '#workspace-main');
|
||||
const paths = async () => diffInventory(expected, await measure()).map(item => item.path);
|
||||
// A removed row.
|
||||
const undoRow = await mutate(page, { remove: ['#workspace-main section.surface:nth-of-type(2) > .setting-row:nth-of-type(3)'] });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['surfaces.1.rows.length', 'surfaces.1.tree.length']));
|
||||
await undoRow();
|
||||
// A changed label and a changed option.
|
||||
await page.evaluate(() => { document.querySelector('#workspace-main label[for="renderer"]').textContent = 'Renderer'; });
|
||||
expect(await paths()).toEqual(['surfaces.0.rows.1.text.0.1']);
|
||||
await page.evaluate(() => { document.querySelector('#workspace-main label[for="renderer"]').textContent = 'Host compatibility'; });
|
||||
await page.evaluate(() => { document.querySelector('#workspace-main #send-destination option').textContent = 'Always ask'; });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['surfaces.0.rows.0.control.options.0.0']));
|
||||
await page.evaluate(() => { document.querySelector('#workspace-main #send-destination option').textContent = 'Ask each time'; });
|
||||
// An unsupported option that became selectable.
|
||||
await page.evaluate(() => { document.querySelectorAll('#workspace-main #renderer option')[1].disabled = false; });
|
||||
expect(await paths()).toEqual(['surfaces.0.rows.1.control.options.1.1']);
|
||||
await page.evaluate(() => { document.querySelectorAll('#workspace-main #renderer option')[1].disabled = true; });
|
||||
// A select squeezed until its chosen option is cut (S10 review M3).
|
||||
const undoSqueeze = await mutate(page, { css: '#workspace-main #renderer { max-width: 70px !important; }' });
|
||||
expect(await paths()).toEqual(['selectsFit.1']);
|
||||
await undoSqueeze();
|
||||
// A swapped token.
|
||||
const undoToken = await mutate(page, { css: '#workspace-main .settings-layout { --line: rgb(255, 0, 0); }' });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['row.borderTopColor']));
|
||||
await undoToken();
|
||||
expect(diffInventory(expected, await measure())).toEqual([]);
|
||||
});
|
||||
|
||||
/** Runs in the page: the open review dialog: its title, body and the concept's tokens. */
|
||||
function measureReview() {
|
||||
const dialog = document.querySelector('dialog#modal[open]');
|
||||
const css = (node, ...keys) => node ? Object.fromEntries(keys.map(key => [key, getComputedStyle(node)[key]])) : null;
|
||||
const name = node => [node.localName, ...[...node.classList].sort()].join('.');
|
||||
const text = node => node?.textContent.trim() ?? null;
|
||||
const body = dialog.querySelector('#modal-body');
|
||||
const grid = body.querySelector(':scope > dl.meta-grid');
|
||||
const note = [...body.querySelectorAll(':scope > p.small')].at(-1);
|
||||
const actions = body.querySelector(':scope > .dialog-actions');
|
||||
return {
|
||||
title: text(dialog.querySelector('#modal-title')),
|
||||
tree: [...body.children].map(name),
|
||||
heading: { text: text(body.querySelector(':scope > h3')),
|
||||
...css(body.querySelector(':scope > h3'), 'fontSize', 'fontWeight', 'color', 'marginBottom') },
|
||||
grid: { rows: [...grid.children].map(row => [text(row.querySelector('dt')), text(row.querySelector('dd'))]),
|
||||
...css(grid, 'display', 'gridTemplateColumns', 'columnGap', 'rowGap'),
|
||||
dt: css(grid.querySelector('dt'), 'fontSize', 'color', 'textTransform'), dd: css(grid.querySelector('dd'), 'fontSize', 'color') },
|
||||
details: body.querySelector(':scope > details') && { summary: text(body.querySelector(':scope > details > summary')),
|
||||
tree: [...body.querySelector(':scope > details').children].map(name) },
|
||||
backup: body.querySelector(':scope > .notice') && { tree: [...body.querySelector(':scope > .notice').children].map(name),
|
||||
text: [...body.querySelectorAll(':scope > .notice > div:first-child > *')].map(node => [node.localName, text(node)]),
|
||||
buttons: [...body.querySelectorAll(':scope > .notice button')].map(text) },
|
||||
note: { text: text(note), ...css(note, 'fontSize', 'color', 'marginTop') },
|
||||
actions: { tree: [...actions.children].map(name), ...css(actions, 'display', 'justifyContent', 'columnGap', 'marginTop'),
|
||||
buttons: [...actions.children].map(item => ({ text: text(item), className: item.className,
|
||||
...css(item, 'backgroundColor', 'color', 'fontSize', 'paddingTop', 'paddingLeft', 'borderTopColor') })) },
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* D107: the server's import plan in the concept's review: "Review LoRA import" with "LoRA reference data" as its `h3`,
|
||||
* the concept's four rows with the plan's values plus Found and Network, the LoRAs it found (`details`), the hold
|
||||
* line with the cache sentence in the concept's `p.small`, the D55 backup box, and Cancel / "Start import".
|
||||
*/
|
||||
function mapImportConcept(reference, plan) {
|
||||
const [cancel, start] = reference.actions.buttons;
|
||||
return {
|
||||
...reference,
|
||||
title: 'Review LoRA import',
|
||||
tree: ['h3', 'dl.meta-grid', 'details', 'p.small', 'div.notice', 'div.dialog-actions'],
|
||||
heading: { ...reference.heading, text: 'LoRA reference data' },
|
||||
grid: { ...reference.grid, rows: [['Scope', plan.scope.label], ['Found', '2 metadata files · 2 readable'],
|
||||
['Protected', 'Nothing is protected'], ['Network', 'Downloads up to 2 example images from CivitAI'],
|
||||
['Backup', `Create and verify before changes (${plan.backup.destination_label})`],
|
||||
['Media on disk', 'No files are removed; image links are replaced']] },
|
||||
details: { summary: 'LoRAs found (2)', tree: ['summary', 'pre'] },
|
||||
backup: { tree: ['div', 'div.row-actions'], text: [['h3', 'Back up the database first (recommended)'],
|
||||
['p', 'Downloads a copy of the prompts database before this operation changes any records.']], buttons: ['Back up now'] },
|
||||
note: { ...reference.note, text: 'Nothing changes until you apply. This review holds for 15 minutes. '
|
||||
+ 'Example images downloaded before a failure stay in the image cache.' },
|
||||
actions: { ...reference.actions, buttons: [cancel, { ...start, text: 'Start import' }] },
|
||||
};
|
||||
}
|
||||
|
||||
async function conceptReview(page, theme) {
|
||||
await openConcept(page, 'system', theme);
|
||||
await page.locator('#main .section-nav').getByRole('button', { name: 'Maintenance', exact: true }).click();
|
||||
await page.locator('#main [data-plan="duplicate prompts"]').click();
|
||||
await expect(page.locator('dialog#modal[open] #modal-title')).toHaveText('Review maintenance plan');
|
||||
await page.mouse.move(0, 0);
|
||||
return page.evaluate(measureReview);
|
||||
}
|
||||
|
||||
async function productionReview(page, request, theme) {
|
||||
await openProduction(page, 'system/integrations', theme);
|
||||
await expect(page.locator('#workspace-main #lora-path')).toHaveValue('custom_nodes/ComfyUI-Lora-Manager');
|
||||
// The renderer line above settles first: its new text moves the rows below (a click must not land mid-shift).
|
||||
await expect(page.locator('#workspace-main #renderer-status')).toHaveText(RENDERER_NONE);
|
||||
// The import reviews what is enabled: save the integration on first.
|
||||
await page.locator('#workspace-main #lora-enable').check();
|
||||
await page.getByRole('button', { name: 'Save preferences', exact: true }).click();
|
||||
await expect(page.locator('#settings-status')).toHaveText('Saved to the server configuration.');
|
||||
const answer = page.waitForResponse(response => response.url().endsWith(PREVIEW_URL));
|
||||
await page.locator('#workspace-main').getByRole('button', { name: 'Review import', exact: true }).click();
|
||||
const plan = await (await answer).json();
|
||||
await expect(page.locator('dialog#modal[open] #modal-title')).toHaveText('Review LoRA import');
|
||||
await page.mouse.move(0, 0);
|
||||
return { plan, production: await page.evaluate(measureReview) };
|
||||
}
|
||||
|
||||
for (const theme of THEMES) for (const [width, height] of SIZES) {
|
||||
test(`LoRA import review matches the concept ${theme} ${width}x${height}`, async ({ page, request }, testInfo) => {
|
||||
await page.setViewportSize({ width, height });
|
||||
await fakeLoraManager(request, { loras: 2, civitai: true });
|
||||
const concept = await conceptReview(page, theme);
|
||||
expect(concept.tree).toEqual(['p.demo-note', 'h3', 'dl.meta-grid', 'p.small', 'div.dialog-actions']);
|
||||
await page.screenshot({ path: testInfo.outputPath(`concept-plan-review-${theme}-${width}.png`) });
|
||||
const { plan, production } = await productionReview(page, request, theme);
|
||||
await page.screenshot({ path: testInfo.outputPath(`production-lora-review-${theme}-${width}.png`) });
|
||||
expect(diffInventory(mapImportConcept(concept, plan), production)).toEqual([]);
|
||||
});
|
||||
}
|
||||
|
||||
test('LoRA import review comparison detects mutations', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
await fakeLoraManager(request, { loras: 2, civitai: true });
|
||||
const concept = await conceptReview(page, 'dark');
|
||||
const { plan } = await productionReview(page, request, 'dark');
|
||||
const expected = mapImportConcept(concept, plan);
|
||||
const paths = async () => diffInventory(expected, await page.evaluate(measureReview)).map(item => item.path);
|
||||
expect(await paths()).toEqual([]);
|
||||
await page.evaluate(() => { document.querySelectorAll('dialog#modal[open] dl.meta-grid dt')[3].textContent = 'Downloads'; });
|
||||
expect(await paths()).toEqual(['grid.rows.3.0']);
|
||||
await page.evaluate(() => { document.querySelectorAll('dialog#modal[open] dl.meta-grid dt')[3].textContent = 'Network'; });
|
||||
const undoGrid = await mutate(page, { css: 'dialog#modal dl.meta-grid { column-gap: 3px; }' });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['grid.columnGap']));
|
||||
await undoGrid();
|
||||
const undoBox = await mutate(page, { remove: ['dialog#modal[open] #modal-body > .notice'] });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['backup', 'tree.length']));
|
||||
await undoBox();
|
||||
expect(await paths()).toEqual([]);
|
||||
});
|
||||
@@ -0,0 +1,332 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { SIZES, THEMES, seedConcept, openConcept, openProduction, diffInventory, mutate } from './concept-inventory.mjs';
|
||||
|
||||
// #249 System S7 Task 20: System › Maintenance (app.js renderSystem 688–690), its plan dialog (`maintenancePlan`,
|
||||
// 953–958) for each operation and its duplicates dialog (`duplicates`, 1457–1461) against the immutable concept, on
|
||||
// the concept-shaped real fixture. Each concept measurement is mapped only by the recorded deviations (D100, D101,
|
||||
// D102; the health report is D95, compared in system-concept.spec), each commented where it applies; after that the
|
||||
// difference must be empty, and a mutated label, row or token must be noticed.
|
||||
test.use({ locale: 'en-US', timezoneId: 'UTC' });
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
await seedConcept(request);
|
||||
await expect(await request.post('/__test__/system/reset')).toBeOK();
|
||||
await expect(await request.post('/__test__/jobs/restart')).toBeOK();
|
||||
page.conceptErrors = [];
|
||||
page.on('pageerror', error => page.conceptErrors.push(error.message));
|
||||
});
|
||||
|
||||
test.afterEach(async ({ page, request, browserName }) => {
|
||||
await saveCoverage(page, browserName);
|
||||
await request.post('/__test__/jobs/restart');
|
||||
await request.post('/__test__/system/reset');
|
||||
expect(page.conceptErrors).toEqual([]);
|
||||
});
|
||||
|
||||
const PREVIEW_URL = '/prompt_manager/maintenance/preview';
|
||||
|
||||
/** Runs in the page: Maintenance's notice and surfaces under `mainSelector`, with the concept's geometry and tokens. */
|
||||
function measureMaintenance(mainSelector) {
|
||||
const layout = document.querySelector(`${mainSelector} .settings-layout`);
|
||||
const css = (node, ...keys) => node ? Object.fromEntries(keys.map(key => [key, getComputedStyle(node)[key]])) : null;
|
||||
const name = node => [node.localName, ...[...node.classList].sort()].join('.');
|
||||
const text = node => node?.textContent.trim() ?? null;
|
||||
const button = node => ({ text: text(node), className: node.className,
|
||||
...css(node, 'backgroundColor', 'color', 'fontSize', 'paddingTop', 'paddingLeft', 'borderTopColor', 'opacity', 'cursor') });
|
||||
const box = node => {
|
||||
const rect = node.getBoundingClientRect(), base = layout.getBoundingClientRect();
|
||||
return { left: Math.round(rect.left - base.left), width: Math.round(rect.width) };
|
||||
};
|
||||
const notice = layout.querySelector(':scope > .notice');
|
||||
const surfaces = [...layout.querySelectorAll(':scope > section.surface')];
|
||||
const operation = surfaces[0].querySelector('.operation');
|
||||
return {
|
||||
layout: [...layout.children].map(name),
|
||||
notice: { tree: [...notice.children].map(name), text: [...notice.querySelectorAll(':scope > div > *')].map(node => [node.localName, text(node)]),
|
||||
run: button(notice.querySelector(':scope > button')),
|
||||
...css(notice, 'display', 'alignItems', 'justifyContent', 'columnGap', 'borderTopColor', 'backgroundColor', 'paddingTop', 'marginBottom') },
|
||||
surfaces: surfaces.map(surface => ({ tree: [...surface.children].map(name), heading: text(surface.querySelector(':scope > h2')),
|
||||
box: box(surface),
|
||||
rows: [...surface.querySelectorAll(':scope > .operation')].map(row => ({ tree: [...row.children].map(name),
|
||||
title: text(row.querySelector('h3')), description: text(row.querySelector('p')),
|
||||
actions: [...row.querySelectorAll('.row-actions > button')].map(button) })) })),
|
||||
surface: css(surfaces[0], 'paddingTop', 'paddingLeft', 'backgroundColor', 'borderTopWidth', 'borderTopColor', 'borderTopLeftRadius', 'marginTop'),
|
||||
h2: css(surfaces[0].querySelector('h2'), 'fontSize', 'fontWeight', 'color', 'marginBottom'),
|
||||
operation: css(operation, 'display', 'alignItems', 'justifyContent', 'columnGap', 'paddingTop', 'paddingBottom',
|
||||
'borderTopWidth', 'borderTopColor'),
|
||||
h3: css(operation.querySelector('h3'), 'fontSize', 'fontWeight', 'color'),
|
||||
p: css(operation.querySelector('p'), 'fontSize', 'color', 'marginTop'),
|
||||
actions: css(operation.querySelector('.row-actions'), 'display', 'columnGap', 'flexShrink'),
|
||||
pageScroll: document.documentElement.scrollWidth > document.documentElement.clientWidth,
|
||||
};
|
||||
}
|
||||
|
||||
/** D102: the Metadata cache row says why its review is unavailable and its button is aria-disabled (D93's dimmed style). */
|
||||
function mapMaintenanceConcept(reference) {
|
||||
return {
|
||||
...reference,
|
||||
surfaces: reference.surfaces.map(surface => ({ ...surface, rows: surface.rows.map(row => row.title !== 'Metadata cache' ? row : {
|
||||
...row, description: `${row.description} Nothing is cached in this browser.`,
|
||||
actions: row.actions.map(action => ({ ...action, opacity: '0.55', cursor: 'not-allowed' })) }) })),
|
||||
};
|
||||
}
|
||||
|
||||
async function conceptMaintenance(page, theme) {
|
||||
await openConcept(page, 'system', theme);
|
||||
await page.locator('#main .section-nav').getByRole('button', { name: 'Maintenance', exact: true }).click();
|
||||
await expect(page.locator('#main .settings-layout > section.surface')).toHaveCount(2);
|
||||
await page.mouse.move(0, 0);
|
||||
return page.evaluate(measureMaintenance, '#main');
|
||||
}
|
||||
|
||||
async function productionMaintenance(page, theme) {
|
||||
await openProduction(page, 'system/maintenance', theme);
|
||||
await expect(page.locator('#workspace-main .settings-layout > section.surface')).toHaveCount(2);
|
||||
// The Metadata cache row reads this browser's storage once: compare after it answered.
|
||||
await expect(page.locator('#workspace-main .operation').filter({ hasText: 'Metadata cache' }).locator('p'))
|
||||
.toHaveText(/Nothing is cached in this browser\.$/);
|
||||
await page.mouse.move(0, 0);
|
||||
return page.evaluate(measureMaintenance, '#workspace-main');
|
||||
}
|
||||
|
||||
for (const theme of THEMES) for (const [width, height] of SIZES) {
|
||||
test(`maintenance matches the concept ${theme} ${width}x${height}`, async ({ page }, testInfo) => {
|
||||
await page.setViewportSize({ width, height });
|
||||
const concept = await conceptMaintenance(page, theme);
|
||||
await page.screenshot({ path: testInfo.outputPath(`concept-maintenance-${theme}-${width}.png`), fullPage: true });
|
||||
const production = await productionMaintenance(page, theme);
|
||||
await page.screenshot({ path: testInfo.outputPath(`production-maintenance-${theme}-${width}.png`), fullPage: true });
|
||||
expect(diffInventory(mapMaintenanceConcept(concept), production)).toEqual([]);
|
||||
expect(production.pageScroll).toBe(false);
|
||||
});
|
||||
}
|
||||
|
||||
/** Runs in the page: the open review dialog (plan or duplicates): its title, body and the concept's tokens. */
|
||||
function measureReview() {
|
||||
const dialog = document.querySelector('dialog#modal[open]');
|
||||
const css = (node, ...keys) => node ? Object.fromEntries(keys.map(key => [key, getComputedStyle(node)[key]])) : null;
|
||||
const name = node => [node.localName, ...[...node.classList].sort()].join('.');
|
||||
const text = node => node?.textContent.trim() ?? null;
|
||||
const body = dialog.querySelector('#modal-body');
|
||||
const grid = body.querySelector(':scope > dl.meta-grid');
|
||||
const compare = body.querySelector('.compare-grid');
|
||||
const item = compare?.querySelector('.compare-item');
|
||||
const notes = [...body.querySelectorAll(':scope > p.small')];
|
||||
const note = notes.at(-1);
|
||||
const actions = body.querySelector(':scope > .dialog-actions');
|
||||
return {
|
||||
title: text(dialog.querySelector('#modal-title')),
|
||||
tree: [...body.children].map(name),
|
||||
heading: body.querySelector(':scope > h3') && { text: text(body.querySelector(':scope > h3')),
|
||||
...css(body.querySelector(':scope > h3'), 'fontSize', 'fontWeight', 'color', 'marginBottom') },
|
||||
grid: grid && { rows: [...grid.children].map(row => [text(row.querySelector('dt')), text(row.querySelector('dd'))]),
|
||||
...css(grid, 'display', 'gridTemplateColumns', 'columnGap', 'rowGap'),
|
||||
dt: css(grid.querySelector('dt'), 'fontSize', 'color', 'textTransform'), dd: css(grid.querySelector('dd'), 'fontSize', 'color') },
|
||||
details: body.querySelector(':scope > details') && { summary: text(body.querySelector(':scope > details > summary')),
|
||||
tree: [...body.querySelector(':scope > details').children].map(name) },
|
||||
backup: body.querySelector(':scope > .notice') && { tree: [...body.querySelector(':scope > .notice').children].map(name),
|
||||
text: [...body.querySelectorAll(':scope > .notice > div:first-child > *')].map(node => [node.localName, text(node)]),
|
||||
buttons: [...body.querySelectorAll(':scope > .notice button')].map(text) },
|
||||
compare: compare && { tree: [...compare.children].map(name), ...css(compare, 'display', 'gridTemplateColumns', 'columnGap'),
|
||||
items: [...compare.children].map(entry => ({ tree: [...entry.children].map(name), label: text(entry.querySelector('label')),
|
||||
checked: entry.querySelector('input[type=radio]').checked, image: entry.querySelector('img')?.className ?? null })),
|
||||
item: css(item, 'paddingTop', 'paddingLeft', 'backgroundColor', 'borderTopWidth', 'borderTopColor', 'borderTopLeftRadius'),
|
||||
image: css(item.querySelector('img'), 'width', 'objectFit'),
|
||||
label: css(item.querySelector('label'), 'fontSize', 'color'),
|
||||
radio: { name: item.querySelector('input[type=radio]').name.startsWith('keep-') ? 'keep-*' : 'other',
|
||||
type: item.querySelector('input').type } },
|
||||
note: note && { text: text(note), ...css(note, 'fontSize', 'color', 'marginTop') },
|
||||
actions: { tree: [...actions.children].map(name), ...css(actions, 'display', 'justifyContent', 'columnGap', 'marginTop'),
|
||||
buttons: [...actions.children].map(item => ({ text: text(item), className: item.className,
|
||||
...css(item, 'backgroundColor', 'color', 'fontSize', 'paddingTop', 'paddingLeft', 'borderTopColor') })) },
|
||||
};
|
||||
}
|
||||
|
||||
// The concept's plan names (`data-plan`) and production's operation, row and button for each.
|
||||
const PLANS = Object.freeze([
|
||||
['duplicate prompts', 'duplicate_prompts', 'Duplicate prompt text'],
|
||||
['database optimization', 'optimize', 'Optimize database'],
|
||||
['unused prompt components', 'unused_components', 'Unused prompt components'],
|
||||
['missing-file links', 'missing_links', 'Missing-file links'],
|
||||
]);
|
||||
const BACKUP_BOX = Object.freeze({ tree: ['div', 'div.row-actions'], text: [['h3', 'Back up the database first (recommended)'],
|
||||
['p', 'Downloads a copy of the prompts database before this operation changes any records.']], buttons: ['Back up now'] });
|
||||
|
||||
/**
|
||||
* D100: the server's plan in the concept's dialog: the operation's row title as its `h3` (the concept prints its
|
||||
* `data-plan` name), the four rows with the plan's values, the listed records (`details`), the hold line in the
|
||||
* concept's `p.small`, the D55 backup box before the actions, and "Apply reviewed operation". No sample note.
|
||||
*/
|
||||
function mapPlanConcept(reference, title, plan) {
|
||||
const backup = plan.backup.policy === 'off' ? 'No safety backup (the policy is off)'
|
||||
: `Create and verify before changes (${plan.backup.destination_label})`;
|
||||
const tree = ['h3', 'dl.meta-grid', ...(plan.details.length ? ['details'] : []), 'p.small', 'div.notice', 'div.dialog-actions'];
|
||||
const [cancel, apply] = reference.actions.buttons;
|
||||
return {
|
||||
...reference,
|
||||
tree,
|
||||
heading: { ...reference.heading, text: title },
|
||||
grid: { ...reference.grid, rows: [['Scope', plan.scope.label],
|
||||
['Protected', plan.protected.reasons.length ? plan.protected.reasons.join(' · ') : 'Nothing is protected'],
|
||||
['Backup', backup], ['Media on disk', plan.media.label]] },
|
||||
details: plan.details.length ? { summary: `Listed records (${plan.details.length})`, tree: ['summary', 'pre'] } : null,
|
||||
backup: BACKUP_BOX,
|
||||
note: { ...reference.note, text: 'Nothing changes until you apply. This review holds for 15 minutes.' },
|
||||
actions: { ...reference.actions, buttons: [cancel, { ...apply, text: 'Apply reviewed operation' }] },
|
||||
};
|
||||
}
|
||||
|
||||
async function conceptPlan(page, theme, name) {
|
||||
await conceptMaintenance(page, theme);
|
||||
await page.locator(`#main [data-plan="${name}"]`).click();
|
||||
await expect(page.locator('dialog#modal[open] #modal-title')).toHaveText('Review maintenance plan');
|
||||
await page.mouse.move(0, 0);
|
||||
return page.evaluate(measureReview);
|
||||
}
|
||||
|
||||
/** Production's plan dialog for the row `title`, with the server's answer its review read. */
|
||||
async function productionPlan(page, title) {
|
||||
const answer = page.waitForResponse(response => response.url().endsWith(PREVIEW_URL));
|
||||
await page.locator('#workspace-main .operation').filter({ has: page.locator('h3', { hasText: title }) })
|
||||
.getByRole('button', { name: 'Review plan', exact: true }).click();
|
||||
const plan = await (await answer).json();
|
||||
await expect(page.locator('dialog#modal[open] #modal-body > h3')).toHaveText(title);
|
||||
await page.mouse.move(0, 0);
|
||||
return { plan, production: await page.evaluate(measureReview) };
|
||||
}
|
||||
|
||||
for (const theme of THEMES) for (const [width, height] of SIZES) {
|
||||
test(`plan dialogs match the concept ${theme} ${width}x${height}`, async ({ page, request }, testInfo) => {
|
||||
await page.setViewportSize({ width, height });
|
||||
// Something to merge, and a missing-file link, so no plan is empty.
|
||||
await expect(await request.post('/__test__/system/duplicate-prompts')).toBeOK();
|
||||
await expect(await request.post('/__test__/system/missing-file')).toBeOK();
|
||||
const concepts = [];
|
||||
for (const [name] of PLANS) {
|
||||
concepts.push(await conceptPlan(page, theme, name));
|
||||
await page.screenshot({ path: testInfo.outputPath(`concept-plan-${name.replace(/\W+/g, '-')}-${theme}-${width}.png`) });
|
||||
}
|
||||
expect(concepts[0].tree).toEqual(['p.demo-note', 'h3', 'dl.meta-grid', 'p.small', 'div.dialog-actions']);
|
||||
await productionMaintenance(page, theme);
|
||||
for (const [index, [, operation, title]] of PLANS.entries()) {
|
||||
const { plan, production } = await productionPlan(page, title);
|
||||
expect(plan.operation).toBe(operation);
|
||||
await page.screenshot({ path: testInfo.outputPath(`production-plan-${operation}-${theme}-${width}.png`) });
|
||||
expect(diffInventory(mapPlanConcept(concepts[index], title, plan), production)).toEqual([]);
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(page.locator('dialog#modal[open]')).toHaveCount(0);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* D101: the real group in the concept's markup: the status line (the concept's sample note's place), a group's
|
||||
* `p.small` and `.compare-grid` in one wrapper, a "Keep <source>/<path>" radio per copy, the oldest checked, and the
|
||||
* production note in the concept's `p.small`.
|
||||
*/
|
||||
function mapDuplicatesConcept(reference) {
|
||||
const items = reference.compare.items.map((item, index) => ({ ...item,
|
||||
label: `Keep output/s7-dupes/group_00_${index ? 'b' : 'a'}.png` }));
|
||||
return {
|
||||
...reference,
|
||||
tree: ['p.small', 'div.duplicate-groups', 'p.small', 'div.dialog-actions'],
|
||||
compare: { ...reference.compare, items },
|
||||
note: { ...reference.note, text: 'Removing a copy deletes its file from disk; its database links move to the kept copy. '
|
||||
+ 'History keeps the removed paths in its run records. Nothing is deleted until you review and apply the cleanup.' },
|
||||
};
|
||||
}
|
||||
|
||||
async function conceptDuplicates(page, theme) {
|
||||
await conceptMaintenance(page, theme);
|
||||
await page.locator('#main').getByRole('button', { name: 'Find duplicates', exact: true }).click();
|
||||
await expect(page.locator('dialog#modal[open] #modal-title')).toHaveText('Review duplicate image files');
|
||||
await page.mouse.move(0, 0);
|
||||
return page.evaluate(measureReview);
|
||||
}
|
||||
|
||||
async function productionDuplicates(page, theme) {
|
||||
await productionMaintenance(page, theme);
|
||||
await page.locator('#workspace-main').getByRole('button', { name: 'Find duplicates', exact: true }).click();
|
||||
await page.locator('#workspace-main').getByRole('button', { name: 'Review groups', exact: true }).click();
|
||||
await expect(page.locator('dialog#modal[open] .compare-item img')).toHaveCount(2);
|
||||
await page.locator('dialog#modal[open] .compare-item img').evaluateAll(images => Promise.all(images.map(image => image.decode())));
|
||||
await page.mouse.move(0, 0);
|
||||
return page.evaluate(measureReview);
|
||||
}
|
||||
|
||||
for (const theme of THEMES) for (const [width, height] of SIZES) {
|
||||
test(`duplicates dialog matches the concept ${theme} ${width}x${height}`, async ({ page, request }, testInfo) => {
|
||||
await page.setViewportSize({ width, height });
|
||||
await expect(await request.post('/__test__/system/duplicate-files', { data: { groups: 1, trio: false } })).toBeOK();
|
||||
const concept = await conceptDuplicates(page, theme);
|
||||
expect(concept.tree).toEqual(['p.demo-note', 'div.compare-grid', 'p.small', 'div.dialog-actions']);
|
||||
await page.screenshot({ path: testInfo.outputPath(`concept-duplicates-${theme}-${width}.png`) });
|
||||
const production = await productionDuplicates(page, theme);
|
||||
await page.screenshot({ path: testInfo.outputPath(`production-duplicates-${theme}-${width}.png`) });
|
||||
expect(diffInventory(mapDuplicatesConcept(concept), production)).toEqual([]);
|
||||
});
|
||||
}
|
||||
|
||||
test('maintenance comparison detects mutations', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const concept = await conceptMaintenance(page, 'dark');
|
||||
const expected = mapMaintenanceConcept(concept);
|
||||
const production = await productionMaintenance(page, 'dark');
|
||||
expect(diffInventory(expected, production)).toEqual([]);
|
||||
// Unmapped, the section differs only on the D102 row.
|
||||
expect(diffInventory(concept, production).map(item => item.path).sort()).toEqual([
|
||||
'surfaces.1.rows.3.actions.0.cursor', 'surfaces.1.rows.3.actions.0.opacity', 'surfaces.1.rows.3.description']);
|
||||
const measure = () => page.evaluate(measureMaintenance, '#workspace-main');
|
||||
const paths = async () => diffInventory(expected, await measure()).map(item => item.path);
|
||||
// A removed row, a changed label, a lost danger mark and a swapped token.
|
||||
const undoRow = await mutate(page, { remove: ['#workspace-main section.surface:nth-of-type(1) > .operation:nth-of-type(2)'] });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['surfaces.0.rows.length', 'surfaces.0.tree.length']));
|
||||
await undoRow();
|
||||
await page.evaluate(() => { document.querySelector('#workspace-main section.surface h3').textContent = 'Integrity checks'; });
|
||||
expect(await paths()).toEqual(['surfaces.0.rows.0.title']);
|
||||
await page.evaluate(() => { document.querySelector('#workspace-main section.surface h3').textContent = 'Integrity & relationship checks'; });
|
||||
await page.evaluate(() => { document.querySelector('#workspace-main button.danger').classList.remove('danger'); });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['surfaces.0.rows.3.actions.0.className']));
|
||||
await page.evaluate(() => { document.querySelectorAll('#workspace-main section.surface')[0].querySelectorAll('.row-actions > button')[3].classList.add('danger'); });
|
||||
const undoToken = await mutate(page, { css: '#workspace-main .settings-layout { --line: rgb(255, 0, 0); }' });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['operation.borderTopColor']));
|
||||
await undoToken();
|
||||
expect(diffInventory(expected, await measure())).toEqual([]);
|
||||
|
||||
// The plan dialog: a changed grid term and the duplicates dialog's grid token are noticed.
|
||||
await expect(await request.post('/__test__/system/duplicate-prompts')).toBeOK();
|
||||
const conceptPlanDialog = await conceptPlan(page, 'dark', 'duplicate prompts');
|
||||
await productionMaintenance(page, 'dark');
|
||||
const { plan } = await productionPlan(page, 'Duplicate prompt text');
|
||||
const expectedPlan = mapPlanConcept(conceptPlanDialog, 'Duplicate prompt text', plan);
|
||||
const planPaths = async () => diffInventory(expectedPlan, await page.evaluate(measureReview)).map(item => item.path);
|
||||
expect(await planPaths()).toEqual([]);
|
||||
await page.evaluate(() => { document.querySelector('dialog#modal[open] dl.meta-grid dt').textContent = 'Impact'; });
|
||||
expect(await planPaths()).toEqual(['grid.rows.0.0']);
|
||||
await page.evaluate(() => { document.querySelector('dialog#modal[open] dl.meta-grid dt').textContent = 'Scope'; });
|
||||
const undoGrid = await mutate(page, { css: 'dialog#modal dl.meta-grid { column-gap: 3px; }' });
|
||||
expect(await planPaths()).toEqual(expect.arrayContaining(['grid.columnGap']));
|
||||
await undoGrid();
|
||||
expect(await planPaths()).toEqual([]);
|
||||
});
|
||||
|
||||
test('duplicates comparison detects mutations', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
await expect(await request.post('/__test__/system/duplicate-files', { data: { groups: 1, trio: false } })).toBeOK();
|
||||
const expected = mapDuplicatesConcept(await conceptDuplicates(page, 'dark'));
|
||||
await productionDuplicates(page, 'dark');
|
||||
const paths = async () => diffInventory(expected, await page.evaluate(measureReview)).map(item => item.path);
|
||||
expect(await paths()).toEqual([]);
|
||||
const undoToken = await mutate(page, { css: 'dialog#modal .compare-item { background: rgb(255, 0, 0); }' });
|
||||
expect(await paths()).toEqual(['compare.item.backgroundColor']);
|
||||
await undoToken();
|
||||
await page.evaluate(() => { document.querySelector('dialog#modal[open] .dialog-actions button:last-child').classList.add('ca-primary'); });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['actions.buttons.1.className']));
|
||||
await page.evaluate(() => { document.querySelector('dialog#modal[open] .dialog-actions button:last-child').classList.remove('ca-primary'); });
|
||||
const undoItem = await mutate(page, { remove: ['dialog#modal[open] .compare-item:last-child'] });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['compare.items.length']));
|
||||
await undoItem();
|
||||
expect(await paths()).toEqual([]);
|
||||
});
|
||||
@@ -99,6 +99,34 @@ for (const leave of ['beforeunload', 'pagehide']) {
|
||||
});
|
||||
}
|
||||
|
||||
// #222: the gallery's Clear cache clears the server's thumbnails by its own request and keeps this browser's
|
||||
// preferences and other tools' storage; nothing of Prompt Manager's is cached in a browser, and the toast says so.
|
||||
test('the gallery cache clear keeps preferences and other tools caches', async ({ page }) => {
|
||||
await page.addInitScript(() => {
|
||||
if (sessionStorage.getItem('seeded')) return;
|
||||
sessionStorage.setItem('seeded', '1');
|
||||
localStorage.setItem('pm_workspace_theme', 'light');
|
||||
localStorage.setItem('other-tool-setting', 'keep');
|
||||
localStorage.setItem('pm_other_tool', 'keep');
|
||||
});
|
||||
await page.goto('/prompt_manager/gallery');
|
||||
await expect(page.locator('#galleryGrid .image-item')).toHaveCount(2);
|
||||
await page.evaluate(async () => { await (await caches.open('other-tool-gallery-thumbnails')).put('/x', new Response('x')); });
|
||||
const clears = [];
|
||||
page.on('request', req => { if (req.url().includes('/clear-thumbnails')) clears.push(req.method()); });
|
||||
page.once('dialog', dialog => dialog.accept());
|
||||
await page.locator('#settingsBtn').click();
|
||||
await page.locator('#clearCacheBtn').click();
|
||||
await expect(page.getByText("Thumbnails cleared; original images untouched. Nothing of Prompt Manager's is cached in this browser."))
|
||||
.toBeVisible();
|
||||
expect(clears).toEqual(['POST']);
|
||||
expect(await page.evaluate(async () => ({
|
||||
theme: localStorage.getItem('pm_workspace_theme'), other: localStorage.getItem('other-tool-setting'),
|
||||
prefixed: localStorage.getItem('pm_other_tool'), notice: localStorage.getItem('pm_update_notice_dismissed'),
|
||||
caches: await caches.keys(),
|
||||
}))).toEqual({ theme: 'light', other: 'keep', prefixed: 'keep', notice: '3.2.0', caches: ['other-tool-gallery-thumbnails'] });
|
||||
});
|
||||
|
||||
test('actual gallery review generates and persists tags for the selected root', async ({ page, request }) => {
|
||||
await page.goto('/prompt_manager/gallery');
|
||||
await expect(page.locator('#galleryGrid .image-item')).toHaveCount(2);
|
||||
|
||||
@@ -2184,6 +2184,8 @@ test('scan for metadata opens the review dialog from Import', async ({ page, req
|
||||
await dialog.getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(scanReview(page)).toHaveCount(0);
|
||||
await expect(page.getByRole('button', { name: 'Import', exact: true })).toBeFocused();
|
||||
const jobs = await (await request.get('/prompt_manager/jobs')).json();
|
||||
expect(jobs.jobs).toEqual([]);
|
||||
// No scan started: the one job is the backup's finished registry hold
|
||||
// (#249 S8: a backup download holds a guarded slot while it streams).
|
||||
await expect.poll(async () => (await (await request.get('/prompt_manager/jobs')).json()).jobs
|
||||
.map(job => [job.kind, job.state])).toEqual([['backup', 'finished']]);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,489 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { seedConcept } from './concept-inventory.mjs';
|
||||
|
||||
// System › Backup & export (#249 System S9 Task 24) on the S8 backend and the concept-shaped fixture: Back up and its
|
||||
// status line (D103), the restore review (D105) and its job, the exports, Import / inspect and the recovery policy
|
||||
// (D104). A restore here replaces this worker's own temp database; every test starts from a fresh concept fixture.
|
||||
const workspace = '/prompt_manager/workspace';
|
||||
const STATUS_URL = '/prompt_manager/backup/status';
|
||||
const PREVIEW_URL = '/prompt_manager/restore/preview';
|
||||
const APPLY_URL = '/prompt_manager/restore/apply';
|
||||
const WHEN = { dateStyle: 'medium', timeStyle: 'short' };
|
||||
const SAFETY_DESCRIPTION = 'Keep a verified copy of the database on disk before maintenance, cleanups and LoRA imports change ' +
|
||||
'records. A restore always keeps one.';
|
||||
|
||||
test.use({ locale: 'en-US', timezoneId: 'UTC' });
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
await seedConcept(request);
|
||||
await expect(await request.post('/__test__/system/reset')).toBeOK();
|
||||
await expect(await request.post('/__test__/jobs/restart')).toBeOK();
|
||||
page.systemErrors = [];
|
||||
page.on('pageerror', error => page.systemErrors.push(error.message));
|
||||
});
|
||||
|
||||
test.afterEach(async ({ page, request, browserName }) => {
|
||||
await saveCoverage(page, browserName);
|
||||
await request.post('/__test__/release');
|
||||
await request.post('/__test__/jobs/restart');
|
||||
await request.post('/__test__/system/reset');
|
||||
expect(page.systemErrors).toEqual([]);
|
||||
});
|
||||
|
||||
const toast = page => page.locator('#workspace-toast');
|
||||
const layoutOf = page => page.locator('#workspace-main .settings-layout');
|
||||
const dialog = page => page.locator('dialog#modal[open]');
|
||||
const statusLine = page => layoutOf(page).locator('#backup-status');
|
||||
const settingsStatus = page => page.locator('#settings-status');
|
||||
const save = page => page.getByRole('button', { name: 'Save preferences', exact: true });
|
||||
const buttonOf = (page, name) => layoutOf(page).getByRole('button', { name, exact: true });
|
||||
const settingOf = (page, label) => layoutOf(page).locator('.setting-row').filter({ has: page.locator(`label:text-is("${label}")`) });
|
||||
|
||||
async function serverJson(request, url) {
|
||||
const response = await request.get(url);
|
||||
await expect(response).toBeOK();
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async function openBackups(page) {
|
||||
// A fresh page each time: the drafts live for the page session (Q10), and a hash-only goto would not reload.
|
||||
if (page.url().endsWith('#/system/backups')) await page.reload();
|
||||
else await page.goto(`${workspace}#/system/backups`);
|
||||
await expect(layoutOf(page).locator(':scope > .two-col > section.surface')).toHaveCount(2);
|
||||
await expect(statusLine(page)).not.toHaveText('Checking backups…');
|
||||
// The policy row is painted once the server settings were read.
|
||||
await expect(settingOf(page, 'Create a safety backup before maintenance').locator('input')).toBeEnabled();
|
||||
return layoutOf(page);
|
||||
}
|
||||
|
||||
/** The status line the page should show for the server's current status (its own wording, in this locale). */
|
||||
async function expectedStatus(page, request) {
|
||||
const status = await serverJson(request, STATUS_URL);
|
||||
return page.evaluate(([answer, options]) => {
|
||||
const when = value => new Date(value).toLocaleString(undefined, options);
|
||||
const parts = [];
|
||||
if (answer.last_download) parts.push(`Downloaded backup · ${when(answer.last_download.at)} · ${answer.last_download.verified ? 'verified' : 'not verified'}.`);
|
||||
if (answer.safety_backups.length) parts.push(`Safety backup · ${when(answer.safety_backups[0].modified_at)} · ${answer.safety_backups.length} on disk.`);
|
||||
return parts.join(' ') || 'No backup recorded.';
|
||||
}, [status, WHEN]);
|
||||
}
|
||||
|
||||
/** A verified copy of the library as it is now (this records the session's last download, as Back up does). */
|
||||
async function backupBytes(request) {
|
||||
const response = await request.get('/prompt_manager/backup');
|
||||
await expect(response).toBeOK();
|
||||
expect(response.headers()['x-promptmanager-verify']).toBe('ok');
|
||||
return response.body();
|
||||
}
|
||||
|
||||
/** Pick `buffer` as the backup file through Review restore…'s file chooser; resolves with the preview's answer. */
|
||||
async function chooseBackup(page, buffer, name = 'prompts_backup_20261025_093000.db') {
|
||||
const chooser = page.waitForEvent('filechooser');
|
||||
const answer = page.waitForResponse(response => response.url().endsWith(PREVIEW_URL) && response.request().method() === 'POST');
|
||||
await buttonOf(page, 'Review restore…').click();
|
||||
await (await chooser).setFiles({ name, mimeType: 'application/octet-stream', buffer });
|
||||
return answer;
|
||||
}
|
||||
|
||||
const gridOf = page => dialog(page).locator('dl.meta-grid > div').evaluateAll(items => items.map(item =>
|
||||
[item.querySelector('dt').textContent, item.querySelector('dd').textContent]));
|
||||
|
||||
const libraryTotal = async request => (await serverJson(request, '/prompt_manager/library?limit=1')).total;
|
||||
|
||||
test('Back up downloads through the api client and revokes the object URL (D103)', async ({ page, request }) => {
|
||||
await page.addInitScript(() => {
|
||||
window.objectUrls = { created: [], revoked: [] };
|
||||
const create = URL.createObjectURL.bind(URL), revoke = URL.revokeObjectURL.bind(URL);
|
||||
URL.createObjectURL = blob => { const url = create(blob); window.objectUrls.created.push(url); return url; };
|
||||
URL.revokeObjectURL = url => { window.objectUrls.revoked.push(url); revoke(url); };
|
||||
});
|
||||
await openBackups(page);
|
||||
await expect(statusLine(page)).toHaveText('No backup recorded.');
|
||||
const download = page.waitForEvent('download');
|
||||
const answer = page.waitForResponse(response => new URL(response.url()).pathname === '/prompt_manager/backup');
|
||||
await buttonOf(page, 'Back up').click();
|
||||
expect((await answer).headers()['x-promptmanager-verify']).toBe('ok');
|
||||
expect((await download).suggestedFilename()).toMatch(/^prompts_backup_\d{8}_\d{6}\.db$/);
|
||||
await expect(toast(page)).toHaveText('Database backup downloaded.');
|
||||
// The status line is read again: this session's verified download, as the server records it.
|
||||
await expect(statusLine(page)).toHaveText(/^Downloaded backup · .+ · verified\.$/);
|
||||
await expect(statusLine(page)).toHaveText(await expectedStatus(page, request));
|
||||
await expect(buttonOf(page, 'Back up')).not.toHaveAttribute('aria-disabled', 'true');
|
||||
const urls = await page.evaluate(() => window.objectUrls.created);
|
||||
expect(urls).toHaveLength(1);
|
||||
await expect.poll(() => page.evaluate(() => window.objectUrls.revoked)).toEqual(urls);
|
||||
});
|
||||
|
||||
test('status line shows only server-reported backups (D103)', async ({ page, request }) => {
|
||||
await openBackups(page);
|
||||
await expect(statusLine(page)).toHaveText('No backup recorded.');
|
||||
await expect(await request.post('/__test__/system/safety-backup')).toBeOK();
|
||||
await openBackups(page);
|
||||
await expect(statusLine(page)).toHaveText(/^Safety backup · .+ · 1 on disk\.$/);
|
||||
await expect(statusLine(page)).toHaveText(await expectedStatus(page, request));
|
||||
// A status the server could not report is never shown as "no backup".
|
||||
await page.route(`**${STATUS_URL}`, route => route.fulfill({ status: 500, json: { success: false, error: 'Failed' } }));
|
||||
await page.reload();
|
||||
await expect(statusLine(page)).toHaveText('The backup status could not be read.');
|
||||
});
|
||||
|
||||
test('Overview backup row uses the same status (D93)', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/system/safety-backup')).toBeOK();
|
||||
await backupBytes(request);
|
||||
await page.goto(`${workspace}#/system/overview`);
|
||||
const row = layoutOf(page).locator('.operation').filter({ has: page.locator('h3', { hasText: 'Backup status' }) });
|
||||
const expected = await expectedStatus(page, request);
|
||||
expect(expected).toMatch(/^Downloaded backup · .+ · verified\. Safety backup · .+ · 1 on disk\.$/);
|
||||
await expect(row.locator('p')).toHaveText(expected);
|
||||
await row.getByRole('button', { name: 'View backups', exact: true }).click();
|
||||
await expect(page).toHaveURL(`${workspace}#/system/backups`);
|
||||
await expect(statusLine(page)).toHaveText(expected);
|
||||
});
|
||||
|
||||
test('Review restore stages the chosen file and shows the real summary with the backup box (D105)', async ({ page, request }) => {
|
||||
const bytes = await backupBytes(request);
|
||||
await expect(await request.post('/prompt_manager/save', { data: { text: 'S9 prompt saved after the backup' } })).toBeOK();
|
||||
const before = await serverJson(request, '/__test__/system/digest');
|
||||
await openBackups(page);
|
||||
const answer = await chooseBackup(page, bytes);
|
||||
const preview = await answer.json();
|
||||
expect(preview.integrity).toBe('ok');
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Restore preview');
|
||||
const counts = summary => `${summary.prompts.toLocaleString('en-US')} ${summary.prompts === 1 ? 'prompt' : 'prompts'} · ` +
|
||||
`${summary.image_links} ${summary.image_links === 1 ? 'image link' : 'image links'} · ` +
|
||||
`${summary.pairs} ${summary.pairs === 1 ? 'saved pair' : 'saved pairs'} · ${summary.runs} ${summary.runs === 1 ? 'run' : 'runs'}`;
|
||||
const rows = await gridOf(page);
|
||||
expect(rows.map(([term]) => term)).toEqual(['Snapshot', 'Integrity', 'Migration', 'Safety backup', 'In the backup', 'In the library now']);
|
||||
expect(rows[0][1]).toMatch(/^prompts_backup_20261025_093000.db · (\d+ KB|\d+\.\d MB) · .+$/);
|
||||
expect(rows.slice(1)).toEqual([['Integrity', 'Verified'], ['Migration', 'Current schema'],
|
||||
['Safety backup', `Created before replacing (${preview.safety_backup.destination_label})`],
|
||||
['In the backup', counts(preview.summary)], ['In the library now', counts(preview.live)]]);
|
||||
// The prompt saved after the backup is the difference the review shows.
|
||||
expect(preview.live.prompts).toBe(preview.summary.prompts + 1);
|
||||
await expect(dialog(page).locator('#modal-body > p.small')).toHaveText('Restore replaces the whole library with this backup ' +
|
||||
'and runs alone in Activity. Nothing changes until you restore. This review holds for 15 minutes.');
|
||||
await expect(dialog(page).locator('.notice h3')).toHaveText('Back up the database first (recommended)');
|
||||
await expect(dialog(page).locator('.notice p')).toHaveText('Downloads a copy of the prompts database before the restore replaces it.');
|
||||
const actions = dialog(page).locator('.dialog-actions > button');
|
||||
await expect(actions).toHaveText(['Cancel', 'Restore']);
|
||||
await expect(actions.nth(1)).toHaveClass('danger');
|
||||
// The backup box downloads a copy and keeps the review open.
|
||||
const download = page.waitForEvent('download');
|
||||
await dialog(page).getByRole('button', { name: 'Back up now', exact: true }).click();
|
||||
await download;
|
||||
await expect(toast(page)).toHaveText('Database backup downloaded.');
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Restore preview');
|
||||
// A review writes nothing: the rows and the media are as they were.
|
||||
expect(await serverJson(request, '/__test__/system/digest')).toEqual(before);
|
||||
expect((await serverJson(request, '/__test__/system/backups')).stages).toBe(1);
|
||||
});
|
||||
|
||||
test('Cancel discards the stage; Escape too', async ({ page, request }) => {
|
||||
const bytes = await backupBytes(request);
|
||||
const before = await serverJson(request, '/__test__/system/digest');
|
||||
await openBackups(page);
|
||||
await chooseBackup(page, bytes);
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Restore preview');
|
||||
expect((await serverJson(request, '/__test__/system/backups')).stages).toBe(1);
|
||||
let discarded = page.waitForResponse(response => response.url().includes(`${PREVIEW_URL}/`) && response.request().method() === 'DELETE');
|
||||
await dialog(page).getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
expect((await discarded).status()).toBe(200);
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
await expect(buttonOf(page, 'Review restore…')).toBeFocused();
|
||||
expect((await serverJson(request, '/__test__/system/backups')).stages).toBe(0);
|
||||
await chooseBackup(page, bytes);
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Restore preview');
|
||||
discarded = page.waitForResponse(response => response.url().includes(`${PREVIEW_URL}/`) && response.request().method() === 'DELETE');
|
||||
await page.keyboard.press('Escape');
|
||||
expect((await discarded).status()).toBe(200);
|
||||
expect((await serverJson(request, '/__test__/system/backups')).stages).toBe(0);
|
||||
expect(await serverJson(request, '/__test__/system/digest')).toEqual(before);
|
||||
});
|
||||
|
||||
test('Restore starts the job, opens Activity; views reload after it finishes', async ({ page, request }) => {
|
||||
const bytes = await backupBytes(request);
|
||||
const restoredTotal = await libraryTotal(request);
|
||||
await expect(await request.post('/prompt_manager/save', { data: { text: 'S9 prompt the restore removes' } })).toBeOK();
|
||||
expect(await libraryTotal(request)).toBe(restoredTotal + 1);
|
||||
await openBackups(page);
|
||||
const libraryCount = page.locator('aside.sidebar a[data-route="library"] .nav-count');
|
||||
await expect(libraryCount).toHaveText((restoredTotal + 1).toLocaleString('en-US'));
|
||||
await chooseBackup(page, bytes);
|
||||
const applied = page.waitForResponse(response => response.url().endsWith(APPLY_URL));
|
||||
await dialog(page).getByRole('button', { name: 'Restore', exact: true }).click();
|
||||
const { job } = await (await applied).json();
|
||||
expect([job.kind, job.title, job.cancellable]).toEqual(['restore', 'Database restore', false]);
|
||||
await expect(page).toHaveURL(`${workspace}#/activity/${job.id}`);
|
||||
await expect(toast(page)).toHaveText(/^Database restore finished · Restored \d+ prompts?, /);
|
||||
// The library is the backup's again, and the page says so without a reload: the sidebar counts were read again.
|
||||
expect(await libraryTotal(request)).toBe(restoredTotal);
|
||||
await expect(libraryCount).toHaveText(restoredTotal.toLocaleString('en-US'));
|
||||
expect((await serverJson(request, '/__test__/system/backups')).safety).toHaveLength(1);
|
||||
// The Activity job view stayed, and Backup & export now names the safety backup the restore kept.
|
||||
await expect(page).toHaveURL(`${workspace}#/activity/${job.id}`);
|
||||
await page.goto(`${workspace}#/system/backups`);
|
||||
await expect(statusLine(page)).toHaveText(/Safety backup · .+ · 1 on disk\.$/);
|
||||
});
|
||||
|
||||
test('a finished restore shows the open page again from the restored data', async ({ page, request }) => {
|
||||
const bytes = await backupBytes(request);
|
||||
const restoredTotal = await libraryTotal(request);
|
||||
await expect(await request.post('/prompt_manager/save', { data: { text: 'S9 prompt the restore removes' } })).toBeOK();
|
||||
await openBackups(page);
|
||||
await chooseBackup(page, bytes);
|
||||
const preview = page.locator('dialog#modal[open] #modal-title');
|
||||
await expect(preview).toHaveText('Restore preview');
|
||||
// The restore is held at its start, so the reader is on the Library (showing the prompt) when it finishes.
|
||||
await expect(await request.post('/__test__/hold', { data: { path: APPLY_URL } })).toBeOK();
|
||||
const applied = page.waitForResponse(response => response.url().endsWith(APPLY_URL));
|
||||
await dialog(page).getByRole('button', { name: 'Restore', exact: true }).click();
|
||||
await expect.poll(async () => (await serverJson(request, '/__test__/hold')).held[APPLY_URL]).toBe(1);
|
||||
await page.evaluate(() => { window.location.hash = '#/library'; });
|
||||
const card = page.locator('#workspace-main article.prompt-card').filter({ hasText: 'S9 prompt the restore removes' });
|
||||
await expect(card).toHaveCount(1);
|
||||
await expect(await request.post('/__test__/release')).toBeOK();
|
||||
await applied;
|
||||
await expect(toast(page)).toHaveText(/^Database restore finished · /);
|
||||
// The shelf was read again from the restored library, on the same route.
|
||||
await expect(card).toHaveCount(0);
|
||||
await expect(page).toHaveURL(`${workspace}#/library`);
|
||||
expect(await libraryTotal(request)).toBe(restoredTotal);
|
||||
});
|
||||
|
||||
test('invalid file shows the validator message; nothing is staged', async ({ page, request }) => {
|
||||
const before = await serverJson(request, '/__test__/system/digest');
|
||||
await openBackups(page);
|
||||
const answer = await chooseBackup(page, Buffer.from('this is not a database'), 'notes.db');
|
||||
expect(answer.status()).toBe(400);
|
||||
const body = await answer.json();
|
||||
expect(body.code).toBe('invalid-backup');
|
||||
await expect(toast(page)).toHaveText(`This file can't be restored: ${body.error.replace(/\.?$/, '.')}`);
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
await expect(buttonOf(page, 'Review restore…')).not.toHaveAttribute('aria-disabled', 'true');
|
||||
expect((await serverJson(request, '/__test__/system/backups')).stages).toBe(0);
|
||||
expect(await serverJson(request, '/__test__/system/digest')).toEqual(before);
|
||||
});
|
||||
|
||||
test('Export JSON downloads /library/export; secondary buttons download all prompts (/export) and /pairs/export (D103)', async ({ page }) => {
|
||||
const layout = await openBackups(page);
|
||||
const portable = layout.locator('section.surface').filter({ has: page.locator('h2', { hasText: 'Portable export' }) });
|
||||
expect(await portable.locator('.row-actions > button').allTextContents())
|
||||
.toEqual(['Export JSON', 'Import / inspect file', 'Export all prompts JSON', 'Export saved pairs JSON']);
|
||||
for (const [label, url, name, kind] of [
|
||||
['Export JSON', '/prompt_manager/library/export', /^prompt-manager-library-v1\.json$/, 'prompt-manager-library'],
|
||||
['Export all prompts JSON', '/prompt_manager/export', /^prompt_manager_\d{8}_\d{6}\.json$/, null],
|
||||
['Export saved pairs JSON', '/prompt_manager/pairs/export', /^prompt-pairs\.json$/, 'prompt-manager-pairs']]) {
|
||||
const download = page.waitForEvent('download');
|
||||
const answer = page.waitForResponse(response => new URL(response.url()).pathname === url);
|
||||
await portable.getByRole('button', { name: label, exact: true }).click();
|
||||
expect((await answer).status()).toBe(200);
|
||||
const file = await download;
|
||||
expect(file.suggestedFilename()).toMatch(name);
|
||||
const parsed = JSON.parse(await (await file.createReadStream()).toArray().then(chunks => Buffer.concat(chunks).toString('utf8')));
|
||||
if (kind) expect(parsed.kind).toBe(kind);
|
||||
else expect(Array.isArray(parsed.prompts)).toBe(true);
|
||||
}
|
||||
// A failed export says so and saves nothing.
|
||||
await page.route('**/prompt_manager/pairs/export', route => route.fulfill({ status: 500, json: { success: false } }));
|
||||
await portable.getByRole('button', { name: 'Export saved pairs JSON', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('The export could not be downloaded. Nothing was saved to this device.');
|
||||
});
|
||||
|
||||
test('Import / inspect lists Inspect, Rescan and the shared scan review (D15, D55)', async ({ page, request }) => {
|
||||
await openBackups(page);
|
||||
const open = async () => {
|
||||
await buttonOf(page, 'Import / inspect file').click();
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Import & inspect');
|
||||
};
|
||||
await open();
|
||||
const rows = dialog(page).locator('.operation-list > .operation');
|
||||
expect(await rows.locator('h3').allTextContents()).toEqual(['Inspect a local image', 'Scan an output folder', 'Scan for metadata']);
|
||||
expect(await rows.locator('button').allTextContents()).toEqual(['Open inspector', 'Rescan', 'Review scan']);
|
||||
// Rescan reads the output folders again and says what it found; while it runs, a reopened dialog says so.
|
||||
await expect(await request.post('/__test__/hold', { data: { path: '/prompt_manager/gallery/rescan' } })).toBeOK();
|
||||
const rescan = page.waitForResponse(response => response.url().endsWith('/prompt_manager/gallery/rescan'));
|
||||
await rows.getByRole('button', { name: 'Rescan', exact: true }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
await expect(toast(page)).toHaveText('Rescanning output folders…');
|
||||
await open();
|
||||
await expect(rows.nth(1).locator('button')).toHaveText('Rescanning…');
|
||||
await expect(rows.nth(1).locator('button')).toHaveAttribute('aria-disabled', 'true');
|
||||
await expect(await request.post('/__test__/release')).toBeOK();
|
||||
expect((await rescan).status()).toBe(200);
|
||||
await expect(toast(page)).toHaveText(/^Rescan finished\. \d+ files? in \d+ output folders?\.$/);
|
||||
await expect(rows.nth(1).locator('button')).toHaveText('Rescan');
|
||||
// Scan for metadata opens the shared review over the server's output folders; Cancel starts nothing.
|
||||
await dialog(page).getByRole('button', { name: 'Review scan', exact: true }).click();
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Review metadata scan');
|
||||
await expect(dialog(page).locator('dl.meta-grid dd').first()).toHaveText(/^\d+ output folders?$/);
|
||||
await dialog(page).getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
// Open inspector goes to Inspect a file.
|
||||
await open();
|
||||
await dialog(page).getByRole('button', { name: 'Open inspector', exact: true }).click();
|
||||
await expect(page).toHaveURL(`${workspace}#/inspect`);
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
});
|
||||
|
||||
test('safety backup toggle saves through Save preferences with revision (D104)', async ({ page, request }) => {
|
||||
const before = await serverJson(request, '/prompt_manager/settings');
|
||||
expect(before.settings.safety_backup).toBe(true);
|
||||
await openBackups(page);
|
||||
const row = settingOf(page, 'Create a safety backup before maintenance');
|
||||
// D104: what the policy really does (the concept's "Preview a verified backup…" line does not say it).
|
||||
await expect(row.locator('p').first()).toHaveText(SAFETY_DESCRIPTION);
|
||||
const toggle = row.locator('input');
|
||||
await expect(toggle).toBeChecked();
|
||||
await toggle.uncheck();
|
||||
await expect(settingsStatus(page)).toHaveText('Unsaved changes.');
|
||||
const saved = page.waitForRequest(item => item.url().endsWith('/prompt_manager/settings') && item.method() === 'POST');
|
||||
await save(page).click();
|
||||
const body = (await saved).postDataJSON();
|
||||
expect(body).toEqual({ safety_backup: false, expected_revision: before.revision });
|
||||
await expect(settingsStatus(page)).toHaveText('Saved to the server configuration.');
|
||||
expect((await serverJson(request, '/prompt_manager/settings')).settings.safety_backup).toBe(false);
|
||||
// A maintenance review now says so.
|
||||
const plan = await request.post('/prompt_manager/maintenance/preview', { data: { operation: 'optimize', options: {} } });
|
||||
expect((await plan.json()).backup.policy).toBe('off');
|
||||
await openBackups(page);
|
||||
await expect(toggle).not.toBeChecked();
|
||||
// A configuration changed elsewhere is a conflict, never a blind write.
|
||||
await toggle.check();
|
||||
await expect(await request.post('/prompt_manager/settings', { data: { result_timeout: 7,
|
||||
expected_revision: (await serverJson(request, '/prompt_manager/settings')).revision } })).toBeOK();
|
||||
await save(page).click();
|
||||
await expect(settingsStatus(page)).toHaveText('Settings changed elsewhere. Reload this section to see them.');
|
||||
expect((await serverJson(request, '/prompt_manager/settings')).settings.safety_backup).toBe(false);
|
||||
});
|
||||
|
||||
test('schedule offers Manual only; Daily/Weekly disabled with reason (D104)', async ({ page }) => {
|
||||
await openBackups(page);
|
||||
const schedule = settingOf(page, 'Scheduled database backups');
|
||||
expect(await schedule.locator('option').evaluateAll(options => options.map(option => [option.textContent, option.disabled])))
|
||||
.toEqual([['Manual only', false], ['Daily', true], ['Weekly', true]]);
|
||||
await expect(schedule.locator('select')).toHaveValue('Manual only');
|
||||
await expect(schedule.locator('p')).toHaveText('Daily and Weekly are not available yet: back up with Back up whenever you choose.');
|
||||
const policy = layoutOf(page).locator(':scope > section.surface');
|
||||
await expect(policy.locator('h2')).toHaveText('Recovery policy Proposed');
|
||||
await expect(policy.locator('h2 > span.chip.blue')).toHaveText('Proposed');
|
||||
await expect(policy.locator('p.warning-note')).toHaveText('Restores are staged, migrated and validated before replacing live data.');
|
||||
// Nothing on the schedule is ever drafted.
|
||||
await expect(settingsStatus(page)).toHaveText('No unsaved changes.');
|
||||
});
|
||||
|
||||
test('a failed settings read leaves the policy disabled with the reason', async ({ page }) => {
|
||||
await page.route('**/prompt_manager/settings', route => (route.request().method() === 'GET'
|
||||
? route.fulfill({ status: 500, json: { success: false, error: 'Failed' } }) : route.continue()));
|
||||
await page.goto(`${workspace}#/system/backups`);
|
||||
const row = settingOf(page, 'Create a safety backup before maintenance');
|
||||
await expect(row.locator('p.small')).toHaveText('Server settings could not be read.');
|
||||
await expect(row.locator('input')).toBeDisabled();
|
||||
});
|
||||
|
||||
test('the legacy admin Backup and Restore keep working', async ({ page, request }) => {
|
||||
const bytes = await backupBytes(request);
|
||||
const restored = await request.post('/prompt_manager/restore', { multipart: { database_file: {
|
||||
name: 'prompts_backup.db', mimeType: 'application/octet-stream', buffer: bytes } } });
|
||||
await expect(restored).toBeOK();
|
||||
expect((await restored.json()).success).toBe(true);
|
||||
await page.goto('/prompt_manager/admin');
|
||||
await expect(page.locator('#restoreFileInput')).toHaveCount(1);
|
||||
});
|
||||
|
||||
/** Start a fake running job (a metadata scan another tab started), which an exclusive restore must wait for. */
|
||||
async function runningScan(request) {
|
||||
const response = await request.post('/__test__/jobs/start', { data: { kind: 'scan', title: 'Metadata scan', total: 4 } });
|
||||
await expect(response).toBeOK();
|
||||
return (await response.json()).job;
|
||||
}
|
||||
|
||||
const stagesOf = async request => (await serverJson(request, '/__test__/system/backups')).stages;
|
||||
const deleted = page => page.waitForResponse(response => response.url().includes(`${PREVIEW_URL}/`) &&
|
||||
response.request().method() === 'DELETE');
|
||||
|
||||
test('a refused restore reopens its review: Restore retries it, Discard releases it (I1)', async ({ page, request }) => {
|
||||
const bytes = await backupBytes(request);
|
||||
await openBackups(page);
|
||||
await chooseBackup(page, bytes);
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Restore preview');
|
||||
const scan = await runningScan(request);
|
||||
const refused = page.waitForResponse(response => response.url().endsWith(APPLY_URL));
|
||||
await dialog(page).getByRole('button', { name: 'Restore', exact: true }).click();
|
||||
expect((await refused).status()).toBe(409);
|
||||
await expect(toast(page)).toHaveText('Another job is running: Metadata scan.');
|
||||
// The same review is open again, its stage still held: Restore retries, Discard releases it.
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Restore preview');
|
||||
await expect(dialog(page).locator('.dialog-actions > button')).toHaveText(['Discard', 'Restore']);
|
||||
expect(await stagesOf(request)).toBe(1);
|
||||
await expect(await request.post(`/__test__/jobs/${scan.id}/finish`, { data: { state: 'finished' } })).toBeOK();
|
||||
const applied = page.waitForResponse(response => response.url().endsWith(APPLY_URL));
|
||||
await dialog(page).getByRole('button', { name: 'Restore', exact: true }).click();
|
||||
const { job } = await (await applied).json();
|
||||
await expect(page).toHaveURL(`${workspace}#/activity/${job.id}`);
|
||||
await expect(toast(page)).toHaveText(/^Database restore finished · /);
|
||||
expect(await stagesOf(request)).toBe(0);
|
||||
});
|
||||
|
||||
test('two refused restores in a row, then a third Review restore… still opens the preview (I1)', async ({ page, request }) => {
|
||||
const bytes = await backupBytes(request);
|
||||
await openBackups(page);
|
||||
await runningScan(request);
|
||||
for (const leave of ['discard', 'escape']) {
|
||||
await chooseBackup(page, bytes);
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Restore preview');
|
||||
const refused = page.waitForResponse(response => response.url().endsWith(APPLY_URL));
|
||||
await dialog(page).getByRole('button', { name: 'Restore', exact: true }).click();
|
||||
expect((await refused).status()).toBe(409);
|
||||
await expect(dialog(page).locator('.dialog-actions > button')).toHaveText(['Discard', 'Restore']);
|
||||
const released = deleted(page);
|
||||
if (leave === 'discard') await dialog(page).getByRole('button', { name: 'Discard', exact: true }).click();
|
||||
else await page.keyboard.press('Escape');
|
||||
expect((await released).status()).toBe(200);
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
expect(await stagesOf(request)).toBe(0);
|
||||
}
|
||||
const answer = await chooseBackup(page, bytes);
|
||||
expect((await answer).status()).toBe(200);
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Restore preview');
|
||||
expect(await stagesOf(request)).toBe(1);
|
||||
});
|
||||
|
||||
test('× and leaving the section or the page discard the stage (M1)', async ({ page, request }) => {
|
||||
const bytes = await backupBytes(request);
|
||||
await openBackups(page);
|
||||
const leaves = [
|
||||
['×', () => page.locator('#modal-close').click()],
|
||||
// The modal covers the page, so the section changes as a link or the Back button changes it: by its hash.
|
||||
['section', () => page.evaluate(() => { window.location.hash = '#/system/maintenance'; })],
|
||||
['route', () => page.evaluate(() => { window.location.hash = '#/images'; })],
|
||||
];
|
||||
for (const [name, leave] of leaves) {
|
||||
if (!page.url().endsWith('#/system/backups')) {
|
||||
await page.goto(`${workspace}#/system/backups`);
|
||||
await expect(statusLine(page)).not.toHaveText('Checking backups…');
|
||||
}
|
||||
await chooseBackup(page, bytes);
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Restore preview');
|
||||
expect(await stagesOf(request), name).toBe(1);
|
||||
// A keepalive DELETE (it may outlive the page) is asserted by its request and the server's stage count.
|
||||
const released = page.waitForRequest(item => item.url().includes(`${PREVIEW_URL}/`) && item.method() === 'DELETE');
|
||||
await leave();
|
||||
await released;
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
await expect.poll(() => stagesOf(request), { message: name }).toBe(0);
|
||||
}
|
||||
});
|
||||
|
||||
test('a safety backup file (no .db extension) can be picked and reviewed (M3)', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/system/safety-backup')).toBeOK();
|
||||
const bytes = await backupBytes(request);
|
||||
await openBackups(page);
|
||||
// The picker lists every file; the server decides what is a database.
|
||||
await expect(page.locator('#restore-file')).not.toHaveAttribute('accept', /./);
|
||||
const answer = await chooseBackup(page, bytes, 'prompts.db.backup_20261025_093000_000000');
|
||||
expect((await answer).status()).toBe(200);
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Restore preview');
|
||||
expect((await gridOf(page))[0][1]).toMatch(/^prompts\.db\.backup_20261025_093000_000000 · \d+ KB · .+$/);
|
||||
});
|
||||
@@ -0,0 +1,432 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { seedConcept } from './concept-inventory.mjs';
|
||||
|
||||
// System › Integrations (#249 System S10 Task 27) on the concept-shaped fixture with a fake LoraManager install
|
||||
// (workspace_integrations_fixture.py): the D106 ComfyUI rows and renderer status, the LoRA rows and the D114 key, saved
|
||||
// through the one Save row, and the D107 import review run as an Activity job. No test reaches the network: CivitAI
|
||||
// downloads are answered by the fixture's stub opener.
|
||||
const workspace = '/prompt_manager/workspace';
|
||||
const PREVIEW_URL = '/prompt_manager/lora/import/preview';
|
||||
const APPLY_URL = '/prompt_manager/lora/import/apply';
|
||||
const ENABLE_URL = '/prompt_manager/lora/enable';
|
||||
const KEPT = 'The import failed; the previous LoRA imports were kept. Images already downloaded stay in the image cache.';
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
await seedConcept(request);
|
||||
await expect(await request.post('/__test__/system/reset')).toBeOK();
|
||||
await expect(await request.post('/__test__/integrations/reset')).toBeOK();
|
||||
await expect(await request.post('/__test__/jobs/restart')).toBeOK();
|
||||
page.systemErrors = [];
|
||||
page.on('pageerror', error => page.systemErrors.push(error.message));
|
||||
});
|
||||
|
||||
test.afterEach(async ({ page, request, browserName }) => {
|
||||
await saveCoverage(page, browserName);
|
||||
await request.post('/__test__/integrations/gate', { data: { hold: false } });
|
||||
await request.post('/__test__/release');
|
||||
await request.post('/__test__/jobs/restart');
|
||||
await request.post('/__test__/integrations/reset');
|
||||
await request.post('/__test__/system/reset');
|
||||
expect(page.systemErrors).toEqual([]);
|
||||
});
|
||||
|
||||
const toast = page => page.locator('#workspace-toast');
|
||||
const layoutOf = page => page.locator('#workspace-main .settings-layout');
|
||||
const dialog = page => page.locator('dialog#modal[open]');
|
||||
const status = page => page.locator('#settings-status');
|
||||
const save = page => page.getByRole('button', { name: 'Save preferences', exact: true });
|
||||
const settingOf = (page, label) => layoutOf(page).locator('.setting-row').filter({ has: page.locator(`label:text-is("${label}")`) });
|
||||
|
||||
async function serverJson(request, url) {
|
||||
const response = await request.get(url);
|
||||
await expect(response).toBeOK();
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async function lora(request, data = {}) {
|
||||
await expect(await request.post('/__test__/integrations/lora-manager', { data })).toBeOK();
|
||||
}
|
||||
|
||||
async function openIntegrations(page) {
|
||||
// A fresh page each time: the drafts live for the page session (Q10), and a hash-only goto would not reload.
|
||||
if (page.url().endsWith('#/system/integrations')) await page.reload();
|
||||
else await page.goto(`${workspace}#/system/integrations`);
|
||||
await expect(layoutOf(page).locator(':scope > section.surface')).toHaveCount(2);
|
||||
// The LoRA rows are painted once the server's LoRA state was read.
|
||||
await expect(settingOf(page, 'LoraManager integration').locator('input')).toBeEnabled();
|
||||
// The renderer line settles first: its new text moves the rows below, so no click lands mid-shift.
|
||||
await expect(settingOf(page, 'Host compatibility').getByRole('status')).not.toHaveText('Looking for open ComfyUI windows…');
|
||||
return layoutOf(page);
|
||||
}
|
||||
|
||||
/** Enable the integration (and optionally its trigger words) through the Save row. */
|
||||
async function enable(page, { triggers = false } = {}) {
|
||||
await settingOf(page, 'LoraManager integration').locator('input').check();
|
||||
if (triggers) await settingOf(page, 'Automatically include trigger words').locator('input').check();
|
||||
await save(page).click();
|
||||
await expect(status(page)).toHaveText('Saved to the server configuration.');
|
||||
}
|
||||
|
||||
/** The open review's `[dt, dd]` rows. */
|
||||
const gridOf = page => dialog(page).locator('dl.meta-grid > div').evaluateAll(items => items.map(item =>
|
||||
[item.querySelector('dt').textContent, item.querySelector('dd').textContent]));
|
||||
|
||||
async function ended(request, id) {
|
||||
await expect.poll(async () => (await serverJson(request, `/prompt_manager/jobs/${id}`)).job.state).not.toBe('running');
|
||||
return (await serverJson(request, `/prompt_manager/jobs/${id}`)).job;
|
||||
}
|
||||
|
||||
/** A second page of this origin that serves a fake ComfyUI window on the send bridge. */
|
||||
async function fakeComfyWindow(context, renderer) {
|
||||
const host = await context.newPage();
|
||||
await host.goto(`${workspace}#/overview`);
|
||||
await host.evaluate(async drawn => {
|
||||
const { createBridgeHost } = await import('/prompt_manager/js/shared/toolkit/prompt-bridge.js');
|
||||
window.fakeHost = createBridgeHost({ namespace: '/', origin: location.origin, label: 'PromptManager — ComfyUI',
|
||||
adapter: { listTargets: () => (drawn ? [{ id: '7', label: 'Prompt Manager', type: 'PromptManager', roles: ['positive'],
|
||||
renderer: drawn }] : []), preview() { throw new Error('unused'); }, apply() { throw new Error('unused'); } } });
|
||||
}, renderer);
|
||||
return host;
|
||||
}
|
||||
|
||||
test('send destination offers Ask each time; unsupported options disabled with the reason (D106)', async ({ page }) => {
|
||||
const layout = await openIntegrations(page);
|
||||
expect(await layout.locator(':scope > section.surface > h2').allTextContents()).toEqual(['ComfyUI', 'LoRA & resources']);
|
||||
const send = settingOf(page, 'Default send destination');
|
||||
expect(await send.locator('option').evaluateAll(options => options.map(option => [option.textContent, option.disabled])))
|
||||
.toEqual([['Ask each time', false], ['Selected Prompt Manager node', true], ['Create a Prompt Manager (+/-) node', true]]);
|
||||
await expect(send.locator('select')).toHaveValue('Ask each time');
|
||||
await expect(send.locator('p.small')).toHaveText(
|
||||
'Sending always asks for the ComfyUI window and node, so “Ask each time” is the only option.');
|
||||
const renderer = settingOf(page, 'Host compatibility');
|
||||
expect(await renderer.locator('option').evaluateAll(options => options.map(option => [option.textContent, option.disabled])))
|
||||
.toEqual([['Follow host renderer', false], ['Classic', true], ['Nodes 2.0', true]]);
|
||||
await expect(renderer.locator('p.small').first()).toHaveText('Prompt Manager nodes always follow ComfyUI’s own renderer setting.');
|
||||
const back = settingOf(page, 'Return to canvas after send');
|
||||
await expect(back.locator('input')).toBeDisabled();
|
||||
await expect(back.locator('input')).not.toBeChecked();
|
||||
await expect(back.locator('p.small')).toHaveText('Not available: sending applies the text in ComfyUI and keeps this page open.');
|
||||
// Nothing on this surface is ever drafted: the Save row stays clean.
|
||||
await expect(status(page)).toHaveText('No unsaved changes.');
|
||||
await expect(save(page)).toHaveAttribute('aria-disabled', 'true');
|
||||
});
|
||||
|
||||
test('host compatibility shows the detected renderer as status (D106)', async ({ page, context }) => {
|
||||
await openIntegrations(page);
|
||||
const line = settingOf(page, 'Host compatibility').getByRole('status');
|
||||
await expect(line).toHaveText('No ComfyUI window is open in this browser, so its renderer is not known.');
|
||||
const host = await fakeComfyWindow(context, 'nodes2');
|
||||
try {
|
||||
await page.reload();
|
||||
await expect(line).toHaveText('Detected: Nodes 2.0 (1 ComfyUI window).');
|
||||
await host.evaluate(() => window.fakeHost.dispose());
|
||||
await host.evaluate(async () => {
|
||||
const { createBridgeHost } = await import('/prompt_manager/js/shared/toolkit/prompt-bridge.js');
|
||||
window.fakeHost = createBridgeHost({ namespace: '/', origin: location.origin, label: 'PromptManager — ComfyUI',
|
||||
adapter: { listTargets: () => [], preview() {}, apply() {} } });
|
||||
});
|
||||
await page.reload();
|
||||
await expect(line).toHaveText('ComfyUI is open; add a Prompt Manager node to see which renderer it uses.');
|
||||
} finally {
|
||||
await host.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('LoRA rows show detected / enabled / cache-loaded separately', async ({ page, request }) => {
|
||||
await openIntegrations(page);
|
||||
const integration = settingOf(page, 'LoraManager integration');
|
||||
const triggers = settingOf(page, 'Automatically include trigger words');
|
||||
await expect(integration.locator('p').first()).toHaveText('Not detected in ComfyUI’s custom_nodes folders.');
|
||||
await expect(integration.locator('input')).not.toBeChecked();
|
||||
await expect(triggers.locator('p.small')).toHaveText('Trigger words are not loaded.');
|
||||
// Enabling what is not installed is refused by the server: the toggle stays highlighted, nothing was saved.
|
||||
await integration.locator('input').check();
|
||||
await save(page).click();
|
||||
await expect(status(page)).toHaveText('Some changes were not saved. Review the highlighted settings.');
|
||||
await expect(integration.locator('input')).toHaveAttribute('aria-invalid', 'true');
|
||||
expect((await serverJson(request, '/__test__/integrations/state')).enabled).toBe(false);
|
||||
|
||||
await lora(request);
|
||||
await openIntegrations(page);
|
||||
await expect(integration.locator('p').first()).toHaveText('Detected: custom_nodes/ComfyUI-Lora-Manager.');
|
||||
await expect(integration.locator('input')).not.toBeChecked();
|
||||
await enable(page, { triggers: true });
|
||||
await expect(integration.locator('input')).toBeChecked();
|
||||
await expect(triggers.locator('p.small')).toHaveText('Trigger words are loaded.');
|
||||
expect(await serverJson(request, '/__test__/integrations/state')).toMatchObject({ enabled: true, trigger_words: true });
|
||||
// A reload reads the saved state back.
|
||||
await openIntegrations(page);
|
||||
await expect(integration.locator('input')).toBeChecked();
|
||||
await expect(triggers.locator('input')).toBeChecked();
|
||||
await expect(status(page)).toHaveText('No unsaved changes.');
|
||||
});
|
||||
|
||||
test('LoraManager path row shows the detected path read-only, as legacy (D106)', async ({ page, request }) => {
|
||||
await openIntegrations(page);
|
||||
const row = settingOf(page, 'LoraManager path');
|
||||
const input = row.locator('input');
|
||||
await expect(input).toHaveAttribute('readonly', '');
|
||||
await expect(input).toHaveValue('');
|
||||
await expect(input).toHaveAttribute('placeholder', 'Not detected');
|
||||
await lora(request);
|
||||
await openIntegrations(page);
|
||||
await expect(input).toHaveValue('custom_nodes/ComfyUI-Lora-Manager');
|
||||
await input.focus();
|
||||
await page.keyboard.type('elsewhere');
|
||||
await expect(input).toHaveValue('custom_nodes/ComfyUI-Lora-Manager');
|
||||
await expect(status(page)).toHaveText('No unsaved changes.');
|
||||
});
|
||||
|
||||
test('key never prefilled; blank keeps; Clear saved key clears; saved through Save preferences (D114)', async ({ page, request }) => {
|
||||
const enables = [];
|
||||
page.on('request', item => { if (item.url().endsWith(ENABLE_URL)) enables.push(item.postDataJSON()); });
|
||||
await lora(request);
|
||||
await openIntegrations(page);
|
||||
const key = settingOf(page, 'CivitAI API key');
|
||||
const field = key.locator('input[type="password"]');
|
||||
const clear = key.getByRole('button', { name: 'Clear saved key', exact: true });
|
||||
await expect(field).toHaveValue('');
|
||||
await expect(key.locator('p.small')).toHaveText('No key saved.');
|
||||
await expect(clear).toHaveCount(0);
|
||||
// A typed key is dropped when the section is left: the field comes back empty with nothing hidden to save (N4).
|
||||
await field.fill('sk-dropped');
|
||||
await expect(status(page)).toHaveText('Unsaved changes.');
|
||||
await page.locator('#workspace-main .section-nav').getByRole('button', { name: 'Appearance', exact: true }).click();
|
||||
await page.locator('#workspace-main .section-nav').getByRole('button', { name: 'Integrations', exact: true }).click();
|
||||
await expect(status(page)).toHaveText('No unsaved changes.');
|
||||
await expect(key.locator('p.small')).toHaveText('No key saved.');
|
||||
// Typing a key drafts it; Save sends it once through /lora/enable with the settings revision.
|
||||
await field.fill('sk-test-key');
|
||||
await expect(key.locator('p.small')).toHaveText('The new key replaces the saved one when you save.');
|
||||
await expect(status(page)).toHaveText('Unsaved changes.');
|
||||
await save(page).click();
|
||||
await expect(status(page)).toHaveText('Saved to the server configuration.');
|
||||
expect(enables).toHaveLength(1);
|
||||
expect(enables[0]).toMatchObject({ civitai_api_key: 'sk-test-key', enabled: false });
|
||||
expect(enables[0].expected_revision).toMatch(/^(sha256:|absent)/);
|
||||
await expect(field).toHaveValue('');
|
||||
await expect(key.locator('p.small')).toHaveText('A key is saved.');
|
||||
expect((await serverJson(request, '/__test__/integrations/state')).key_saved).toBe(true);
|
||||
// Never prefilled, even after a reload; the page never receives the key.
|
||||
const statusBodies = [];
|
||||
page.on('response', async response => {
|
||||
if (response.url().endsWith('/prompt_manager/lora/status')) statusBodies.push(await response.text());
|
||||
});
|
||||
await openIntegrations(page);
|
||||
await expect(field).toHaveValue('');
|
||||
await expect(key.locator('p.small')).toHaveText('A key is saved.');
|
||||
expect(statusBodies.join('')).not.toContain('sk-test-key');
|
||||
// The announced line is rewritten only when its text changes, not on every edit elsewhere (S10 re-review N8).
|
||||
await page.evaluate(() => {
|
||||
window.__keyNoteWrites = 0;
|
||||
new MutationObserver(records => { window.__keyNoteWrites += records.length; })
|
||||
.observe(document.querySelector('#civitai-key-status'), { childList: true, characterData: true, subtree: true });
|
||||
});
|
||||
const triggers = settingOf(page, 'Automatically include trigger words').locator('input');
|
||||
await triggers.check();
|
||||
await expect(status(page)).toHaveText('Unsaved changes.');
|
||||
await triggers.uncheck();
|
||||
await expect(status(page)).toHaveText('No unsaved changes.');
|
||||
expect(await page.evaluate(() => window.__keyNoteWrites)).toBe(0);
|
||||
// A blank field keeps the saved key: nothing to save.
|
||||
await field.fill(' ');
|
||||
await expect(status(page)).toHaveText('No unsaved changes.');
|
||||
await field.fill('');
|
||||
// Clear saved key drafts the removal; Save removes it. The keyboard stays on the key field, and the line that
|
||||
// says so is announced (S10 review M2).
|
||||
await expect(key.locator('p.small')).toHaveAttribute('role', 'status');
|
||||
await expect(field).toHaveAttribute('aria-describedby', 'civitai-key-status');
|
||||
await clear.focus();
|
||||
await page.keyboard.press('Enter');
|
||||
await expect(key.locator('p.small')).toHaveText('The saved key is removed when you save.');
|
||||
await expect(clear).toHaveCount(0);
|
||||
await expect(field).toBeFocused();
|
||||
await save(page).click();
|
||||
await expect(status(page)).toHaveText('Saved to the server configuration.');
|
||||
expect(enables.at(-1)).toMatchObject({ clear_civitai_api_key: true });
|
||||
expect(enables.at(-1)).not.toHaveProperty('civitai_api_key');
|
||||
await expect(key.locator('p.small')).toHaveText('No key saved.');
|
||||
expect((await serverJson(request, '/__test__/integrations/state')).key_saved).toBe(false);
|
||||
});
|
||||
|
||||
test('a LoRA save over settings changed elsewhere is a conflict, never a blind write', async ({ page, request }) => {
|
||||
await lora(request);
|
||||
await openIntegrations(page);
|
||||
await settingOf(page, 'Automatically include trigger words').locator('input').check();
|
||||
// Another page saves the settings meanwhile: the revision this page read is stale.
|
||||
await expect(await request.post('/prompt_manager/settings', { data: { result_timeout: 42 } })).toBeOK();
|
||||
await save(page).click();
|
||||
await expect(status(page)).toHaveText('Settings changed elsewhere. Reload this section to see them.');
|
||||
expect((await serverJson(request, '/__test__/integrations/state')).trigger_words).toBe(false);
|
||||
// The draft stays; the next Save carries the reread revision and succeeds.
|
||||
await save(page).click();
|
||||
await expect(status(page)).toHaveText('Saved to the server configuration.');
|
||||
expect((await serverJson(request, '/__test__/integrations/state')).trigger_words).toBe(true);
|
||||
});
|
||||
|
||||
test('a save where only the LoRA part is written says so; the next save needs no conflict (S10 review I3)',
|
||||
async ({ page, request }) => {
|
||||
await lora(request);
|
||||
await expect(await request.post('/__test__/system/missing-source', { data: { inside: true } })).toBeOK();
|
||||
await openIntegrations(page);
|
||||
const triggers = settingOf(page, 'Automatically include trigger words').locator('input');
|
||||
await triggers.check();
|
||||
// In Sources, the drive is unplugged after the page read the list, and another source is removed.
|
||||
const nav = page.locator('#workspace-main .section-nav');
|
||||
await nav.getByRole('button', { name: 'Sources & scans', exact: true }).click();
|
||||
const sourceRow = title => layoutOf(page).locator('#source-list .operation').filter({ has: page.locator('h3', { hasText: title }) });
|
||||
await expect(sourceRow('unplugged')).toBeVisible();
|
||||
await expect(await request.post('/__test__/system/unplug')).toBeOK();
|
||||
await sourceRow('ComfyUI input').getByRole('button', { name: 'Remove', exact: true }).click();
|
||||
await save(page).click();
|
||||
// The LoRA part was saved, the sources were refused: the status gives the server's reason for the source, and
|
||||
// only the source row is highlighted.
|
||||
await expect(status(page)).toHaveText("output/unplugged can't be used: Gallery root does not exist. Remove it or reconnect it, then save.");
|
||||
await expect(sourceRow('unplugged')).toHaveAttribute('aria-invalid', 'true');
|
||||
expect((await serverJson(request, '/__test__/integrations/state')).trigger_words).toBe(true);
|
||||
await nav.getByRole('button', { name: 'Integrations', exact: true }).click();
|
||||
await expect(triggers).toBeChecked();
|
||||
await expect(triggers).not.toHaveAttribute('aria-invalid', 'true');
|
||||
// The next save carries the revision the saved part left: it saves, with no "changed elsewhere".
|
||||
await nav.getByRole('button', { name: 'Sources & scans', exact: true }).click();
|
||||
await sourceRow('unplugged').getByRole('button', { name: 'Remove', exact: true }).click();
|
||||
await save(page).click();
|
||||
await expect(status(page)).toHaveText('Saved to the server configuration.');
|
||||
expect((await serverJson(request, '/prompt_manager/settings')).sources.configured.map(source => source.path)).toEqual(['output']);
|
||||
});
|
||||
|
||||
test('a partial save made from Integrations keeps its outcome when Sources opens (S10 re-review R1)',
|
||||
async ({ page, request }) => {
|
||||
await lora(request);
|
||||
await expect(await request.post('/__test__/system/missing-source', { data: { inside: true } })).toBeOK();
|
||||
await openIntegrations(page);
|
||||
const nav = page.locator('#workspace-main .section-nav');
|
||||
const sourceRow = title => layoutOf(page).locator('#source-list .operation').filter({ has: page.locator('h3', { hasText: title }) });
|
||||
await nav.getByRole('button', { name: 'Sources & scans', exact: true }).click();
|
||||
await expect(sourceRow('unplugged')).toBeVisible();
|
||||
await expect(await request.post('/__test__/system/unplug')).toBeOK();
|
||||
await sourceRow('ComfyUI input').getByRole('button', { name: 'Remove', exact: true }).click();
|
||||
// Back in Integrations, a LoRA change, then Save from there.
|
||||
await nav.getByRole('button', { name: 'Integrations', exact: true }).click();
|
||||
await settingOf(page, 'Automatically include trigger words').locator('input').check();
|
||||
await save(page).click();
|
||||
const reason = "output/unplugged can't be used: Gallery root does not exist. Remove it or reconnect it, then save.";
|
||||
await expect(status(page)).toHaveText(reason);
|
||||
// Leaving Integrations (no key was typed) keeps the outcome: Sources shows the reason on the refused row.
|
||||
await nav.getByRole('button', { name: 'Sources & scans', exact: true }).click();
|
||||
await expect(sourceRow('unplugged')).toHaveAttribute('aria-invalid', 'true');
|
||||
await expect(status(page)).toHaveText(reason);
|
||||
});
|
||||
|
||||
test('Review import shows replacement, blocked items and network effects with the backup box; Start opens the job (D107)',
|
||||
async ({ page, request }) => {
|
||||
await lora(request, { loras: 3, civitai: true, previous: 2 });
|
||||
const previews = [];
|
||||
const applies = [];
|
||||
page.on('request', item => {
|
||||
if (item.url().endsWith(PREVIEW_URL)) previews.push(item.postDataJSON());
|
||||
if (item.url().endsWith(APPLY_URL)) applies.push(item.postDataJSON());
|
||||
});
|
||||
await openIntegrations(page);
|
||||
await enable(page);
|
||||
const before = await serverJson(request, '/__test__/integrations/state');
|
||||
const review = layoutOf(page).locator('.operation').filter({ hasText: 'LoRA reference data' })
|
||||
.getByRole('button', { name: 'Review import', exact: true });
|
||||
await review.click();
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Review LoRA import');
|
||||
await expect(dialog(page).locator('#modal-body > h3')).toHaveText('LoRA reference data');
|
||||
expect(previews).toEqual([{}]);
|
||||
expect(await gridOf(page)).toEqual([
|
||||
['Scope', 'Replace 2 imported prompts with 3 LoRAs'],
|
||||
['Found', '3 metadata files · 3 readable'],
|
||||
['Protected', 'Nothing is protected'],
|
||||
['Network', 'Downloads up to 3 example images from CivitAI'],
|
||||
['Backup', 'Create and verify before changes (browser.db.backup_<date and time>)'],
|
||||
['Media on disk', 'No files are removed; image links are replaced'],
|
||||
]);
|
||||
await expect(dialog(page).locator('details > summary')).toHaveText('LoRAs found (3)');
|
||||
await expect(dialog(page).locator('.notice h3')).toHaveText('Back up the database first (recommended)');
|
||||
await expect(dialog(page).locator('p.small').last()).toContainText('stay in the image cache');
|
||||
// The review wrote and downloaded nothing; Cancel neither.
|
||||
await dialog(page).getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
await expect(review).toBeFocused();
|
||||
expect(await serverJson(request, '/__test__/integrations/state')).toEqual(before);
|
||||
|
||||
await review.click();
|
||||
await dialog(page).getByRole('button', { name: 'Start import', exact: true }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
await expect(page).toHaveURL(/#\/activity\/[0-9a-f]{32}$/);
|
||||
await expect(toast(page)).toHaveText(/LoRA reference import started\.|LoRA import finished · 3 imported · 0 errors/);
|
||||
expect(applies).toHaveLength(1);
|
||||
expect(Object.keys(applies[0])).toEqual(['token']);
|
||||
const job = await ended(request, page.url().split('/').pop());
|
||||
expect([job.kind, job.title, job.state]).toEqual(['lora_import', 'LoRA reference import', 'finished']);
|
||||
expect(job.counters).toMatchObject({ imported: 3, downloaded: 3, download_errors: 0, errors: 0 });
|
||||
const main = page.locator('#workspace-main');
|
||||
await expect(main.locator('h1')).toHaveText('LoRA reference import');
|
||||
await expect(main.locator('.stat-row .stat dt')).toHaveText(['Imported', 'Linked to existing', 'Skipped', 'Images downloaded',
|
||||
'Download errors', 'Errors']);
|
||||
await expect(main.getByRole('button', { name: 'Cancel import', exact: true })).toHaveCount(0);
|
||||
const after = await serverJson(request, '/__test__/integrations/state');
|
||||
expect(after.imports).toEqual(['style_01 example prompt', 'style_02 example prompt', 'style_03 example prompt']);
|
||||
expect(after.downloads).toBe(3);
|
||||
});
|
||||
|
||||
test('saved pairs block the import: the review explains it and offers only Close', async ({ page, request }) => {
|
||||
await lora(request, { previous: 1 });
|
||||
await expect(await request.post('/__test__/integrations/pair')).toBeOK();
|
||||
await openIntegrations(page);
|
||||
await enable(page);
|
||||
await layoutOf(page).getByRole('button', { name: 'Review import', exact: true }).click();
|
||||
expect((await gridOf(page))[2]).toEqual(['Protected', '1 imported prompt used by saved pairs']);
|
||||
await expect(dialog(page).locator('p.small')).toHaveText(
|
||||
'Delete or change the saved pairs that use these prompts, then review the import again. Nothing was changed.');
|
||||
expect(await dialog(page).locator('.dialog-actions button').allTextContents()).toEqual(['Close']);
|
||||
await dialog(page).getByRole('button', { name: 'Close', exact: true }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
});
|
||||
|
||||
test('a failed import keeps old imports and its job says so', async ({ page, request }) => {
|
||||
await lora(request, { loras: 2, previous: 2 });
|
||||
await openIntegrations(page);
|
||||
await enable(page);
|
||||
const before = await serverJson(request, '/__test__/integrations/state');
|
||||
await expect(await request.post('/__test__/integrations/fail-import')).toBeOK();
|
||||
await layoutOf(page).getByRole('button', { name: 'Review import', exact: true }).click();
|
||||
await dialog(page).getByRole('button', { name: 'Start import', exact: true }).click();
|
||||
await expect(page).toHaveURL(/#\/activity\/[0-9a-f]{32}$/);
|
||||
await expect(toast(page)).toHaveText(`LoRA import failed · ${KEPT}`);
|
||||
const job = await ended(request, page.url().split('/').pop());
|
||||
expect([job.state, job.scope.failure, job.counters.errors]).toEqual(['failed', 'import-failed', 1]);
|
||||
await expect(page.locator('#workspace-main')).toContainText(KEPT);
|
||||
expect((await serverJson(request, '/__test__/integrations/state')).imports).toEqual(before.imports);
|
||||
expect(before.imports).toEqual(['previous lora import 1', 'previous lora import 2']);
|
||||
});
|
||||
|
||||
test('the import can be cancelled while it downloads; then nothing is written', async ({ page, request }) => {
|
||||
await lora(request, { loras: 2, civitai: true, previous: 1 });
|
||||
await openIntegrations(page);
|
||||
await enable(page);
|
||||
const before = await serverJson(request, '/__test__/integrations/state');
|
||||
await expect(await request.post('/__test__/integrations/gate', { data: { hold: true } })).toBeOK();
|
||||
await layoutOf(page).getByRole('button', { name: 'Review import', exact: true }).click();
|
||||
await dialog(page).getByRole('button', { name: 'Start import', exact: true }).click();
|
||||
await expect(page).toHaveURL(/#\/activity\/[0-9a-f]{32}$/);
|
||||
const id = page.url().split('/').pop();
|
||||
await expect.poll(async () => (await serverJson(request, '/__test__/integrations/state')).downloads).toBeGreaterThan(0);
|
||||
const cancel = page.locator('#workspace-main').getByRole('button', { name: 'Cancel import', exact: true });
|
||||
await expect(cancel).toBeVisible();
|
||||
// The cancel must have reached the server before the held download goes on (S10 review M1).
|
||||
await Promise.all([
|
||||
page.waitForResponse(response => /\/prompt_manager\/jobs\/[0-9a-f]+\/cancel$/.test(response.url()) && response.ok()),
|
||||
cancel.click(),
|
||||
]);
|
||||
await expect(await request.post('/__test__/integrations/gate', { data: { hold: false } })).toBeOK();
|
||||
const job = await ended(request, id);
|
||||
expect(job.state).toBe('cancelled');
|
||||
expect((await serverJson(request, '/__test__/integrations/state')).imports).toEqual(before.imports);
|
||||
await expect(cancel).toHaveCount(0);
|
||||
});
|
||||
@@ -0,0 +1,483 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { seedConcept } from './concept-inventory.mjs';
|
||||
|
||||
// System › Maintenance (#249 System S7 Task 19) on the concept-shaped real fixture and the S6 backend: the concept's
|
||||
// rows, the read-only health check, the D100 plan review applied as an Activity job, the D101 duplicate search and
|
||||
// review, the S5 thumbnail dialog and the D102 metadata cache row. The hooks are workspace_system_fixture.py's.
|
||||
const workspace = '/prompt_manager/workspace';
|
||||
const PREVIEW_URL = '/prompt_manager/maintenance/preview';
|
||||
const APPLY_URL = '/prompt_manager/maintenance/apply';
|
||||
const CHANGED = 'The library changed since the review; nothing was changed. Review the plan again.';
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
await seedConcept(request);
|
||||
await expect(await request.post('/__test__/system/reset')).toBeOK();
|
||||
await expect(await request.post('/__test__/jobs/restart')).toBeOK();
|
||||
page.systemErrors = [];
|
||||
page.on('pageerror', error => page.systemErrors.push(error.message));
|
||||
});
|
||||
|
||||
test.afterEach(async ({ page, request, browserName }) => {
|
||||
await saveCoverage(page, browserName);
|
||||
await request.post('/__test__/release');
|
||||
await request.post('/__test__/jobs/restart');
|
||||
await request.post('/__test__/system/reset');
|
||||
expect(page.systemErrors).toEqual([]);
|
||||
});
|
||||
|
||||
const toast = page => page.locator('#workspace-toast');
|
||||
const layoutOf = page => page.locator('#workspace-main .settings-layout');
|
||||
const dialog = page => page.locator('dialog#modal[open]');
|
||||
const row = (page, title) => layoutOf(page).locator('.operation').filter({ has: page.locator('h3', { hasText: title }) });
|
||||
|
||||
async function serverJson(request, url) {
|
||||
const response = await request.get(url);
|
||||
await expect(response).toBeOK();
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async function openMaintenance(page) {
|
||||
await page.goto(`${workspace}#/system/maintenance`);
|
||||
await expect(layoutOf(page).locator(':scope > section.surface')).toHaveCount(2);
|
||||
return layoutOf(page);
|
||||
}
|
||||
|
||||
/** The open plan dialog's `[dt, dd]` rows. */
|
||||
const gridOf = page => dialog(page).locator('dl.meta-grid > div').evaluateAll(items => items.map(item =>
|
||||
[item.querySelector('dt').textContent, item.querySelector('dd').textContent]));
|
||||
|
||||
/** Wait until the job `id` ended on the server; answers its summary. */
|
||||
async function ended(request, id) {
|
||||
await expect.poll(async () => (await serverJson(request, `/prompt_manager/jobs/${id}`)).job.state).not.toBe('running');
|
||||
return (await serverJson(request, `/prompt_manager/jobs/${id}`)).job;
|
||||
}
|
||||
|
||||
test('the section shows the concept rows; Metadata cache follows Q5 (D102)', async ({ page }) => {
|
||||
const layout = await openMaintenance(page);
|
||||
await expect(layout.locator(':scope > .notice h2')).toHaveText('Check first. Review changes. Then apply.');
|
||||
await expect(layout.locator(':scope > section.handoff')).toHaveCount(0);
|
||||
expect(await layout.locator(':scope > section.surface').evaluateAll(surfaces => surfaces.map(surface => [
|
||||
surface.querySelector('h2').textContent,
|
||||
[...surface.querySelectorAll('.operation')].map(item => [item.querySelector('h3').textContent,
|
||||
item.querySelector('.row-actions > button').textContent, item.querySelector('.row-actions > button').className]),
|
||||
]))).toEqual([
|
||||
['Database', [['Integrity & relationship checks', 'Check', ''], ['Duplicate prompt text', 'Review plan', ''],
|
||||
['Optimize database', 'Review plan', ''], ['Unused prompt components', 'Review plan', 'danger']]],
|
||||
['Images & cache', [['Missing-file links', 'Review plan', 'danger'], ['Duplicate image files', 'Find duplicates', ''],
|
||||
['Thumbnail cache', 'Review options', ''], ['Metadata cache', 'Review plan', '']]],
|
||||
]);
|
||||
// Nothing of Prompt Manager's is cached in a browser (#222): the row says so and its review is unavailable.
|
||||
const cache = row(page, 'Metadata cache');
|
||||
await expect(cache.locator('p')).toHaveText('Clear only Prompt Manager metadata cached by this browser. Nothing is cached in this browser.');
|
||||
const review = cache.getByRole('button', { name: 'Review plan', exact: true });
|
||||
await expect(review).toHaveAttribute('aria-disabled', 'true');
|
||||
// An aria-disabled button stays focusable and clickable; its press opens nothing.
|
||||
await review.dispatchEvent('click');
|
||||
await expect(review).toHaveAttribute('aria-disabled', 'true');
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
});
|
||||
|
||||
test('Metadata cache reviews and clears only owned browser entries when there are some (D102)', async ({ page }) => {
|
||||
// The shipped inventory is empty (#222); this test names one owned entry through the module's inventory seam.
|
||||
await page.addInitScript(() => {
|
||||
localStorage.setItem('pm-owned-test-entry', 'cached');
|
||||
localStorage.setItem('pm_workspace_theme', 'dark');
|
||||
let wrapped;
|
||||
Object.defineProperty(window, 'CacheOwnership', { configurable: true, get: () => wrapped, set(api) {
|
||||
const owned = { storageKeys: ['pm-owned-test-entry'], cacheNames: [] };
|
||||
wrapped = { ...api, ownedEntries: options => api.ownedEntries({ ...options, owned }),
|
||||
clearOwned: options => api.clearOwned({ ...options, owned }) };
|
||||
} });
|
||||
});
|
||||
await openMaintenance(page);
|
||||
const cache = row(page, 'Metadata cache');
|
||||
const review = cache.getByRole('button', { name: 'Review plan', exact: true });
|
||||
await expect(review).not.toHaveAttribute('aria-disabled', 'true');
|
||||
await expect(cache.locator('p')).toHaveText('Clear only Prompt Manager metadata cached by this browser.');
|
||||
await review.click();
|
||||
await expect(dialog(page).locator('#modal-body > h3')).toHaveText('Metadata cache');
|
||||
expect(await gridOf(page)).toEqual([['Scope', '1 cached entry in this browser'],
|
||||
['Protected', 'Preferences and other tools’ data in this browser'], ['Backup', 'Not needed: the database is not changed'],
|
||||
['Media on disk', 'No files are removed']]);
|
||||
await dialog(page).getByRole('button', { name: 'Clear browser cache', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('Cleared 1 cached entry from this browser.');
|
||||
await expect(review).toHaveAttribute('aria-disabled', 'true');
|
||||
expect(await page.evaluate(() => [localStorage.getItem('pm-owned-test-entry'), localStorage.getItem('pm_workspace_theme')]))
|
||||
.toEqual([null, 'dark']);
|
||||
});
|
||||
|
||||
test('Run health checks and Check open the Health report; Review missing links starts a fresh preview, never a cleanup',
|
||||
async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/system/missing-file')).toBeOK();
|
||||
const before = await serverJson(request, '/__test__/system/digest');
|
||||
const previews = [];
|
||||
page.on('request', item => { if (item.url().endsWith(PREVIEW_URL)) previews.push(item.postDataJSON()); });
|
||||
await openMaintenance(page);
|
||||
const check = row(page, 'Integrity & relationship checks').getByRole('button', { name: 'Check', exact: true });
|
||||
await check.click();
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Health report');
|
||||
await expect(dialog(page).locator('.operation').filter({ hasText: 'Missing files · output' }).locator('.chip'))
|
||||
.toHaveText('1 link needs review');
|
||||
await expect(check).toHaveText('Check');
|
||||
await dialog(page).getByRole('button', { name: 'Review missing links', exact: true }).click();
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Review maintenance plan');
|
||||
await expect(dialog(page).locator('#modal-body > h3')).toHaveText('Missing-file links');
|
||||
expect(previews).toEqual([{ operation: 'missing_links', options: {} }]);
|
||||
expect((await gridOf(page))[0]).toEqual(['Scope', '1 link to a missing file']);
|
||||
await dialog(page).getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
await expect(check).toBeFocused();
|
||||
// The notice's Run health checks opens the same report.
|
||||
await layoutOf(page).getByRole('button', { name: 'Run health checks', exact: true }).click();
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Health report');
|
||||
expect(await serverJson(request, '/__test__/system/digest')).toEqual(before);
|
||||
});
|
||||
|
||||
test("plan dialog shows the server's scope, protected, backup policy and media lines with the backup box (D100); Cancel writes nothing",
|
||||
async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/system/duplicate-prompts')).toBeOK();
|
||||
const before = await serverJson(request, '/__test__/system/digest');
|
||||
const answers = [];
|
||||
page.on('response', response => { if (response.url().endsWith(PREVIEW_URL)) answers.push(response.json()); });
|
||||
await openMaintenance(page);
|
||||
const reviewButton = row(page, 'Duplicate prompt text').getByRole('button', { name: 'Review plan', exact: true });
|
||||
await reviewButton.click();
|
||||
const plan = dialog(page);
|
||||
await expect(plan.locator('#modal-title')).toHaveText('Review maintenance plan');
|
||||
const answer = await answers[0];
|
||||
expect(answer.scope.count).toBe(1);
|
||||
await expect(plan.locator('#modal-body > h3')).toHaveText('Duplicate prompt text');
|
||||
expect(await gridOf(page)).toEqual([
|
||||
['Scope', answer.scope.label],
|
||||
['Protected', answer.protected.reasons.length ? answer.protected.reasons.join(' · ') : 'Nothing is protected'],
|
||||
['Backup', `Create and verify before changes (${answer.backup.destination_label})`],
|
||||
['Media on disk', answer.media.label],
|
||||
]);
|
||||
expect(await plan.locator('#modal-body').evaluate(body => [...body.children].map(node =>
|
||||
[node.localName, ...node.classList].join('.')))).toEqual(['h3', 'dl.meta-grid', 'details', 'p.small', 'div.notice',
|
||||
'div.dialog-actions']);
|
||||
await expect(plan.locator('details > summary')).toHaveText('Listed records (1)');
|
||||
await expect(plan.locator('#modal-body > p.small')).toHaveText(/^Nothing changes until you apply\. This review holds for 1[45] minutes\.$/);
|
||||
await expect(plan.locator('.notice h3')).toHaveText('Back up the database first (recommended)');
|
||||
await expect(plan.locator('.dialog-actions button')).toHaveText(['Cancel', 'Apply reviewed operation']);
|
||||
await plan.getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
await expect(reviewButton).toBeFocused();
|
||||
// Optimize and Unused prompt components are reviewed the same way.
|
||||
for (const [title, scope] of [['Optimize database', 'The whole database'], ['Unused prompt components', null]]) {
|
||||
await row(page, title).getByRole('button', { name: 'Review plan', exact: true }).click();
|
||||
await expect(dialog(page).locator('#modal-body > h3')).toHaveText(title);
|
||||
const value = (await answers.at(-1)).scope.label;
|
||||
expect((await gridOf(page))[0]).toEqual(['Scope', scope ?? value]);
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
}
|
||||
expect(await serverJson(request, '/__test__/system/digest')).toEqual(before);
|
||||
});
|
||||
|
||||
test('a plan with nothing to do shows only Close, never Apply', async ({ page, request }) => {
|
||||
const applies = [];
|
||||
page.on('request', item => { if (item.url().endsWith(APPLY_URL)) applies.push(item.url()); });
|
||||
await openMaintenance(page);
|
||||
// The concept fixture has no duplicate prompt text.
|
||||
const reviewButton = row(page, 'Duplicate prompt text').getByRole('button', { name: 'Review plan', exact: true });
|
||||
await reviewButton.click();
|
||||
await expect(dialog(page).locator('#modal-body > h3')).toHaveText('Duplicate prompt text');
|
||||
expect((await gridOf(page))[0]).toEqual(['Scope', 'No duplicate prompt text']);
|
||||
await expect(dialog(page).locator('#modal-body > p.small')).toHaveText('There is nothing to apply; nothing was changed.');
|
||||
await expect(dialog(page).locator('.dialog-actions button')).toHaveText(['Close']);
|
||||
await expect(dialog(page).locator('.notice')).toHaveCount(0);
|
||||
await dialog(page).getByRole('button', { name: 'Close', exact: true }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
await expect(reviewButton).toBeFocused();
|
||||
expect(applies).toEqual([]);
|
||||
});
|
||||
|
||||
test('Apply reviewed operation closes, toasts, opens the job in Activity', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/system/duplicate-prompts')).toBeOK();
|
||||
const applies = [];
|
||||
page.on('request', item => { if (item.url().endsWith(APPLY_URL)) applies.push(item.postDataJSON()); });
|
||||
await openMaintenance(page);
|
||||
await row(page, 'Duplicate prompt text').getByRole('button', { name: 'Review plan', exact: true }).click();
|
||||
await dialog(page).getByRole('button', { name: 'Apply reviewed operation', exact: true }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
await expect(page).toHaveURL(/#\/activity\/[0-9a-f]{32}$/);
|
||||
await expect(toast(page)).toHaveText(/Duplicate prompt text (started|finished · 1 changed)/);
|
||||
expect(applies).toHaveLength(1);
|
||||
expect(Object.keys(applies[0])).toEqual(['token']);
|
||||
const job = await ended(request, page.url().split('/').pop());
|
||||
expect([job.kind, job.state, job.counters.changed]).toEqual(['maintenance', 'finished', 1]);
|
||||
const main = page.locator('#workspace-main');
|
||||
await expect(main.locator('h1')).toHaveText('Duplicate prompt text');
|
||||
await expect(main.locator('.stat-row .stat dt')).toHaveText(['Changed', 'Protected', 'Errors']);
|
||||
await expect(main.locator('.stat-row .stat dd').first()).toHaveText('1');
|
||||
// Not cancellable: no Cancel, a report like every job.
|
||||
await expect(main.getByRole('button', { name: 'Cancel job', exact: true })).toHaveCount(0);
|
||||
await expect(main.getByRole('button', { name: 'Download report', exact: true })).toBeVisible();
|
||||
});
|
||||
|
||||
test('plan-changed asks for a new review', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/system/duplicate-prompts')).toBeOK();
|
||||
await openMaintenance(page);
|
||||
await row(page, 'Duplicate prompt text').getByRole('button', { name: 'Review plan', exact: true }).click();
|
||||
await expect(dialog(page).locator('#modal-body > h3')).toHaveText('Duplicate prompt text');
|
||||
// The library changes after the review: the apply's recheck refuses it and nothing is changed.
|
||||
await expect(await request.post('/__test__/system/duplicate-prompts', { data: { again: true } })).toBeOK();
|
||||
const before = await serverJson(request, '/__test__/system/digest');
|
||||
await dialog(page).getByRole('button', { name: 'Apply reviewed operation', exact: true }).click();
|
||||
await expect(page).toHaveURL(/#\/activity\/[0-9a-f]{32}$/);
|
||||
await expect(toast(page)).toHaveText(`Duplicate prompt text failed · ${CHANGED}`);
|
||||
await expect(page.locator('#workspace-main')).toContainText(CHANGED);
|
||||
expect((await serverJson(request, '/__test__/system/digest')).rows).toEqual(before.rows);
|
||||
// S7 review M2: the failed job offers Review again, which reviews the same operation afresh.
|
||||
const previews = [];
|
||||
page.on('request', item => { if (item.url().endsWith(PREVIEW_URL)) previews.push(item.postDataJSON()); });
|
||||
await page.locator('#workspace-main').getByRole('button', { name: 'Review again', exact: true }).click();
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Review maintenance plan');
|
||||
await expect(dialog(page).locator('#modal-body > h3')).toHaveText('Duplicate prompt text');
|
||||
expect(previews).toEqual([{ operation: 'duplicate_prompts', options: {} }]);
|
||||
});
|
||||
|
||||
test('a review button pressed twice sends one preview (S7 review M4)', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/system/missing-file')).toBeOK();
|
||||
await expect(await request.post('/__test__/system/duplicate-files')).toBeOK();
|
||||
const previews = [];
|
||||
page.on('request', item => { if (item.url().endsWith(PREVIEW_URL)) previews.push(item.postDataJSON().operation); });
|
||||
await openMaintenance(page);
|
||||
await row(page, 'Integrity & relationship checks').getByRole('button', { name: 'Check', exact: true }).click();
|
||||
// Hold the preview at the server so the second press lands while it is read.
|
||||
await expect(await request.post('/__test__/hold', { data: { path: PREVIEW_URL } })).toBeOK();
|
||||
// The pressed button reads "Reviewing…" while held, so it is found by its place, not its name.
|
||||
const missing = dialog(page).locator('.dialog-actions > button').last();
|
||||
await expect(missing).toHaveText('Review missing links');
|
||||
await missing.click();
|
||||
await expect(missing).toHaveAttribute('aria-disabled', 'true');
|
||||
await expect(missing).toHaveText('Reviewing…');
|
||||
await missing.dispatchEvent('click');
|
||||
await expect.poll(async () => (await serverJson(request, '/__test__/hold')).held[PREVIEW_URL]).toBe(1);
|
||||
await expect(await request.post('/__test__/release')).toBeOK();
|
||||
await expect(dialog(page).locator('#modal-body > h3')).toHaveText('Missing-file links');
|
||||
expect(previews).toEqual(['missing_links']);
|
||||
await page.keyboard.press('Escape');
|
||||
|
||||
await row(page, 'Duplicate image files').getByRole('button', { name: 'Find duplicates', exact: true }).click();
|
||||
await page.locator('#workspace-main').getByRole('button', { name: 'Review groups', exact: true }).click();
|
||||
await expect(await request.post('/__test__/hold', { data: { path: PREVIEW_URL } })).toBeOK();
|
||||
const cleanup = dialog(page).locator('.dialog-actions > button').last();
|
||||
await expect(cleanup).toHaveText('Review duplicate cleanup');
|
||||
await cleanup.click();
|
||||
await expect(cleanup).toHaveAttribute('aria-disabled', 'true');
|
||||
await cleanup.dispatchEvent('click');
|
||||
await expect.poll(async () => (await serverJson(request, '/__test__/hold')).held[PREVIEW_URL]).toBe(1);
|
||||
await expect(await request.post('/__test__/release')).toBeOK();
|
||||
await expect(dialog(page).locator('#modal-body > h3')).toHaveText('Duplicate image files');
|
||||
expect(previews).toEqual(['missing_links', 'duplicate_files']);
|
||||
});
|
||||
|
||||
test('plan-used links the job that applied it', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/system/duplicate-prompts')).toBeOK();
|
||||
// A double-submitted apply: the first request applies the plan, the second one's answer reaches the page.
|
||||
await page.route(`**${APPLY_URL}`, async route => {
|
||||
await route.fetch();
|
||||
await route.fulfill({ response: await route.fetch() });
|
||||
});
|
||||
await openMaintenance(page);
|
||||
await row(page, 'Duplicate prompt text').getByRole('button', { name: 'Review plan', exact: true }).click();
|
||||
await dialog(page).getByRole('button', { name: 'Apply reviewed operation', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('This plan was already applied: Duplicate prompt text.');
|
||||
const link = toast(page).getByRole('link', { name: 'Duplicate prompt text', exact: true });
|
||||
const { jobs } = await serverJson(request, '/prompt_manager/jobs');
|
||||
const applied = jobs.find(job => job.kind === 'maintenance');
|
||||
await expect(link).toHaveAttribute('href', `#/activity/${applied.id}`);
|
||||
await expect(page).toHaveURL(`${workspace}#/system/maintenance`);
|
||||
});
|
||||
|
||||
/**
|
||||
* Record every toast the page shows (`[text, [link hrefs]]`): a job's finish toast may replace an earlier one before a
|
||||
* test reads it.
|
||||
*/
|
||||
async function recordToasts(page) {
|
||||
await page.addInitScript(() => {
|
||||
window.toastLog = [];
|
||||
const record = node => window.toastLog.push([node.textContent, [...node.querySelectorAll('a')].map(a => a.getAttribute('href'))]);
|
||||
new MutationObserver(() => {
|
||||
const node = document.getElementById('workspace-toast');
|
||||
const last = window.toastLog.at(-1);
|
||||
if (node && node.textContent && last?.[0] !== node.textContent) record(node);
|
||||
}).observe(document, { subtree: true, childList: true, characterData: true });
|
||||
});
|
||||
return () => page.evaluate(() => window.toastLog);
|
||||
}
|
||||
|
||||
const NOT_CONFIRMED = ['Outcome not confirmed. Check Activity before applying it again.', ['#/activity']];
|
||||
|
||||
/** The maintenance jobs the server lists (the apply ran even though its answer was lost). */
|
||||
async function maintenanceJobs(request) {
|
||||
return (await serverJson(request, '/prompt_manager/jobs')).jobs.filter(job => job.kind === 'maintenance');
|
||||
}
|
||||
|
||||
test('lost apply response says Outcome not confirmed and points to Activity', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/system/duplicate-prompts')).toBeOK();
|
||||
// The server applies the plan; its answer never reaches the page.
|
||||
const reads = [];
|
||||
await page.route(`**${APPLY_URL}`, async route => {
|
||||
await route.fetch();
|
||||
page.on('request', item => { if (item.url().endsWith('/prompt_manager/jobs')) reads.push(item.url()); });
|
||||
await route.abort('failed');
|
||||
});
|
||||
const toasts = await recordToasts(page);
|
||||
await openMaintenance(page);
|
||||
await row(page, 'Duplicate prompt text').getByRole('button', { name: 'Review plan', exact: true }).click();
|
||||
await dialog(page).getByRole('button', { name: 'Apply reviewed operation', exact: true }).click();
|
||||
await expect.poll(toasts).toContainEqual(NOT_CONFIRMED);
|
||||
await expect(page).toHaveURL(`${workspace}#/system/maintenance`);
|
||||
// The Activity list is read again at once, and the job the lost answer started is in it.
|
||||
await expect.poll(() => reads.length).toBeGreaterThan(0);
|
||||
expect((await maintenanceJobs(request)).map(job => job.title)).toEqual(['Duplicate prompt text']);
|
||||
await page.evaluate(() => { location.hash = '#/activity'; });
|
||||
await expect(page.locator('#workspace-main')).toContainText('Duplicate prompt text');
|
||||
});
|
||||
|
||||
test('a proxy error or an unreadable answer is not a refusal: Outcome not confirmed (S7 review I1)', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/system/duplicate-prompts')).toBeOK();
|
||||
const reads = [];
|
||||
// A reverse proxy times out after the server started the job.
|
||||
await page.route(`**${APPLY_URL}`, async route => {
|
||||
await route.fetch();
|
||||
page.on('request', item => { if (item.url().endsWith('/prompt_manager/jobs')) reads.push(item.url()); });
|
||||
await route.fulfill({ status: 504, contentType: 'text/html', body: '<html><body>Gateway Timeout</body></html>' });
|
||||
});
|
||||
const toasts = await recordToasts(page);
|
||||
await openMaintenance(page);
|
||||
await row(page, 'Duplicate prompt text').getByRole('button', { name: 'Review plan', exact: true }).click();
|
||||
await dialog(page).getByRole('button', { name: 'Apply reviewed operation', exact: true }).click();
|
||||
await expect.poll(toasts).toContainEqual(NOT_CONFIRMED);
|
||||
await expect.poll(() => reads.length).toBeGreaterThan(0);
|
||||
expect(await maintenanceJobs(request)).toHaveLength(1);
|
||||
});
|
||||
|
||||
test('a refused apply names the running job', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/system/duplicate-prompts')).toBeOK();
|
||||
await openMaintenance(page);
|
||||
await row(page, 'Duplicate prompt text').getByRole('button', { name: 'Review plan', exact: true }).click();
|
||||
await expect(dialog(page).locator('#modal-body > h3')).toHaveText('Duplicate prompt text');
|
||||
const hold = await (await request.post('/__test__/jobs/hold', { data: { title: 'Database restore (admin page)' } })).json();
|
||||
await dialog(page).getByRole('button', { name: 'Apply reviewed operation', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('Another job is running: Database restore (admin page).');
|
||||
await expect(toast(page).getByRole('link')).toHaveAttribute('href', `#/activity/${hold.job.id}`);
|
||||
await expect(page).toHaveURL(`${workspace}#/system/maintenance`);
|
||||
});
|
||||
|
||||
test('Find duplicates starts a job; its view links Review groups; Keep radios build a duplicate_files plan (D101)',
|
||||
async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/system/duplicate-files')).toBeOK();
|
||||
const previews = [];
|
||||
page.on('request', item => { if (item.url().endsWith(PREVIEW_URL)) previews.push(item.postDataJSON()); });
|
||||
await openMaintenance(page);
|
||||
await row(page, 'Duplicate image files').getByRole('button', { name: 'Find duplicates', exact: true }).click();
|
||||
await expect(page).toHaveURL(/#\/activity\/[0-9a-f]{32}$/);
|
||||
const jobId = page.url().split('/').pop();
|
||||
const main = page.locator('#workspace-main');
|
||||
await expect(main.locator('h1')).toHaveText('Find duplicate image files');
|
||||
const reviewGroups = main.getByRole('button', { name: 'Review groups', exact: true });
|
||||
await expect(reviewGroups).toBeVisible();
|
||||
await expect(main.locator('.stat-row .stat dt')).toHaveText(['Groups found']);
|
||||
await reviewGroups.click();
|
||||
const review = dialog(page);
|
||||
await expect(review.locator('#modal-title')).toHaveText('Review duplicate image files');
|
||||
const group = review.locator('.compare-grid').filter({ hasText: 's7-dupes/copy.png' });
|
||||
await expect(group.locator('.compare-item img')).toHaveCount(3);
|
||||
const labels = (await group.locator('.compare-item label').allTextContents()).map(text => text.trim());
|
||||
expect([...labels].sort()).toEqual(['Keep input/s7-dupes/same.png', 'Keep output/s7-dupes/copy.png',
|
||||
'Keep output/s7-dupes/same.png']);
|
||||
await expect(review.locator('#modal-body > p.small').last())
|
||||
.toHaveText(/History keeps the removed paths in its run records\./);
|
||||
// Keep the input copy: the same relative path as the output copy, told apart by its source index.
|
||||
await group.locator('.compare-item').filter({ hasText: 'input/s7-dupes/same.png' }).getByRole('radio').check();
|
||||
await review.getByRole('button', { name: 'Review duplicate cleanup', exact: true }).click();
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Review maintenance plan');
|
||||
await expect(dialog(page).locator('#modal-body > h3')).toHaveText('Duplicate image files');
|
||||
expect(previews).toHaveLength(1);
|
||||
expect(previews[0].operation).toBe('duplicate_files');
|
||||
expect(previews[0].options.job_id).toBe(jobId);
|
||||
expect(previews[0].options.keep).toContainEqual(expect.objectContaining({ root_index: 1, path: 's7-dupes/same.png' }));
|
||||
for (const entry of previews[0].options.keep) expect(Object.keys(entry).sort()).toEqual(['group_id', 'path', 'root_index']);
|
||||
expect((await gridOf(page))[3][1]).toMatch(/^Deletes 2 files \(.+\) from disk; the kept copies stay\./);
|
||||
await dialog(page).getByRole('button', { name: 'Apply reviewed operation', exact: true }).click();
|
||||
await expect(page).not.toHaveURL(new RegExp(jobId));
|
||||
await expect(page).toHaveURL(/#\/activity\/[0-9a-f]{32}$/);
|
||||
const cleanup = await ended(request, page.url().split('/').pop());
|
||||
expect([cleanup.kind, cleanup.state, cleanup.counters.removed]).toEqual(['duplicate_cleanup', 'finished', 2]);
|
||||
await expect(main.locator('.stat-row .stat dt')).toHaveText(['Removed', 'Kept', 'Errors']);
|
||||
expect((await request.get('/prompt_manager/images/serve/s7-dupes/same.png?root=1')).status()).toBe(200);
|
||||
expect((await request.get('/prompt_manager/images/serve/s7-dupes/same.png?root=0')).status()).toBe(404);
|
||||
expect((await request.get('/prompt_manager/images/serve/s7-dupes/copy.png?root=0')).status()).toBe(404);
|
||||
});
|
||||
|
||||
test('the duplicates review pages ten groups at a time and keeps each choice (D101)', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/system/duplicate-files', { data: { groups: 11 } })).toBeOK();
|
||||
const groupReads = [];
|
||||
page.on('request', item => { if (item.url().includes('/maintenance/duplicates/')) groupReads.push(new URL(item.url()).search); });
|
||||
const previews = [];
|
||||
page.on('request', item => { if (item.url().endsWith(PREVIEW_URL)) previews.push(item.postDataJSON()); });
|
||||
await openMaintenance(page);
|
||||
await row(page, 'Duplicate image files').getByRole('button', { name: 'Find duplicates', exact: true }).click();
|
||||
const main = page.locator('#workspace-main');
|
||||
await main.getByRole('button', { name: 'Review groups', exact: true }).click();
|
||||
const review = dialog(page);
|
||||
const status = review.locator('#modal-body > p.small[role=status]');
|
||||
await expect(status).toHaveText('Groups 1–10 of 12');
|
||||
await expect(review.locator('.compare-grid')).toHaveCount(10);
|
||||
await expect(review.getByRole('button', { name: 'Previous groups', exact: true })).toBeDisabled();
|
||||
// The three-copy group sorts first (the most bytes to free): keep its input copy.
|
||||
const first = review.locator('.compare-grid').first();
|
||||
await first.locator('.compare-item').filter({ hasText: 'input/s7-dupes/same.png' }).getByRole('radio').check();
|
||||
await review.getByRole('button', { name: 'Next groups', exact: true }).click();
|
||||
await expect(status).toHaveText('Groups 11–12 of 12');
|
||||
await expect(review.locator('.compare-grid')).toHaveCount(2);
|
||||
await expect(review.getByRole('button', { name: 'Next groups', exact: true })).toBeDisabled();
|
||||
await review.getByRole('button', { name: 'Previous groups', exact: true }).click();
|
||||
await expect(status).toHaveText('Groups 1–10 of 12');
|
||||
await expect(first.locator('.compare-item').filter({ hasText: 'input/s7-dupes/same.png' }).getByRole('radio')).toBeChecked();
|
||||
expect(groupReads).toEqual(['?offset=0&limit=10', '?offset=10&limit=10', '?offset=0&limit=10']);
|
||||
await review.getByRole('button', { name: 'Review duplicate cleanup', exact: true }).click();
|
||||
await expect(dialog(page).locator('#modal-body > h3')).toHaveText('Duplicate image files');
|
||||
// Every group shown so far is reviewed, each with its chosen copy.
|
||||
expect(previews[0].options.keep).toHaveLength(12);
|
||||
expect(previews[0].options.keep).toContainEqual(expect.objectContaining({ root_index: 1, path: 's7-dupes/same.png' }));
|
||||
await dialog(page).getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
});
|
||||
|
||||
for (const [width, height] of [[1600, 1100], [390, 844]]) {
|
||||
test(`long copy paths wrap; the duplicates review never scrolls sideways ${width}x${height}`, async ({ page, request }) => {
|
||||
await page.setViewportSize({ width, height });
|
||||
await expect(await request.post('/__test__/system/duplicate-files', { data: { long: true, trio: false } })).toBeOK();
|
||||
await openMaintenance(page);
|
||||
await row(page, 'Duplicate image files').getByRole('button', { name: 'Find duplicates', exact: true }).click();
|
||||
await page.locator('#workspace-main').getByRole('button', { name: 'Review groups', exact: true }).click();
|
||||
const review = dialog(page);
|
||||
await expect(review.locator('.compare-item')).toHaveCount(2);
|
||||
const overflow = await review.evaluate(node => {
|
||||
const body = node.querySelector('#modal-body');
|
||||
// An item wider than its box, or a label (inline: measured by its boxes) past its item's right edge.
|
||||
const wide = [...node.querySelectorAll('.compare-item')].filter(item => item.scrollWidth > item.clientWidth + 1
|
||||
|| [...item.querySelector('label').getClientRects()].some(rect => rect.right > item.getBoundingClientRect().right + 1)).length;
|
||||
return { dialog: node.scrollWidth > node.clientWidth + 1, body: body.scrollWidth > body.clientWidth + 1, wide,
|
||||
page: document.documentElement.scrollWidth > document.documentElement.clientWidth };
|
||||
});
|
||||
expect(overflow).toEqual({ dialog: false, body: false, wide: 0, page: false });
|
||||
});
|
||||
}
|
||||
|
||||
test('Thumbnail cache opens the S5 dialog', async ({ page }) => {
|
||||
await openMaintenance(page);
|
||||
const options = row(page, 'Thumbnail cache').getByRole('button', { name: 'Review options', exact: true });
|
||||
await options.click();
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Thumbnail maintenance');
|
||||
await expect(dialog(page).locator('.field > label')).toHaveText(['Scope']);
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(options).toBeFocused();
|
||||
});
|
||||
@@ -36,6 +36,9 @@ const layoutOf = page => page.locator('#workspace-main .settings-layout');
|
||||
async function openOverview(page) {
|
||||
await page.goto(`${workspace}#/system/overview`);
|
||||
await expect(layoutOf(page).locator(':scope > .summary-metrics')).toBeVisible();
|
||||
// The Backup status row is its own read of /backup/status (S9): compare once it answered.
|
||||
await expect(layoutOf(page).locator('.operation').filter({ has: page.locator('h3', { hasText: 'Backup status' }) })
|
||||
.locator('p')).not.toHaveText('Checking backups…');
|
||||
return layoutOf(page);
|
||||
}
|
||||
|
||||
@@ -67,7 +70,7 @@ test('overview shows real counts and Not checked', async ({ page, request }) =>
|
||||
]);
|
||||
expect(await rowsOf(layout, 'Library health')).toEqual([
|
||||
['0 ambiguous image links', 'Review metadata before assigning a positive / negative pair.', 'Review metadata'],
|
||||
['Backup status', 'No retained backup record.', 'View backups'],
|
||||
['Backup status', 'No backup recorded.', 'View backups'],
|
||||
// D129: the fixture's database is configured explicitly, so it reads as config.json's choice and does not move.
|
||||
['Library location', 'browser.db · The database path set in config.json.', 'Details'],
|
||||
['No missing thumbnails', 'Generate only missing previews when necessary.', 'Review thumbnails'],
|
||||
@@ -173,8 +176,8 @@ test('Run health checks opens Health report and updates metric 4', async ({ page
|
||||
await page.goto(`${workspace}#/system/maintenance`);
|
||||
const layout = layoutOf(page);
|
||||
await expect(layout.locator(':scope > .notice h2')).toHaveText('Check first. Review changes. Then apply.');
|
||||
// The rest of Maintenance is still the existing tools' (D94).
|
||||
await expect(layout.locator(':scope > section.handoff h2')).toHaveText('Your current tools are ready');
|
||||
// Maintenance is native (S7): its two surfaces follow the notice.
|
||||
await expect(layout.locator(':scope > section.surface > h2')).toHaveText(['Database', 'Images & cache']);
|
||||
const run = layout.getByRole('button', { name: 'Run health checks', exact: true });
|
||||
await run.click();
|
||||
const dialog = page.locator('dialog#modal[open]');
|
||||
@@ -204,8 +207,11 @@ test('Run health checks opens Health report and updates metric 4', async ({ page
|
||||
.toHaveText(['1 link needs review', 'Passed', 'Not available']);
|
||||
await expect(dialog.locator('.operation h3').filter({ hasText: 'Missing files' }))
|
||||
.toHaveText(['Missing files · output', 'Missing files · input', 'Missing files · unmounted-drive']);
|
||||
// Review missing links goes to Maintenance, where the cleanup's existing tool is linked.
|
||||
// Review missing links reviews a fresh Missing-file links plan (S7); nothing is cleaned up.
|
||||
await dialog.getByRole('button', { name: 'Review missing links', exact: true }).click();
|
||||
await expect(dialog.locator('#modal-title')).toHaveText('Review maintenance plan');
|
||||
await expect(dialog.locator('#modal-body > h3')).toHaveText('Missing-file links');
|
||||
await dialog.getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(page.locator('dialog#modal[open]')).toHaveCount(0);
|
||||
await expect(page).toHaveURL(`${workspace}#/system/maintenance`);
|
||||
await page.locator('#workspace-main .section-nav').getByRole('button', { name: 'Overview', exact: true }).click();
|
||||
@@ -406,11 +412,11 @@ test('out-of-order overview responses never repaint a newer section', async ({ p
|
||||
await nav.getByRole('button', { name: 'Appearance', exact: true }).click();
|
||||
await nav.getByRole('button', { name: 'Overview', exact: true }).click();
|
||||
await expect.poll(held).toBe(2);
|
||||
// Integrations still hands off (S10), so its body is one known panel.
|
||||
await nav.getByRole('button', { name: 'Integrations', exact: true }).click();
|
||||
// Auto-tagging still hands off (S11), so its body is one known panel.
|
||||
await nav.getByRole('button', { name: 'Auto-tagging', exact: true }).click();
|
||||
await expect(await request.post('/__test__/release')).toBeOK();
|
||||
await expect.poll(held).toBe(0);
|
||||
// Both late answers reached a section that had left: Integrations keeps its own body.
|
||||
// Both late answers reached a section that had left: Auto-tagging keeps its own body.
|
||||
await expect(layout.locator(':scope > section.handoff')).toBeVisible();
|
||||
await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve))));
|
||||
await expect(layout.locator('.summary-metrics')).toHaveCount(0);
|
||||
|
||||
@@ -413,7 +413,7 @@ for (const width of [1600, 390]) test(`overview and handoff render with concept
|
||||
|
||||
await sidebarLink(page, 'System').click();
|
||||
// System's Overview is native (S2 Task 6); a section not delivered yet still hands off (D94).
|
||||
await main.locator('.section-nav').getByRole('button', { name: 'Integrations', exact: true }).click();
|
||||
await main.locator('.section-nav').getByRole('button', { name: 'Auto-tagging', exact: true }).click();
|
||||
const handoff = main.locator('section.handoff');
|
||||
await expect(handoff).toHaveClass(/\bsurface\b/);
|
||||
expect(await handoff.evaluate(surfaceStyle)).toMatchObject(SURFACE);
|
||||
@@ -427,7 +427,6 @@ for (const width of [1600, 390]) test(`overview and handoff render with concept
|
||||
const SYSTEM_SECTION_LABELS = ['Overview', 'Appearance', 'Sources & scans', 'Integrations', 'Auto-tagging', 'Maintenance',
|
||||
'Backup & export', 'Diagnostics & logs'];
|
||||
const SYSTEM_HASHES = ['overview', 'appearance', 'sources', 'integrations', 'tagging', 'maintenance', 'backups', 'diagnostics'];
|
||||
const SYSTEM_HANDOFFS = { maintenance: ['/prompt_manager/admin', '/prompt_manager/gallery'] };
|
||||
|
||||
test('System is native: the concept shell, section navigation, sections, hand-offs and the Save row', async ({ page }) => {
|
||||
const mutations = [];
|
||||
@@ -464,20 +463,24 @@ test('System is native: the concept shell, section navigation, sections, hand-of
|
||||
// The section changes in place: focus stays on the pressed section, not the page title.
|
||||
await expect(button).toBeFocused();
|
||||
// S2 Task 6: Overview is native (workspace-system.spec.mjs), and so are Appearance (S3 Task 8) and Sources & scans
|
||||
// (S4 Task 11, workspace-system-sources.spec.mjs, its update notice shown in a fresh browser); Maintenance shows
|
||||
// its health-check notice above the hand-off (D123); every other section hands off.
|
||||
// (S4 Task 11, workspace-system-sources.spec.mjs, its update notice shown in a fresh browser), Integrations (S10
|
||||
// Task 27, workspace-system-integrations.spec.mjs), Maintenance (S7 Task 19,
|
||||
// workspace-system-maintenance.spec.mjs) and Backup & export (S9 Task 24, workspace-system-backups.spec.mjs);
|
||||
// every other section hands off.
|
||||
if (hash === 'overview') await expect(layout.locator(':scope > .summary-metrics')).toBeVisible();
|
||||
if (hash === 'sources') await expect(layout.locator('#source-list .operation')).toHaveCount(2);
|
||||
expect(await layout.evaluate(node => [...node.children].map(child => `${child.localName}.${child.className}`)))
|
||||
.toEqual({ overview: ['div.summary-metrics', 'div.two-col', 'div.save-row'], appearance: ['section.surface', 'section.surface', 'div.save-row'],
|
||||
sources: ['div.notice', 'section.surface', 'div.save-row'],
|
||||
maintenance: ['div.notice', 'section.surface handoff', 'div.save-row'] }[hash]
|
||||
integrations: ['section.surface', 'section.surface', 'div.save-row'],
|
||||
maintenance: ['div.notice', 'section.surface', 'section.surface', 'div.save-row'],
|
||||
backups: ['div.two-col', 'section.surface', 'div.save-row'] }[hash]
|
||||
?? ['section.surface handoff', 'div.save-row']);
|
||||
if (['overview', 'appearance', 'sources'].includes(hash)) continue;
|
||||
if (['overview', 'appearance', 'sources', 'integrations', 'maintenance', 'backups'].includes(hash)) continue;
|
||||
const handoff = layout.locator('section.handoff');
|
||||
await expect(handoff.locator('h2')).toHaveText('Your current tools are ready');
|
||||
expect(await handoff.locator('a.workspace-link').evaluateAll(links => links.map(link => link.getAttribute('href'))))
|
||||
.toEqual(SYSTEM_HANDOFFS[hash] ?? ['/prompt_manager/admin']);
|
||||
.toEqual(['/prompt_manager/admin']);
|
||||
await expect(layout.locator('#settings-status')).toHaveText('No unsaved changes.');
|
||||
await expect(page.locator('aside.sidebar a[href="#/system"]')).toHaveAttribute('aria-current', 'page');
|
||||
}
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
"""System › Integrations test hooks for the browser tests (#249 System S10).
|
||||
|
||||
A fake LoraManager install lives under this server's temp root; nothing ever
|
||||
reaches a real LoraManager, ComfyUI install or the network:
|
||||
|
||||
- ``POST /__test__/integrations/reset`` turns the integration off (no path,
|
||||
no key, no trigger words), removes the fake install, its LoRA folder and
|
||||
the image cache, forgets every LoRA import and injected failure, and opens
|
||||
the download gate;
|
||||
- ``POST /__test__/integrations/lora-manager`` ``{loras, civitai, previous}``
|
||||
writes ``custom_nodes/ComfyUI-Lora-Manager`` (detected through the
|
||||
``folder_paths`` stub's base path) and ``loras`` metadata files under
|
||||
``models/loras``, each with a local preview PNG; ``civitai`` adds one
|
||||
CivitAI example image per LoRA (served by the stub opener below);
|
||||
``previous`` saves that many earlier ``lora-manager`` imports with tags and
|
||||
image links;
|
||||
- ``POST /__test__/integrations/gate`` ``{hold}`` holds every CivitAI
|
||||
download until it is released (``{"hold": false}``), so a test sees the job
|
||||
while it downloads;
|
||||
- ``POST /__test__/integrations/fail-import`` makes the next collection
|
||||
replacement fail (a database error after staging);
|
||||
- ``POST /__test__/integrations/pair`` saves a pair whose positive is the first
|
||||
imported prompt (a saved pair that blocks the replacement);
|
||||
- ``GET /__test__/integrations/state`` answers the imported prompt texts,
|
||||
whether a CivitAI key is saved and the downloads served.
|
||||
|
||||
CivitAI downloads are always answered by a stub opener (a small PNG), so no
|
||||
test ever opens a network connection.
|
||||
"""
|
||||
|
||||
import io
|
||||
import json
|
||||
import shutil
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
from PIL import Image
|
||||
|
||||
MAX_LORAS = 20
|
||||
CATEGORY = "lora-manager"
|
||||
|
||||
|
||||
def _png():
|
||||
buffer = io.BytesIO()
|
||||
Image.new("RGB", (8, 8), "#3a6ea5").save(buffer, "PNG")
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
class _Response:
|
||||
def __init__(self, body):
|
||||
self._body = io.BytesIO(body)
|
||||
self.headers = {"Content-Length": str(len(body))}
|
||||
|
||||
def read(self, amount=-1):
|
||||
return self._body.read(amount)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
|
||||
def register_integrations_commands(app, routes, api, root: Path):
|
||||
from py import lora_utils
|
||||
from py.config import IntegrationConfig
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
lora_manager = root / "custom_nodes" / "ComfyUI-Lora-Manager"
|
||||
loras = root / "models" / "loras"
|
||||
gate = threading.Event()
|
||||
gate.set()
|
||||
served = []
|
||||
failures = {"replace": 0}
|
||||
pairs = [] # the blocking pairs this fixture saved
|
||||
real_replace = type(api.db).replace_category_prompts
|
||||
|
||||
def stub_open(request, timeout=10):
|
||||
served.append(request.full_url)
|
||||
gate.wait(30)
|
||||
return _Response(_png())
|
||||
|
||||
lora_utils._open_url = stub_open
|
||||
|
||||
def replace(db, *args, **kwargs):
|
||||
if failures["replace"] > 0:
|
||||
failures["replace"] -= 1
|
||||
raise RuntimeError("Injected replacement failure")
|
||||
return real_replace(db, *args, **kwargs)
|
||||
|
||||
type(api.db).replace_category_prompts = replace
|
||||
|
||||
def reset():
|
||||
IntegrationConfig.LORA_MANAGER_ENABLED = False
|
||||
IntegrationConfig.LORA_MANAGER_PATH = ""
|
||||
IntegrationConfig.LORA_TRIGGER_WORDS_ENABLED = False
|
||||
IntegrationConfig.CIVITAI_API_KEY = ""
|
||||
lora_utils.get_trigger_cache().clear()
|
||||
gate.set()
|
||||
served.clear()
|
||||
failures["replace"] = 0
|
||||
cache = Path(lora_utils.get_lora_image_cache_dir(create=False))
|
||||
for folder in (lora_manager, loras, cache):
|
||||
shutil.rmtree(folder, ignore_errors=True)
|
||||
for pair_id in pairs:
|
||||
api.db.delete_prompt_pair(pair_id)
|
||||
pairs.clear()
|
||||
real_replace(api.db, CATEGORY, [], allow_empty=True)
|
||||
|
||||
@routes.post("/__test__/integrations/reset")
|
||||
async def integrations_reset(request):
|
||||
reset()
|
||||
return web.json_response({"success": True})
|
||||
|
||||
@routes.post("/__test__/integrations/lora-manager")
|
||||
async def make_lora_manager(request):
|
||||
data = await request.json() if request.can_read_body else {}
|
||||
count = data.get("loras", 2)
|
||||
previous = data.get("previous", 0)
|
||||
for value in (count, previous):
|
||||
if isinstance(value, bool) or not isinstance(value, int):
|
||||
raise web.HTTPBadRequest()
|
||||
if not 0 <= count <= MAX_LORAS or not 0 <= previous <= MAX_LORAS:
|
||||
raise web.HTTPBadRequest()
|
||||
(lora_manager / "py").mkdir(parents=True, exist_ok=True)
|
||||
(lora_manager / "__init__.py").write_text("")
|
||||
loras.mkdir(parents=True, exist_ok=True)
|
||||
for number in range(count):
|
||||
stem = f"style_{number + 1:02}"
|
||||
civitai = {"trainedWords": [f"trigger{number + 1}"]}
|
||||
if data.get("civitai") is True:
|
||||
civitai["images"] = [
|
||||
{
|
||||
"url": f"https://image.civitai.com/x/{stem}.jpeg",
|
||||
"meta": {"prompt": f"{stem} example prompt"},
|
||||
}
|
||||
]
|
||||
meta = {
|
||||
"file_name": f"{stem}.safetensors",
|
||||
"model_name": f"Style {number + 1:02}",
|
||||
"civitai": civitai,
|
||||
}
|
||||
(loras / f"{stem}.metadata.json").write_text(json.dumps(meta))
|
||||
(loras / f"{stem}.png").write_bytes(_png())
|
||||
for number in range(previous):
|
||||
text = f"previous lora import {number + 1}"
|
||||
prompt_id = api.db.save_prompt(
|
||||
text=text,
|
||||
category=CATEGORY,
|
||||
tags=[CATEGORY, f"lora:previous-{number + 1}"],
|
||||
prompt_hash=generate_prompt_hash(text),
|
||||
)
|
||||
api.db.link_image_to_prompt(str(prompt_id), str(loras / "old.png"))
|
||||
return web.json_response({"success": True})
|
||||
|
||||
@routes.post("/__test__/integrations/gate")
|
||||
async def download_gate(request):
|
||||
data = await request.json()
|
||||
if not isinstance(data.get("hold"), bool):
|
||||
raise web.HTTPBadRequest()
|
||||
if data["hold"]:
|
||||
gate.clear()
|
||||
else:
|
||||
gate.set()
|
||||
return web.json_response({"success": True})
|
||||
|
||||
@routes.post("/__test__/integrations/fail-import")
|
||||
async def fail_import(request):
|
||||
failures["replace"] = 1
|
||||
return web.json_response({"success": True})
|
||||
|
||||
@routes.post("/__test__/integrations/pair")
|
||||
async def pair(request):
|
||||
with api.db.model.get_connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT text FROM prompts WHERE category = ? ORDER BY id LIMIT 1",
|
||||
(CATEGORY,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise web.HTTPConflict()
|
||||
pairs.append(api.db.save_prompt_pair(row[0], "blurry"))
|
||||
return web.json_response({"success": True})
|
||||
|
||||
@routes.get("/__test__/integrations/state")
|
||||
async def state(request):
|
||||
imports = api.db.search_prompts(category=CATEGORY, limit=MAX_LORAS * 2)
|
||||
return web.json_response(
|
||||
{
|
||||
"imports": sorted(prompt["text"] for prompt in imports),
|
||||
"key_saved": bool(IntegrationConfig.CIVITAI_API_KEY),
|
||||
"enabled": IntegrationConfig.LORA_MANAGER_ENABLED,
|
||||
"trigger_words": IntegrationConfig.LORA_TRIGGER_WORDS_ENABLED,
|
||||
"downloads": len(served),
|
||||
}
|
||||
)
|
||||
|
||||
return reset
|
||||
@@ -37,10 +37,33 @@ read. These per-server hooks only change what a System test needs on top:
|
||||
server read as having (``cjxl``) or lacking a JPEG XL encoder without
|
||||
probing; reset restores the thumbnail settings and the real detection.
|
||||
|
||||
- ``POST /__test__/system/duplicate-prompts`` (S7) adds two prompts whose
|
||||
text differs only in case and spacing (a duplicate prompt text group);
|
||||
``{"again": true}`` adds a third, so a reviewed plan no longer matches;
|
||||
- ``POST /__test__/system/duplicate-files`` (S7) writes one PNG's bytes three
|
||||
times: ``output/s7-dupes/same.png``, ``input/s7-dupes/same.png`` (the same
|
||||
relative path in the other source) and ``output/s7-dupes/copy.png``;
|
||||
``{"groups": n}`` adds ``n`` more groups of two copies each, and
|
||||
``{"trio": false}`` leaves the three ``same.png``/``copy.png`` copies out,
|
||||
and ``{"long": true}`` adds a group under long names with no spaces;
|
||||
- ``GET /__test__/system/digest`` (S7) answers the sha256 of the database's
|
||||
rows (a read-only dump) and of every media file under ``output`` and
|
||||
``input`` (not the automatic previews in ``thumbnails``), so a test proves
|
||||
a Cancel wrote nothing.
|
||||
- ``POST /__test__/system/safety-backup`` (S9) writes one verified safety
|
||||
backup beside the database, as a maintenance apply does;
|
||||
``GET /__test__/system/backups`` answers ``{safety: [names], stages: n}``:
|
||||
the ``<db>.backup_*`` files and the held ``.pm-restore-*`` stage folders.
|
||||
Reset drops every held restore review, removes the safety backups and
|
||||
forgets the session's last download.
|
||||
|
||||
Delays use the shared request holds (``request_hold.py``).
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
@@ -102,8 +125,12 @@ def register_system_commands(app, routes, api, root: Path):
|
||||
|
||||
unplugged = root / "output" / "unplugged"
|
||||
|
||||
dupes = [root / "output" / "s7-dupes", root / "input" / "s7-dupes"]
|
||||
|
||||
def reset():
|
||||
failures.clear()
|
||||
for folder in dupes:
|
||||
shutil.rmtree(folder, ignore_errors=True)
|
||||
if unplugged.is_dir():
|
||||
unplugged.rmdir()
|
||||
GalleryConfig.MONITORING_DIRECTORIES = list(configured)
|
||||
@@ -130,11 +157,45 @@ def register_system_commands(app, routes, api, root: Path):
|
||||
staged.write_bytes(config_bytes)
|
||||
os.replace(staged, config_path)
|
||||
|
||||
def backup_files():
|
||||
db_path = Path(api.db.model.db_path)
|
||||
safety = sorted(
|
||||
path.name
|
||||
for path in db_path.parent.glob(db_path.name + ".backup_*")
|
||||
if path.is_file()
|
||||
)
|
||||
stages = [
|
||||
path
|
||||
for path in db_path.parent.glob(".pm-restore-*")
|
||||
if path.is_dir() and not path.name.startswith(".pm-restore-upload-")
|
||||
]
|
||||
return db_path, safety, stages
|
||||
|
||||
async def reset_backups():
|
||||
await api._discard_held(api._restore_plans.pop_all())
|
||||
db_path, safety, _stages = backup_files()
|
||||
for name in safety:
|
||||
(db_path.parent / name).unlink(missing_ok=True)
|
||||
api._last_backup_download = None
|
||||
|
||||
@routes.post("/__test__/system/reset")
|
||||
async def system_reset(request):
|
||||
reset()
|
||||
await reset_backups()
|
||||
return web.json_response({"success": True})
|
||||
|
||||
@routes.post("/__test__/system/safety-backup")
|
||||
async def safety_backup(request):
|
||||
from py.api.maintenance import take_safety_backup
|
||||
|
||||
name = await api._run_in_executor(take_safety_backup, api.db.model)
|
||||
return web.json_response({"success": True, "name": name})
|
||||
|
||||
@routes.get("/__test__/system/backups")
|
||||
async def backups(request):
|
||||
_db_path, safety, stages = backup_files()
|
||||
return web.json_response({"safety": safety, "stages": len(stages)})
|
||||
|
||||
@routes.post("/__test__/system/fail")
|
||||
async def system_fail(request):
|
||||
data = await request.json()
|
||||
@@ -254,6 +315,75 @@ def register_system_commands(app, routes, api, root: Path):
|
||||
)
|
||||
return web.json_response({"success": True})
|
||||
|
||||
@routes.post("/__test__/system/duplicate-prompts")
|
||||
async def duplicate_prompts(request):
|
||||
data = await request.json() if request.can_read_body else {}
|
||||
texts = [("S7 duplicate prompt", "2001-01-01 00:00:00")]
|
||||
texts.append((" s7 DUPLICATE prompt", "2002-01-01 00:00:00"))
|
||||
if data.get("again") is True:
|
||||
texts = [("S7 Duplicate Prompt ", "2003-01-01 00:00:00")]
|
||||
with api.db.model.get_connection() as conn:
|
||||
conn.executemany(
|
||||
"INSERT INTO prompts(text, created_at) VALUES(?, ?)", texts
|
||||
)
|
||||
conn.commit()
|
||||
return web.json_response({"success": True})
|
||||
|
||||
@routes.post("/__test__/system/duplicate-files")
|
||||
async def duplicate_files(request):
|
||||
body = await request.json() if request.can_read_body else {}
|
||||
for folder in dupes:
|
||||
folder.mkdir(exist_ok=True)
|
||||
extra = body.get("groups", 0)
|
||||
if (
|
||||
isinstance(extra, bool)
|
||||
or not isinstance(extra, int)
|
||||
or not 0 <= extra <= 20
|
||||
):
|
||||
raise web.HTTPBadRequest()
|
||||
for number in range(extra):
|
||||
# One more group each: two copies of a picture no other file has.
|
||||
with Image.new("RGB", (16, 16), (number * 9, 40, 90)) as image:
|
||||
image.save(dupes[0] / f"group_{number:02}_a.png")
|
||||
copy = (dupes[0] / f"group_{number:02}_a.png").read_bytes()
|
||||
(dupes[0] / f"group_{number:02}_b.png").write_bytes(copy)
|
||||
if body.get("long") is True:
|
||||
# One group whose copies sit under long folder and file names with
|
||||
# no space to break at, as real dataset folders do.
|
||||
deep = dupes[0] / ("mabes9sans_DATASET_ideogram_v1_" * 3).rstrip("_")
|
||||
deep.mkdir(exist_ok=True)
|
||||
with Image.new("RGB", (16, 16), "#204060") as image:
|
||||
image.save(deep / ("mabes9sans_DATASET072_" * 4 + "a.png"))
|
||||
copy = (deep / ("mabes9sans_DATASET072_" * 4 + "a.png")).read_bytes()
|
||||
(deep / ("mabes9sans_DATASET072_" * 4 + "b.png")).write_bytes(copy)
|
||||
if body.get("trio", True) is not False:
|
||||
with Image.new("RGB", (48, 48), "#7a3b2e") as image:
|
||||
image.save(dupes[0] / "same.png")
|
||||
data = (dupes[0] / "same.png").read_bytes()
|
||||
(dupes[1] / "same.png").write_bytes(data)
|
||||
(dupes[0] / "copy.png").write_bytes(data)
|
||||
api.invalidate_gallery_cache()
|
||||
return web.json_response({"success": True})
|
||||
|
||||
@routes.get("/__test__/system/digest")
|
||||
async def digest(request):
|
||||
uri = Path(api.db.model.db_path).as_uri() + "?mode=ro"
|
||||
rows = hashlib.sha256()
|
||||
conn = sqlite3.connect(uri, uri=True)
|
||||
try:
|
||||
for line in conn.iterdump():
|
||||
rows.update(line.encode("utf-8") + b"\n")
|
||||
finally:
|
||||
conn.close()
|
||||
media = hashlib.sha256()
|
||||
for name in ("output", "input"):
|
||||
for path in sorted((root / name).rglob("*")):
|
||||
# Automatic previews are written in the background (#316): media only.
|
||||
if path.is_file() and "thumbnails" not in path.relative_to(root).parts:
|
||||
media.update(str(path.relative_to(root)).encode("utf-8"))
|
||||
media.update(hashlib.sha256(path.read_bytes()).digest())
|
||||
return web.json_response({"rows": rows.hexdigest(), "media": media.hexdigest()})
|
||||
|
||||
@routes.post("/__test__/system/library")
|
||||
async def library_state(request):
|
||||
data = await request.json()
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
// #343: a link drives a PromptManager node's text (and the Pair's negative) on the real pinned host.
|
||||
// The workflow is built by the real frontend through the nodes' sockets and queued as the frontend serializes it;
|
||||
// only inference is replaced (fixtures/ledger_nodes.py). The linked text must be saved, counted, recorded in the
|
||||
// execution history, and the run's image must link to it.
|
||||
import { execFile } from 'node:child_process';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { promisify } from 'node:util';
|
||||
import { expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, readExecutionRuns } from './execution-fixture.mjs';
|
||||
import { installHostHooks } from './host-fixture.mjs';
|
||||
|
||||
// The worker's owned host must be up before the shared host hooks open the page.
|
||||
test.beforeEach(async ({ execution }) => { expect(execution.databasePath).toBeTruthy(); });
|
||||
installHostHooks(test, ['PromptManager', 'PromptManagerText', 'PromptManagerFocused', 'PromptManagerPair', 'PMHistoryInputs']);
|
||||
|
||||
// `primitive`: a core string node the queue-time resolver follows. `legacy`: the frontend's virtual Primitive node,
|
||||
// whose value is queued as literal text. `preview`: PrimitiveString -> PreviewAny, the shape of the maintainer's
|
||||
// TextGenerate -> PreviewAny chain, whose text is only known when the node runs.
|
||||
function build(page, { kind, positive, negative, prefix }) {
|
||||
return page.evaluate(async ({ kind, positive, negative, prefix }) => {
|
||||
const { app } = await import('/scripts/app.js');
|
||||
app.graph.clear();
|
||||
const add = (type, x, y) => { const node = LiteGraph.createNode(type); app.graph.add(node); node.pos = [x, y]; return node; };
|
||||
const slot = (node, name) => { const index = node.inputs.findIndex(input => input.name === name); if (index < 0) throw new Error(`${node.type} has no ${name} socket`); return index; };
|
||||
const connect = (from, output, to, name) => { if (!from.connect(output, to, slot(to, name))) throw new Error(`${from.type} could not link to ${to.type}.${name}`); };
|
||||
const inputs = add('PMHistoryInputs', 0, 0);
|
||||
const prompt = add(kind, 700, 0);
|
||||
const set = (name, value) => { const widget = prompt.widgets.find(item => item.name === name); widget.value = value; if (widget.inputEl) widget.inputEl.value = value; };
|
||||
set('text', positive.typed);
|
||||
if (negative) set('negative_text', negative.typed);
|
||||
const drive = (name, side, y) => {
|
||||
if (!side.via) return;
|
||||
if (side.via === 'legacy') {
|
||||
// The frontend's own Primitive: a virtual node whose value is applied when the workflow is queued.
|
||||
const primitive = add('PrimitiveNode', 0, y);
|
||||
connect(primitive, 0, prompt, name);
|
||||
primitive.widgets[0].value = side.text;
|
||||
return;
|
||||
}
|
||||
const text = add('PrimitiveString', 0, y);
|
||||
text.widgets.find(widget => widget.name === 'value').value = side.text;
|
||||
let source = text;
|
||||
if (side.via === 'preview') { source = add('PreviewAny', 350, y); connect(text, 0, source, 'source'); }
|
||||
connect(source, 0, prompt, name);
|
||||
};
|
||||
drive('text', positive, 300);
|
||||
if (negative) drive('negative_text', negative, 500);
|
||||
if (kind !== 'PromptManagerText') connect(inputs, 0, prompt, 'clip');
|
||||
const sampler = add('KSampler', 1400, 0);
|
||||
connect(inputs, 1, sampler, 'model');
|
||||
connect(inputs, 2, sampler, 'latent_image');
|
||||
if (kind === 'PromptManagerText') {
|
||||
const encode = add('CLIPTextEncode', 1100, 0);
|
||||
encode.convertWidgetToInput?.(encode.widgets.find(widget => widget.name === 'text'));
|
||||
connect(inputs, 0, encode, 'clip');
|
||||
connect(prompt, 0, encode, 'text');
|
||||
connect(encode, 0, sampler, 'positive');
|
||||
} else connect(prompt, 0, sampler, 'positive');
|
||||
if (kind === 'PromptManagerPair') connect(prompt, 1, sampler, 'negative');
|
||||
else {
|
||||
const empty = add('CLIPTextEncode', 1100, 300);
|
||||
connect(inputs, 0, empty, 'clip');
|
||||
connect(empty, 0, sampler, 'negative');
|
||||
}
|
||||
const images = add('PMHistoryImages', 1800, 0);
|
||||
images.widgets.find(widget => widget.name === 'value').value = 99;
|
||||
connect(sampler, 0, images, 'latent');
|
||||
const save = add('SaveImage', 2100, 0);
|
||||
save.widgets.find(widget => widget.name === 'filename_prefix').value = prefix;
|
||||
connect(images, 0, save, 'images');
|
||||
// The Classic nodes install their own text serializer shortly after creation; queue after it, as a user does.
|
||||
await prompt._pmTextSerializerReady;
|
||||
const { output, workflow } = await app.graphToPrompt();
|
||||
return { output, workflow, promptNode: String(prompt.id) };
|
||||
}, { kind, positive, negative, prefix });
|
||||
}
|
||||
|
||||
async function run(execution, page, testCase) {
|
||||
const prefix = `textlink/${randomUUID()}`;
|
||||
const { output, workflow, promptNode } = await build(page, { ...testCase, prefix });
|
||||
const id = randomUUID();
|
||||
const response = await execution.request.post('/prompt', { data: { prompt: output, prompt_id: id, extra_data: { extra_pnginfo: { workflow } } } });
|
||||
expect(response.ok(), await response.text()).toBe(true);
|
||||
let item;
|
||||
await expect.poll(async () => {
|
||||
item = (await (await execution.request.get(`/history/${id}`)).json())[id];
|
||||
return item?.status?.status_str ?? null;
|
||||
}, { timeout: 20000 }).toBe('success');
|
||||
let result;
|
||||
await expect.poll(async () => {
|
||||
result = await readExecutionRuns(execution.databasePath, id);
|
||||
const run = result.rows[0];
|
||||
return run ? result.captures[run.run_id].state : null;
|
||||
}, { timeout: 10000, message: 'the run and its prompt context reach SQLite' }).toBe('complete');
|
||||
const run = result.rows[0];
|
||||
const files = Object.values(item.outputs).flatMap(value => value.images || []).map(image => image.filename);
|
||||
return { output, promptNode, context: result.captures[run.run_id].contexts[0], usage: result.usage[run.run_id], files };
|
||||
}
|
||||
|
||||
async function scan(execution) {
|
||||
const response = await execution.request.post('/prompt_manager/scan', { timeout: 60000 });
|
||||
expect(response.ok()).toBe(true);
|
||||
const events = (await response.text()).split('\n').filter(line => line.startsWith('data: ')).map(line => JSON.parse(line.slice(6)));
|
||||
expect(events.at(-1)?.type, JSON.stringify(events.at(-1))).toBe('complete');
|
||||
}
|
||||
|
||||
// filename -> linked prompt id (null when the image is not linked), read-only from the owned host's library.
|
||||
const LINKS = `import json, sqlite3, sys, pathlib
|
||||
db = sqlite3.connect(pathlib.Path(sys.argv[1]).resolve().as_uri() + "?mode=ro", uri=True)
|
||||
links = {name: None for name in sys.argv[2:]}
|
||||
for name, prompt in db.execute("SELECT filename, prompt_id FROM generated_images WHERE filename IN (%s)" % ",".join("?" * (len(sys.argv) - 2)), sys.argv[2:]):
|
||||
links[name] = prompt
|
||||
print(json.dumps(links))`;
|
||||
async function linkedPrompts(execution, files) {
|
||||
const { stdout } = await promisify(execFile)(process.env.PM_TEST_PYTHON || 'python', ['-c', LINKS, execution.databasePath, ...files], { timeout: 5000 });
|
||||
return JSON.parse(stdout);
|
||||
}
|
||||
|
||||
// Text a link resolves from the queued graph (a primitive) links its images through their embedded metadata.
|
||||
// Text only known at run time (PreviewAny) cannot be read from the image: its attribution from the recorded run is
|
||||
// #342's. Here it must never be linked to another prompt.
|
||||
async function expectImagesLinked(execution, evidence, viaRuntime) {
|
||||
await scan(execution);
|
||||
const links = await linkedPrompts(execution, evidence.files);
|
||||
expect(evidence.files.length).toBeGreaterThan(0);
|
||||
for (const [file, prompt] of Object.entries(links)) {
|
||||
if (viaRuntime) expect([null, evidence.context.positive_prompt_id], file).toContain(prompt);
|
||||
else expect(prompt, file).toBe(evidence.context.positive_prompt_id);
|
||||
}
|
||||
}
|
||||
|
||||
const unique = label => `${label} ${randomUUID()}`;
|
||||
|
||||
const sources = { primitive: 'PrimitiveString', legacy: 'a legacy Primitive', preview: 'PreviewAny' };
|
||||
for (const kind of ['PromptManager', 'PromptManagerText', 'PromptManagerFocused', 'PromptManagerSplit']) {
|
||||
for (const via of Object.keys(sources)) {
|
||||
test(`${kind} saves, counts, records and links images to text linked from ${sources[via]}`, async ({ execution, page }) => {
|
||||
test.setTimeout(90000);
|
||||
const text = unique(`Linked ${kind} ${via}`);
|
||||
const evidence = await run(execution, page, { kind, positive: { typed: 'Typed under the link', text, via } });
|
||||
const source = evidence.output[evidence.promptNode].inputs.text;
|
||||
// A real node is queued as a link; the virtual Primitive as its own value, never the hidden typed text.
|
||||
if (via === 'legacy') expect(source).toBe(text);
|
||||
else expect(Array.isArray(source), 'the frontend queued the link, not the typed text').toBe(true);
|
||||
expect(evidence.context.positive.raw).toBe(text);
|
||||
expect(evidence.context.positive.effective).toBe(text);
|
||||
expect(evidence.usage).toEqual([{ role: 'positive', run_count: 1 }]);
|
||||
const saved = await (await execution.request.get(`/prompt_manager/library/${evidence.context.positive_prompt_id}`)).json();
|
||||
expect(saved.positive.text).toBe(text);
|
||||
await expectImagesLinked(execution, evidence, via === 'preview');
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// A legacy Primitive's wildcard is drawn once when the workflow is queued: the run saves and counts what was queued.
|
||||
for (const kind of ['PromptManager', 'PromptManagerFocused']) {
|
||||
test(`${kind} saves and counts the text a wildcard legacy Primitive queued (#343 re-review R1)`, async ({ execution, page }) => {
|
||||
test.setTimeout(90000);
|
||||
const stem = unique(`Wildcard ${kind}`);
|
||||
const evidence = await run(execution, page, { kind, positive: { typed: 'Typed under the link', text: `${stem} {alpha|beta}`, via: 'legacy' } });
|
||||
const queuedText = evidence.output[evidence.promptNode].inputs.text;
|
||||
// The Classic node keeps its widget binding: the host applies the Primitive, and its own text serializer
|
||||
// queues the text as written, as on main. The Focused editor queues one choice.
|
||||
if (kind === 'PromptManager') expect(queuedText).toBe(`${stem} {alpha|beta}`);
|
||||
else expect([`${stem} alpha`, `${stem} beta`]).toContain(queuedText);
|
||||
expect(evidence.context.positive.raw).toBe(queuedText);
|
||||
expect(evidence.usage).toEqual([{ role: 'positive', run_count: 1 }]);
|
||||
const saved = await (await execution.request.get(`/prompt_manager/library/${evidence.context.positive_prompt_id}`)).json();
|
||||
expect(saved.positive.text).toBe(queuedText);
|
||||
});
|
||||
}
|
||||
|
||||
for (const [label, linked, negativeVia] of [['positive', ['text'], 'primitive'], ['negative', ['negative_text'], 'primitive'],
|
||||
['both', ['text', 'negative_text'], 'primitive'], ['neither', [], 'primitive'], ['negative from a legacy Primitive', ['negative_text'], 'legacy']]) {
|
||||
test(`PromptManagerPair with ${label} linked saves, counts and records the pair and links its images`, async ({ execution, page }) => {
|
||||
test.setTimeout(90000);
|
||||
const sides = {
|
||||
text: { typed: unique('Typed positive'), text: unique('Linked positive'), via: linked.includes('text') ? 'preview' : null },
|
||||
negative_text: { typed: unique('Typed negative'), text: unique('Linked negative'), via: linked.includes('negative_text') ? negativeVia : null },
|
||||
};
|
||||
const evidence = await run(execution, page, { kind: 'PromptManagerPair', positive: sides.text, negative: sides.negative_text });
|
||||
const expected = name => sides[name].via ? sides[name].text : sides[name].typed;
|
||||
for (const name of ['text', 'negative_text']) {
|
||||
const queuedValue = evidence.output[evidence.promptNode].inputs[name];
|
||||
if (sides[name].via === 'legacy') expect(queuedValue, name).toBe(sides[name].text);
|
||||
else expect(Array.isArray(queuedValue), name).toBe(linked.includes(name));
|
||||
}
|
||||
expect(evidence.context.positive.raw).toBe(expected('text'));
|
||||
expect(evidence.context.negative.raw).toBe(expected('negative_text'));
|
||||
expect(evidence.usage).toEqual([{ role: 'positive', run_count: 1 }, { role: 'negative', run_count: 0 }, { role: 'pair', run_count: 1 }]);
|
||||
const pair = await (await execution.request.get(`/prompt_manager/pairs/${evidence.context.pair_id}`)).json();
|
||||
expect(pair.pair).toMatchObject({ positive_text: expected('text'), negative_text: expected('negative_text') });
|
||||
await expectImagesLinked(execution, evidence, sides.text.via === 'preview');
|
||||
});
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Deterministic nodes and queue controls installed only in an owned test host."""
|
||||
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
|
||||
@@ -255,3 +256,50 @@ async def cache_mode(request: web.Request) -> web.Response:
|
||||
raise web.HTTPBadRequest()
|
||||
_CACHE_MODE = mode
|
||||
return web.json_response({"configured": True})
|
||||
|
||||
|
||||
def _plugin_module(suffix: str):
|
||||
"""A loaded PromptManager module (ComfyUI names the package by its path)."""
|
||||
import sys
|
||||
|
||||
for name, module in list(sys.modules.items()):
|
||||
if name.endswith("promptmanager." + suffix):
|
||||
return module
|
||||
raise web.HTTPServiceUnavailable(text=f"PromptManager {suffix} not loaded")
|
||||
|
||||
|
||||
@PromptServer.instance.routes.post("/pm_execution_fixture/monitor")
|
||||
async def image_monitor(request: web.Request) -> web.Response:
|
||||
"""Start or stop PromptManager's image monitor on the host output folder.
|
||||
|
||||
The managed host's config disables monitoring; only the attribution cases
|
||||
(#342) turn it on, and they turn it off again.
|
||||
"""
|
||||
enabled = (await request.json())["enabled"]
|
||||
if type(enabled) is not bool:
|
||||
raise web.HTTPBadRequest()
|
||||
import folder_paths
|
||||
|
||||
monitor_module = _plugin_module("utils.image_monitor")
|
||||
config = _plugin_module("py.config").GalleryConfig
|
||||
monitor = monitor_module._monitor_instance
|
||||
if monitor is None:
|
||||
database = _plugin_module("database.operations").PromptDatabase()
|
||||
tracker = _plugin_module("utils.prompt_tracker").get_prompt_tracker(database)
|
||||
monitor = monitor_module.get_image_monitor(database, tracker)
|
||||
if enabled:
|
||||
config.MONITORING_ENABLED = True
|
||||
monitor.start_monitoring([folder_paths.get_output_directory()])
|
||||
else:
|
||||
monitor.stop_monitoring()
|
||||
config.MONITORING_ENABLED = False
|
||||
return web.json_response({"monitoring": monitor.is_monitoring})
|
||||
|
||||
|
||||
@PromptServer.instance.routes.get("/pm_execution_fixture/monitor")
|
||||
async def image_monitor_state(request: web.Request) -> web.Response:
|
||||
"""File names of keyless images waiting for their run's recorded outputs."""
|
||||
monitor = _plugin_module("utils.image_monitor")._monitor_instance
|
||||
handler = getattr(monitor, "handler", None)
|
||||
waiting = handler.awaiting_history() if handler is not None else []
|
||||
return web.json_response({"awaiting": [os.path.basename(p) for p in waiting]})
|
||||
|
||||
@@ -0,0 +1,379 @@
|
||||
{
|
||||
"id": "37b6baef-94f5-49d2-9b42-1ffc70e7da92",
|
||||
"revision": 0,
|
||||
"last_node_id": 4,
|
||||
"last_link_id": 2,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "PromptManagerFocused",
|
||||
"pos": [
|
||||
600,
|
||||
0
|
||||
],
|
||||
"size": [
|
||||
550,
|
||||
650
|
||||
],
|
||||
"flags": {},
|
||||
"order": 2,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"localized_name": "clip",
|
||||
"name": "clip",
|
||||
"type": "CLIP",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "prepend_text",
|
||||
"name": "prepend_text",
|
||||
"shape": 7,
|
||||
"type": "STRING",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "append_text",
|
||||
"name": "append_text",
|
||||
"shape": 7,
|
||||
"type": "STRING",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "text",
|
||||
"name": "text",
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "text"
|
||||
},
|
||||
"link": 1
|
||||
},
|
||||
{
|
||||
"localized_name": "category",
|
||||
"name": "category",
|
||||
"shape": 7,
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "category"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "tags",
|
||||
"name": "tags",
|
||||
"shape": 7,
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "tags"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "search_text",
|
||||
"name": "search_text",
|
||||
"shape": 7,
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "search_text"
|
||||
},
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"localized_name": "CONDITIONING",
|
||||
"name": "CONDITIONING",
|
||||
"type": "CONDITIONING",
|
||||
"links": null
|
||||
},
|
||||
{
|
||||
"localized_name": "STRING",
|
||||
"name": "STRING",
|
||||
"type": "STRING",
|
||||
"links": null
|
||||
}
|
||||
],
|
||||
"title": "Focused fed by a Primitive",
|
||||
"properties": {
|
||||
"Node name for S&R": "PromptManagerFocused",
|
||||
"focusedUI": {
|
||||
"libraryOpen": false,
|
||||
"libraryFilters": {
|
||||
"category": "",
|
||||
"tags": ""
|
||||
},
|
||||
"composition": false,
|
||||
"metadata": false,
|
||||
"preview": false
|
||||
}
|
||||
},
|
||||
"widgets_values": [
|
||||
"Primitive focused text",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
""
|
||||
],
|
||||
"widgets_values_named": {
|
||||
"text": "Primitive focused text",
|
||||
"category": "",
|
||||
"tags": "",
|
||||
"search_text": "",
|
||||
"prepend_text": "",
|
||||
"append_text": "",
|
||||
"prompt_manager_focused_ui": ""
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "PromptManagerPair",
|
||||
"pos": [
|
||||
600,
|
||||
1200
|
||||
],
|
||||
"size": [
|
||||
1100,
|
||||
700
|
||||
],
|
||||
"flags": {},
|
||||
"order": 3,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"localized_name": "clip",
|
||||
"name": "clip",
|
||||
"type": "CLIP",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "prepend_text",
|
||||
"name": "prepend_text",
|
||||
"shape": 7,
|
||||
"type": "STRING",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "append_text",
|
||||
"name": "append_text",
|
||||
"shape": 7,
|
||||
"type": "STRING",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "text",
|
||||
"name": "text",
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "text"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "negative_text",
|
||||
"name": "negative_text",
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "negative_text"
|
||||
},
|
||||
"link": 2
|
||||
},
|
||||
{
|
||||
"localized_name": "category",
|
||||
"name": "category",
|
||||
"shape": 7,
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "category"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "tags",
|
||||
"name": "tags",
|
||||
"shape": 7,
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "tags"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "search_text",
|
||||
"name": "search_text",
|
||||
"shape": 7,
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "search_text"
|
||||
},
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"localized_name": "positive",
|
||||
"name": "positive",
|
||||
"type": "CONDITIONING",
|
||||
"links": null
|
||||
},
|
||||
{
|
||||
"localized_name": "negative",
|
||||
"name": "negative",
|
||||
"type": "CONDITIONING",
|
||||
"links": null
|
||||
},
|
||||
{
|
||||
"localized_name": "positive_text",
|
||||
"name": "positive_text",
|
||||
"type": "STRING",
|
||||
"links": null
|
||||
},
|
||||
{
|
||||
"localized_name": "negative_text",
|
||||
"name": "negative_text",
|
||||
"type": "STRING",
|
||||
"links": null
|
||||
}
|
||||
],
|
||||
"title": "Pair negative fed by a Primitive",
|
||||
"properties": {
|
||||
"Node name for S&R": "PromptManagerPair",
|
||||
"pairUI": {
|
||||
"libraryOpen": true,
|
||||
"libraryFilters": {
|
||||
"category": "",
|
||||
"tags": ""
|
||||
},
|
||||
"composition": false,
|
||||
"metadata": false,
|
||||
"preview": false
|
||||
}
|
||||
},
|
||||
"widgets_values": [
|
||||
"Typed pair positive",
|
||||
"Primitive pair negative",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
""
|
||||
],
|
||||
"widgets_values_named": {
|
||||
"text": "Typed pair positive",
|
||||
"negative_text": "Primitive pair negative",
|
||||
"category": "",
|
||||
"tags": "",
|
||||
"search_text": "",
|
||||
"prepend_text": "",
|
||||
"append_text": "",
|
||||
"prompt_manager_focused_ui": ""
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "PrimitiveNode",
|
||||
"pos": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"size": [
|
||||
210,
|
||||
88
|
||||
],
|
||||
"flags": {},
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "STRING",
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "text"
|
||||
},
|
||||
"links": [
|
||||
1
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Run widget replace on values": false
|
||||
},
|
||||
"widgets_values": [
|
||||
"Primitive focused text"
|
||||
],
|
||||
"widgets_values_named": {
|
||||
"text": "Primitive focused text"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "PrimitiveNode",
|
||||
"pos": [
|
||||
0,
|
||||
1200
|
||||
],
|
||||
"size": [
|
||||
210,
|
||||
88
|
||||
],
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "STRING",
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "negative_text"
|
||||
},
|
||||
"links": [
|
||||
2
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Run widget replace on values": false
|
||||
},
|
||||
"widgets_values": [
|
||||
"Primitive pair negative"
|
||||
],
|
||||
"widgets_values_named": {
|
||||
"negative_text": "Primitive pair negative"
|
||||
}
|
||||
}
|
||||
],
|
||||
"links": [
|
||||
[
|
||||
1,
|
||||
3,
|
||||
0,
|
||||
1,
|
||||
3,
|
||||
"STRING"
|
||||
],
|
||||
[
|
||||
2,
|
||||
4,
|
||||
0,
|
||||
2,
|
||||
4,
|
||||
"STRING"
|
||||
]
|
||||
],
|
||||
"groups": [],
|
||||
"config": {},
|
||||
"extra": {
|
||||
"ds": {
|
||||
"scale": 0.9,
|
||||
"offset": [
|
||||
416,
|
||||
110
|
||||
]
|
||||
}
|
||||
},
|
||||
"version": 0.4
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,142 @@
|
||||
// #342 on the real pinned host: the image monitor links an output only to a prompt
|
||||
// that provably produced it. An unknown node (PMHistoryCached) stands in for the
|
||||
// maintainer's LLM; KSampler is the fixture's deterministic stand-in.
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { execFile } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test } from './execution-fixture.mjs';
|
||||
|
||||
const run = promisify(execFile);
|
||||
const reader = fileURLToPath(new URL('./read_execution_history.py', import.meta.url));
|
||||
|
||||
async function readLinks(databasePath, filenames, texts = []) {
|
||||
const { stdout } = await run(process.env.PM_TEST_PYTHON || 'python',
|
||||
[reader, databasePath, '--links', ...filenames, '--texts', ...texts], { timeout: 5000, maxBuffer: 1024 * 1024 });
|
||||
return JSON.parse(stdout);
|
||||
}
|
||||
|
||||
// kind: the positive node; text: a typed editor or 'llm' (linked from the unknown
|
||||
// node); prepend: 'llm' links the unknown node into prepend_text.
|
||||
function graph({ kind = 'PromptManagerPair', text = '', prepend = '', llm = '', seed = 1, prefix }) {
|
||||
const linked = value => value === 'llm' ? ['11', 0] : value;
|
||||
const nodes = {
|
||||
'10': { class_type: 'PMHistoryInputs', inputs: {} },
|
||||
'20': { class_type: kind, inputs: { text: linked(text), clip: ['10', 0], append_text: '',
|
||||
...(prepend ? { prepend_text: linked(prepend) } : {}),
|
||||
...(kind === 'PromptManagerPair' ? { negative_text: '' } : {}) } },
|
||||
'30': { class_type: 'CLIPTextEncode', inputs: { text: '', clip: ['10', 0] } },
|
||||
'40': { class_type: 'KSampler', inputs: { model: ['10', 1], latent_image: ['10', 2],
|
||||
positive: ['20', 0], negative: kind === 'PromptManagerPair' ? ['20', 1] : ['30', 0],
|
||||
seed, steps: 1, cfg: 1, sampler_name: 'euler', scheduler: 'normal', denoise: 1 } },
|
||||
'60': { class_type: 'PMHistoryImages', inputs: { value: 64, batch: 1, latent: ['40', 0] } },
|
||||
'70': { class_type: 'SaveImage', inputs: { images: ['60', 0], filename_prefix: prefix } },
|
||||
};
|
||||
if (llm) nodes['11'] = { class_type: 'PMHistoryCached', inputs: { value: llm } };
|
||||
return nodes;
|
||||
}
|
||||
|
||||
async function saved(request, prompt, history = {}) {
|
||||
const id = randomUUID();
|
||||
const response = await request.post('/prompt', { data: { prompt, prompt_id: id } });
|
||||
expect(response.ok()).toBe(true);
|
||||
let item;
|
||||
await expect.poll(async () => {
|
||||
item = (await (await request.get(`/history/${id}`)).json())[id];
|
||||
return item?.status?.messages?.some(([kind]) => kind === 'execution_success') || false;
|
||||
}, { timeout: 15000 }).toBe(true);
|
||||
const [image] = item.outputs['70'].images;
|
||||
expect(image.type).toBe('output');
|
||||
history.item = item;
|
||||
return image.filename;
|
||||
}
|
||||
|
||||
async function linkedTo(execution, filename, expected) {
|
||||
await expect.poll(async () => (await readLinks(execution.databasePath, [filename])).links[filename],
|
||||
{ timeout: 20000, message: `${filename} is linked by the image monitor` }).toBe(expected);
|
||||
}
|
||||
|
||||
async function monitoring(execution, enabled) {
|
||||
const response = await execution.request.post('/pm_execution_fixture/monitor', { data: { enabled } });
|
||||
expect(response.ok()).toBe(true);
|
||||
expect((await response.json()).monitoring).toBe(enabled);
|
||||
}
|
||||
|
||||
async function cacheMode(execution, mode) {
|
||||
expect((await execution.request.post('/pm_execution_fixture/cache', { data: { mode } })).ok()).toBe(true);
|
||||
}
|
||||
|
||||
test.beforeEach(async ({ execution }) => {
|
||||
expect((await execution.request.post('/pm_execution_fixture/metadata', { data: { disabled: false } })).ok()).toBe(true);
|
||||
await cacheMode(execution, 'CLASSIC'); // keeps the re-run's PromptManager node cached
|
||||
await monitoring(execution, true);
|
||||
});
|
||||
test.afterEach(async ({ execution }) => {
|
||||
await monitoring(execution, false);
|
||||
await cacheMode(execution, 'RAM_PRESSURE');
|
||||
});
|
||||
|
||||
test('an output whose positive is only linked prepend text stays unlinked, never given an earlier prompt', async ({ execution }, testInfo) => {
|
||||
const tag = `${testInfo.project.name} ${randomUUID()}`;
|
||||
const earlier = `a woman holding a coffee cup ${tag}`;
|
||||
const prepended = `llm prepend landscape ${tag}`, llmText = `llm text landscape ${tag}`;
|
||||
|
||||
// A typed prompt runs first: before #342 the next unidentifiable image took it.
|
||||
const first = await saved(execution.request, graph({ text: earlier, prefix: 'misslink/earlier' }));
|
||||
await linkedTo(execution, first, earlier);
|
||||
|
||||
// The maintainer's workflow: empty editor, the generated text arrives in prepend.
|
||||
const orphan = await saved(execution.request,
|
||||
graph({ prepend: 'llm', llm: prepended, prefix: 'misslink/prepend' }));
|
||||
|
||||
// An LLM feeding the editor itself is stored and links through its run key; the
|
||||
// monitor works in creation order, so the prepend image was processed first.
|
||||
const llmGraph = graph({ kind: 'PromptManager', text: 'llm', llm: llmText, prefix: 'misslink/text' });
|
||||
const proven = await saved(execution.request, llmGraph);
|
||||
await linkedTo(execution, proven, llmText);
|
||||
const state = await readLinks(execution.databasePath, [orphan], [prepended]);
|
||||
expect(state).toEqual({ links: { [orphan]: null }, stored: { [prepended]: false } });
|
||||
|
||||
// Only the sampler seed changes: the PromptManager node is served from the cache
|
||||
// without running (the host says so), so no queue entry exists for this run and
|
||||
// only the snapshot's run match can link it.
|
||||
const rerunHistory = {};
|
||||
const rerun = await saved(execution.request, { ...llmGraph, '40': { ...llmGraph['40'],
|
||||
inputs: { ...llmGraph['40'].inputs, seed: 2 } } }, rerunHistory);
|
||||
const cached = rerunHistory.item.status.messages.find(([kind]) => kind === 'execution_cached');
|
||||
expect(cached?.[1].nodes).toEqual(expect.arrayContaining(['11', '20']));
|
||||
await linkedTo(execution, rerun, llmText);
|
||||
});
|
||||
|
||||
test('an output saved without metadata links once its recorded run names a stored prompt', async ({ execution }, testInfo) => {
|
||||
const llmText = `bare output landscape ${testInfo.project.name} ${randomUUID()}`;
|
||||
const request = execution.request;
|
||||
expect((await request.post('/pm_execution_fixture/metadata', { data: { disabled: true } })).ok()).toBe(true);
|
||||
try {
|
||||
// No embedded graph: no run key can prove the tracker entry, so only the
|
||||
// run's recorded history can, and only once the run's outputs are sealed.
|
||||
// The gate after SaveImage holds the run open while the monitor looks.
|
||||
const prompt = graph({ kind: 'PromptManager', text: 'llm', llm: llmText, prefix: 'misslink/bare' });
|
||||
prompt['80'] = { class_type: 'PMHistoryImageOutput', inputs: { images: ['70', 0], mode: 'gate' } };
|
||||
expect((await request.post('/pm_execution_fixture/gate/hold')).ok()).toBe(true);
|
||||
const id = randomUUID();
|
||||
try {
|
||||
expect((await request.post('/prompt', { data: { prompt, prompt_id: id } })).ok()).toBe(true);
|
||||
let waiting = [];
|
||||
await expect.poll(async () => {
|
||||
waiting = (await (await request.get('/pm_execution_fixture/monitor')).json()).awaiting;
|
||||
return waiting.filter(name => name.startsWith('bare_')).length;
|
||||
}, { timeout: 20000, message: 'the monitor processed the bare output while its run was open' }).toBe(1);
|
||||
const [bare] = waiting.filter(name => name.startsWith('bare_'));
|
||||
expect((await readLinks(execution.databasePath, [bare])).links[bare]).toBeNull();
|
||||
expect((await request.post('/pm_execution_fixture/gate/release')).ok()).toBe(true);
|
||||
await linkedTo(execution, bare, llmText);
|
||||
expect((await (await request.get('/pm_execution_fixture/monitor')).json()).awaiting).not.toContain(bare);
|
||||
} finally {
|
||||
expect((await request.post('/pm_execution_fixture/gate/release')).ok()).toBe(true);
|
||||
}
|
||||
} finally {
|
||||
expect((await request.post('/pm_execution_fixture/metadata', { data: { disabled: false } })).ok()).toBe(true);
|
||||
}
|
||||
});
|
||||
@@ -190,3 +190,21 @@ test('bridge remains live through both renderer switches and workflow reload wit
|
||||
await expect(page.locator('.pm-focused').getByRole('textbox',{name:'Main prompt',exact:true})).toHaveValue('After reload');
|
||||
});
|
||||
});
|
||||
|
||||
test('each destination reports its window renderer, and System › Integrations shows it (S10)',async({page,context},testInfo)=>{
|
||||
const initial=testInfo.project.metadata.initialRenderer;
|
||||
const id=await addNode(page,'PromptManager',{text:'Renderer check'});
|
||||
await withClient(page,context,async clientPage=>{
|
||||
const name=renderer=>renderer?'Nodes 2.0':'Classic';
|
||||
const status=clientPage.locator('#workspace-main #renderer-status');
|
||||
expect((await destination(clientPage,id)).target.renderer).toBe(initial?'nodes2':'classic');
|
||||
await clientPage.evaluate(()=>{location.hash='#/system/integrations';});
|
||||
await expect(status).toHaveText(`Detected: ${name(initial)} (1 ComfyUI window).`);
|
||||
// Switching the renderer in ComfyUI is what the next check reports; nothing is sent or queued.
|
||||
await page.evaluate(async renderer=>{const {app}=await import('/scripts/app.js');await app.ui.settings.setSettingValue('Comfy.VueNodes.Enabled',renderer);await app.loadGraphData(app.graph.serialize());},!initial);
|
||||
await expect.poll(()=>page.evaluate(()=>LiteGraph.vueNodesMode)).toBe(!initial);
|
||||
expect((await destination(clientPage,id)).target.renderer).toBe(initial?'classic':'nodes2');
|
||||
await clientPage.reload();
|
||||
await expect(status).toHaveText(`Detected: ${name(!initial)} (1 ComfyUI window).`);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Bounded read-only SQLite inspection for the owned execution fixture."""
|
||||
|
||||
import argparse
|
||||
from contextlib import closing
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
@@ -89,14 +90,48 @@ def read_runs(database: str, job_id: str, session_id: str | None = None) -> dict
|
||||
}
|
||||
|
||||
|
||||
def read_links(database: str, filenames: list, texts: list) -> dict:
|
||||
"""Each output file's linked prompt text (None: unlinked), and which texts
|
||||
are stored as prompts (#342)."""
|
||||
uri = Path(database).resolve().as_uri() + "?mode=ro"
|
||||
with closing(sqlite3.connect(uri, uri=True, timeout=0.25)) as connection:
|
||||
connection.execute("PRAGMA query_only=ON")
|
||||
links = {}
|
||||
for name in filenames[:64]:
|
||||
rows = connection.execute(
|
||||
"SELECT p.text FROM generated_images g JOIN prompts p "
|
||||
"ON p.id = g.prompt_id WHERE g.filename = ? LIMIT 2",
|
||||
(name,),
|
||||
).fetchall()
|
||||
values = [row[0] for row in rows]
|
||||
links[name] = values[0] if len(values) == 1 else None
|
||||
if len(values) > 1:
|
||||
links[name] = "<several links>"
|
||||
stored = {
|
||||
text: connection.execute(
|
||||
"SELECT 1 FROM prompts WHERE text = ? LIMIT 1", (text,)
|
||||
).fetchone()
|
||||
is not None
|
||||
for text in texts[:64]
|
||||
}
|
||||
return {"links": links, "stored": stored}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("database")
|
||||
parser.add_argument("job_id")
|
||||
parser.add_argument("job_id", nargs="?")
|
||||
parser.add_argument("--session-id")
|
||||
parser.add_argument("--links", nargs="*", default=None)
|
||||
parser.add_argument("--texts", nargs="*", default=[])
|
||||
arguments = parser.parse_args()
|
||||
print(
|
||||
json.dumps(
|
||||
read_runs(arguments.database, arguments.job_id, arguments.session_id)
|
||||
if arguments.links is not None:
|
||||
print(
|
||||
json.dumps(read_links(arguments.database, arguments.links, arguments.texts))
|
||||
)
|
||||
else:
|
||||
print(
|
||||
json.dumps(
|
||||
read_runs(arguments.database, arguments.job_id, arguments.session_id)
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
@@ -0,0 +1,404 @@
|
||||
// #343: on the pinned real host, a STRING link dragged with the mouse drives each PromptManager node's text (and the
|
||||
// Pair's negative) in Classic and Nodes 2.0, across renderer switches, a workflow reload, disconnect and removal.
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { installHostHooks } from './host-fixture.mjs';
|
||||
|
||||
installHostHooks(test, ['PromptManager', 'PromptManagerText', 'PromptManagerFocused', 'PromptManagerSplit', 'PromptManagerPair']);
|
||||
|
||||
const editors = { PromptManagerFocused: 'Main prompt', PromptManagerSplit: 'Main prompt', PromptManagerPair: 'Positive prompt' };
|
||||
const note = label => `${label} comes from a link. Your typed text is kept for when you disconnect it.`;
|
||||
|
||||
/** Viewport position of a node's slot: the slot dot in Nodes 2.0, the canvas slot position in Classic. */
|
||||
function slotPosition(page, nodeId, kind, name) {
|
||||
return page.evaluate(async ({ nodeId, kind, name }) => {
|
||||
const { app } = await import('/scripts/app.js');
|
||||
const node = app.graph.getNodeById(nodeId);
|
||||
const slots = kind === 'out' ? node.outputs : node.inputs;
|
||||
const index = typeof name === 'number' ? name : slots.findIndex(slot => slot.name === name);
|
||||
if (LiteGraph.vueNodesMode) {
|
||||
const dot = document.querySelector(`[data-slot-key="${nodeId}-${kind}-${index}"]`);
|
||||
if (!dot) return null;
|
||||
const box = dot.getBoundingClientRect();
|
||||
return [box.x + box.width / 2, box.y + box.height / 2];
|
||||
}
|
||||
const position = app.canvas.ds.convertOffsetToCanvas(kind === 'out' ? node.getOutputPos(index) : node.getInputPos(index));
|
||||
const box = app.canvas.canvas.getBoundingClientRect();
|
||||
return [box.x + position[0], box.y + position[1]];
|
||||
}, { nodeId, kind, name });
|
||||
}
|
||||
|
||||
async function drag(page, from, to) {
|
||||
await page.mouse.move(...from);
|
||||
await page.mouse.down();
|
||||
await page.mouse.move((from[0] + to[0]) / 2, (from[1] + to[1]) / 2, { steps: 5 });
|
||||
await page.mouse.move(...to, { steps: 10 });
|
||||
await page.mouse.up();
|
||||
}
|
||||
|
||||
const center = box => [box.x + box.width / 2, box.y + box.height / 2];
|
||||
const linkedInputs = (page, id) => page.evaluate(async id => {
|
||||
const { app } = await import('/scripts/app.js');
|
||||
return app.graph.getNodeById(id).inputs.filter(input => input.link != null).map(input => input.name);
|
||||
}, id);
|
||||
const queued = (page, id) => page.evaluate(async id => (await (await import('/scripts/app.js')).app.graphToPrompt()).output[String(id)].inputs, id);
|
||||
|
||||
async function setup(page, type) {
|
||||
return page.evaluate(async type => {
|
||||
const { app } = await import('/scripts/app.js');
|
||||
const node = LiteGraph.createNode(type);
|
||||
app.graph.add(node);
|
||||
node.pos = [500, 80];
|
||||
node.setSize([type === 'PromptManagerPair' ? 1100 : 650, 760]);
|
||||
const sources = ['text', 'negative_text'].map((name, index) => {
|
||||
const source = LiteGraph.createNode('PrimitiveString');
|
||||
app.graph.add(source);
|
||||
source.pos = [40, 80 + index * 160];
|
||||
source.widgets.find(widget => widget.name === 'value').value = `Linked ${name} ${type}`;
|
||||
return source.id;
|
||||
});
|
||||
app.canvas.ds.scale = .8;
|
||||
app.canvas.ds.offset = [40, 40];
|
||||
app.canvas.setDirty(true, true);
|
||||
return { id: node.id, sources };
|
||||
}, type);
|
||||
}
|
||||
|
||||
/** The prompt box a user drops the link on: the editor's textarea, or the core text box of the Classic nodes. */
|
||||
async function promptBox(page, type, id) {
|
||||
if (editors[type]) return page.getByRole('textbox', { name: editors[type], exact: true });
|
||||
// The Classic nodes keep the core text widget: Nodes 2.0 draws it as its own textarea, Classic as widget.inputEl.
|
||||
if (await page.evaluate(() => LiteGraph.vueNodesMode)) return page.locator(`[data-node-id="${id}"] textarea`).first();
|
||||
await page.evaluate(async id => {
|
||||
const { app } = await import('/scripts/app.js');
|
||||
app.graph.getNodeById(id).widgets.find(widget => widget.name === 'text').inputEl.dataset.pmTestPromptBox = String(id);
|
||||
}, id);
|
||||
return page.locator(`textarea[data-pm-test-prompt-box="${id}"]`);
|
||||
}
|
||||
|
||||
/** What the node shows while its text is linked. */
|
||||
function linkedNote(page, type) {
|
||||
return editors[type] ? page.getByText(note(editors[type]), { exact: true }) : page.locator('.pm-text-link-note').getByText(note('Text'), { exact: true });
|
||||
}
|
||||
|
||||
for (const type of ['PromptManagerFocused', 'PromptManagerSplit', 'PromptManagerPair', 'PromptManager', 'PromptManagerText']) {
|
||||
test(`${type}: a STRING link dropped on the prompt box drives text and the typed text returns on disconnect`, async ({ page }, testInfo) => {
|
||||
test.setTimeout(90000);
|
||||
const { id, sources } = await setup(page, type);
|
||||
const typed = `Typed ${type} ${testInfo.project.name}`;
|
||||
const box = await promptBox(page, type, id);
|
||||
await box.fill(typed);
|
||||
await expect(linkedNote(page, type)).toBeHidden();
|
||||
await page.screenshot({ path: testInfo.outputPath(`${type}-unlinked.png`) });
|
||||
// The drop lands on text, never on prepend_text (#342).
|
||||
await drag(page, await slotPosition(page, sources[0], 'out', 0), center(await box.boundingBox()));
|
||||
await expect.poll(() => linkedInputs(page, id)).toEqual(['text']);
|
||||
const links = { text: [String(sources[0]), 0] };
|
||||
if (type === 'PromptManagerPair') {
|
||||
await page.getByRole('textbox', { name: 'Negative prompt', exact: true }).fill(`Typed negative ${testInfo.project.name}`);
|
||||
await drag(page, await slotPosition(page, sources[1], 'out', 0), await slotPosition(page, id, 'in', 'negative_text'));
|
||||
await expect.poll(() => linkedInputs(page, id)).toEqual(['text', 'negative_text']);
|
||||
links.negative_text = [String(sources[1]), 0];
|
||||
await expect(page.getByText(note('Negative prompt'), { exact: true })).toBeVisible();
|
||||
}
|
||||
await expect(linkedNote(page, type)).toBeVisible();
|
||||
expect(await queued(page, id)).toMatchObject(links);
|
||||
await page.screenshot({ path: testInfo.outputPath(`${type}-linked.png`) });
|
||||
for (const renderer of [!testInfo.project.metadata.initialRenderer, testInfo.project.metadata.initialRenderer]) {
|
||||
await page.evaluate(async renderer => {
|
||||
const { app } = await import('/scripts/app.js');
|
||||
await app.ui.settings.setSettingValue('Comfy.VueNodes.Enabled', renderer);
|
||||
}, renderer);
|
||||
await expect.poll(() => page.evaluate(() => LiteGraph.vueNodesMode)).toBe(renderer);
|
||||
await expect(linkedNote(page, type)).toBeVisible();
|
||||
expect(await queued(page, id)).toMatchObject(links);
|
||||
}
|
||||
const saved = await page.evaluate(async id => {
|
||||
const { app } = await import('/scripts/app.js');
|
||||
const workflow = app.graph.serialize();
|
||||
await app.loadGraphData(workflow);
|
||||
return workflow.nodes.find(node => String(node.id) === String(id)).widgets_values;
|
||||
}, id);
|
||||
expect(saved[0]).toBe(typed);
|
||||
await expect(linkedNote(page, type)).toBeVisible();
|
||||
expect(await queued(page, id)).toMatchObject(links);
|
||||
const local = await page.evaluate(async id => {
|
||||
const { app } = await import('/scripts/app.js');
|
||||
const node = app.graph.getNodeById(id);
|
||||
for (const input of [...node.inputs]) if (input.link != null) node.disconnectInput(node.inputs.indexOf(input));
|
||||
return (await app.graphToPrompt()).output[String(id)].inputs;
|
||||
}, id);
|
||||
expect(local.text).toBe(typed);
|
||||
if (type === 'PromptManagerPair') expect(local.negative_text).toBe(`Typed negative ${testInfo.project.name}`);
|
||||
await expect(linkedNote(page, type)).toBeHidden();
|
||||
const restored = await promptBox(page, type, id);
|
||||
await expect(restored).toBeVisible();
|
||||
await expect(restored).toHaveValue(typed);
|
||||
await page.screenshot({ path: testInfo.outputPath(`${type}-disconnected.png`) });
|
||||
await page.evaluate(async id => { const { app } = await import('/scripts/app.js'); app.graph.remove(app.graph.getNodeById(id)); }, id);
|
||||
await expect(page.locator('.pm-focused, .pm-classic, .pm-text-link-note')).toHaveCount(0);
|
||||
});
|
||||
}
|
||||
|
||||
async function addEditors(page) {
|
||||
return page.evaluate(async () => {
|
||||
const { app } = await import('/scripts/app.js');
|
||||
const add = (type, x, y) => { const node = LiteGraph.createNode(type); app.graph.add(node); node.pos = [x, y]; return node; };
|
||||
const focused = add('PromptManagerFocused', 420, 40);
|
||||
focused.setSize([560, 640]);
|
||||
const pair = add('PromptManagerPair', 1020, 40);
|
||||
pair.setSize([1000, 640]);
|
||||
for (const [node, name, value] of [[focused, 'text', 'Typed focused'], [pair, 'text', 'Typed positive'], [pair, 'negative_text', 'Typed negative']]) {
|
||||
const widget = node.widgets.find(item => item.name === name);
|
||||
widget.value = value;
|
||||
if (widget.inputEl) widget.inputEl.value = value;
|
||||
}
|
||||
const sources = [0, 1].map(index => add(index ? 'PrimitiveString' : 'PrimitiveNode', 20, 60 + index * 300).id);
|
||||
app.canvas.ds.scale = .7;
|
||||
// Clear of the floating graph menu (top left) and the PM run bar (top right).
|
||||
app.canvas.ds.offset = [110, 150];
|
||||
app.canvas.setDirty(true, true);
|
||||
return { focused: focused.id, pair: pair.id, primitive: sources[0], string: sources[1] };
|
||||
});
|
||||
}
|
||||
|
||||
/** The prompt a Run sends to the server, through the real app.queuePrompt path. */
|
||||
async function runPrompt(page) {
|
||||
let captured;
|
||||
const sent = new Promise(resolve => { captured = resolve; });
|
||||
// Installed before the run starts, so the POST can never reach the real host.
|
||||
await page.route('**/prompt', async route => {
|
||||
if (route.request().method() !== 'POST') return route.fallback();
|
||||
captured(route.request().postDataJSON().prompt);
|
||||
return route.fulfill({ json: { prompt_id: 'captured', number: 1, node_errors: {} } });
|
||||
});
|
||||
await page.evaluate(async () => { const { app } = await import('/scripts/app.js'); await app.queuePrompt(0, 1); });
|
||||
const prompt = await sent;
|
||||
await page.unroute('**/prompt');
|
||||
return prompt;
|
||||
}
|
||||
|
||||
const box = (page, name, nth = 0) => page.getByRole('textbox', { name, exact: true }).nth(nth);
|
||||
|
||||
test('a legacy Primitive dropped on an editor box or socket drives the text it shows as linked (#343 review I1, M1)', async ({ page }) => {
|
||||
test.setTimeout(90000);
|
||||
const ids = await addEditors(page);
|
||||
// A wildcard Primitive aimed at the Main prompt box, a PrimitiveString aimed at the Negative prompt box.
|
||||
await drag(page, await slotPosition(page, ids.primitive, 'out', 0), center(await box(page, 'Main prompt').boundingBox()));
|
||||
await expect.poll(() => linkedInputs(page, ids.focused)).toEqual(['text']);
|
||||
await drag(page, await slotPosition(page, ids.string, 'out', 0), center(await box(page, 'Negative prompt').boundingBox()));
|
||||
await expect.poll(() => linkedInputs(page, ids.pair)).toEqual(['negative_text']);
|
||||
await expect(page.getByText(note('Main prompt'), { exact: true })).toBeVisible();
|
||||
await expect(page.getByText(note('Negative prompt'), { exact: true })).toBeVisible();
|
||||
await page.evaluate(async ({ primitive, string }) => {
|
||||
const { app } = await import('/scripts/app.js');
|
||||
app.graph.getNodeById(primitive).widgets[0].value = 'From the legacy Primitive';
|
||||
app.graph.getNodeById(string).widgets.find(widget => widget.name === 'value').value = 'From PrimitiveString';
|
||||
}, ids);
|
||||
const prompt = await runPrompt(page);
|
||||
expect(prompt[String(ids.focused)].inputs.text).toBe('From the legacy Primitive');
|
||||
expect(prompt[String(ids.pair)].inputs).toMatchObject({ text: 'Typed positive', negative_text: [String(ids.string), 0] });
|
||||
// The Primitive is re-read on every run; the typed text stays for disconnect.
|
||||
await page.evaluate(async ({ primitive }) => { const { app } = await import('/scripts/app.js'); app.graph.getNodeById(primitive).widgets[0].value = 'Changed in the Primitive'; }, ids);
|
||||
expect((await queued(page, ids.focused)).text).toBe('Changed in the Primitive');
|
||||
await page.evaluate(async ({ focused }) => { const { app } = await import('/scripts/app.js'); const node = app.graph.getNodeById(focused); node.disconnectInput(node.inputs.findIndex(input => input.name === 'text')); }, ids);
|
||||
expect((await queued(page, ids.focused)).text).toBe('Typed focused');
|
||||
});
|
||||
|
||||
test('a wildcard legacy Primitive queues one resolved choice, drawn once per run, on every PromptManager node (#343 re-review R1)', async ({ page }) => {
|
||||
const choices = `{${Array.from({ length: 10 }, (_, index) => `c${index}`).join('|')}}`;
|
||||
const ids = await page.evaluate(async choices => {
|
||||
const { app } = await import('/scripts/app.js');
|
||||
const add = (type, x, y) => { const node = LiteGraph.createNode(type); app.graph.add(node); node.pos = [x, y]; return node; };
|
||||
// [node, input, Primitive value, its "Run widget replace on values" option]
|
||||
const targets = [['PromptManagerFocused', 'text', `Focused ${choices}`, false], ['PromptManagerSplit', 'text', `Split %date:yyyy% ${choices}`, true],
|
||||
['PromptManagerPair', 'negative_text', `Pair ${choices}`, false], ['PromptManager', 'text', `Classic ${choices}`, false],
|
||||
['PromptManagerText', 'text', `Text ${choices}`, false]];
|
||||
const ids = {};
|
||||
targets.forEach(([type, name, value, replace], index) => {
|
||||
const node = add(type, 500, index * 800);
|
||||
const primitive = add('PrimitiveNode', 20, index * 800);
|
||||
if (!primitive.connect(0, node, node.inputs.findIndex(input => input.name === name))) throw new Error(`no Primitive link to ${type}.${name}`);
|
||||
primitive.widgets[0].value = value;
|
||||
primitive.properties['Run widget replace on values'] = replace;
|
||||
ids[type] = [String(node.id), name];
|
||||
});
|
||||
// The Classic nodes install their own text serializer shortly after creation; run after it, as a user does.
|
||||
await Promise.all(app.graph.nodes.map(node => node._pmTextSerializerReady));
|
||||
// What another extension's graphToPrompt wrapper sees must be what the server gets.
|
||||
const inner = app.graphToPrompt;
|
||||
const seen = window.__pmSeen = [];
|
||||
app.graphToPrompt = async function (...args) { const result = await inner.apply(this, args); seen.push(structuredClone(result.output)); return result; };
|
||||
// Each draw takes the next choice, so a second draw for the same text always changes it.
|
||||
const random = Math.random;
|
||||
let draw = 0;
|
||||
Math.random = () => ((draw++ % 10) + .5) / 10;
|
||||
window.__pmRestore = () => { app.graphToPrompt = inner; Math.random = random; };
|
||||
return ids;
|
||||
}, choices);
|
||||
const prompt = await runPrompt(page);
|
||||
const seen = await page.evaluate(() => { window.__pmRestore(); return window.__pmSeen; });
|
||||
expect(seen).toHaveLength(1);
|
||||
const year = String(new Date().getFullYear());
|
||||
const stems = { PromptManagerFocused: 'Focused', PromptManagerSplit: `Split ${year}`, PromptManagerPair: 'Pair', PromptManager: 'Classic', PromptManagerText: 'Text' };
|
||||
// The Classic nodes keep their widget binding, so the host applies the Primitive and #343 leaves the value alone.
|
||||
// As on main, their text widget's own serializer queues the text as written, wildcard included.
|
||||
const classic = new Set(['PromptManager', 'PromptManagerText']);
|
||||
for (const [type, [id, name]] of Object.entries(ids)) {
|
||||
const value = prompt[id].inputs[name];
|
||||
if (classic.has(type)) expect(value, type).toBe(`${stems[type]} ${choices}`);
|
||||
else expect(value, type).toMatch(new RegExp(`^${stems[type]} c\\d$`));
|
||||
expect(value, `${type}: one draw per run`).toBe(seen[0][id].inputs[name]);
|
||||
}
|
||||
});
|
||||
|
||||
test('a legacy Primitive inside a subgraph drives the inner editor text it shows as linked (#343 re-review R2)', async ({ page }) => {
|
||||
const result = await page.evaluate(async () => {
|
||||
const { app } = await import('/scripts/app.js');
|
||||
const add = (type, x, y) => { const node = LiteGraph.createNode(type); app.graph.add(node); node.pos = [x, y]; return node; };
|
||||
const focused = add('PromptManagerFocused', 500, 40);
|
||||
focused.widgets.find(widget => widget.name === 'text').value = 'Typed under the link';
|
||||
const primitive = add('PrimitiveNode', 20, 40);
|
||||
if (!primitive.connect(0, focused, focused.inputs.findIndex(input => input.name === 'text'))) throw new Error('no Primitive link');
|
||||
const { node } = app.graph.convertToSubgraph(new Set([focused, primitive]));
|
||||
const inner = node.subgraph.nodes.find(item => item.type === 'PromptManagerFocused');
|
||||
// The value the user gives the Primitive inside the subgraph.
|
||||
node.subgraph.nodes.find(item => item.type === 'PrimitiveNode').widgets[0].value = 'Inner {only choice}';
|
||||
return { host: String(node.id), inner: String(inner.id), linked: inner.inputs.find(input => input.name === 'text').link != null };
|
||||
});
|
||||
expect(result.linked).toBe(true);
|
||||
const prompt = await runPrompt(page);
|
||||
expect(prompt[`${result.host}:${result.inner}`]?.inputs.text).toBe('Inner only choice');
|
||||
});
|
||||
|
||||
test('a workflow saved before #343 with legacy Primitives on editor text keeps running their values (#343 review I1)', async ({ page }) => {
|
||||
const workflow = JSON.parse(await readFile(new URL('./fixtures/workflow-before-343-primitive.json', import.meta.url), 'utf8'));
|
||||
const result = await page.evaluate(async workflow => {
|
||||
const { app } = await import('/scripts/app.js');
|
||||
await app.loadGraphData(structuredClone(workflow));
|
||||
const before = (await app.graphToPrompt()).output;
|
||||
app.graph.getNodeById(4).widgets[0].value = 'Edited after the load';
|
||||
const after = (await app.graphToPrompt()).output;
|
||||
const bound = id => app.graph.getNodeById(id).inputs.filter(input => ['text', 'negative_text'].includes(input.name) && input.widget).length;
|
||||
return { before: [before['1'].inputs.text, before['2'].inputs.text, before['2'].inputs.negative_text], after: after['2'].inputs.negative_text, bound: [bound(1), bound(2)],
|
||||
links: [app.graph.getNodeById(1).inputs.find(input => input.name === 'text').link != null, app.graph.getNodeById(2).inputs.find(input => input.name === 'negative_text').link != null] };
|
||||
}, workflow);
|
||||
expect(result).toEqual({ before: ['Primitive focused text', 'Typed pair positive', 'Primitive pair negative'], after: 'Edited after the load', bound: [0, 0], links: [true, true] });
|
||||
const prompt = await runPrompt(page);
|
||||
expect([prompt['1'].inputs.text, prompt['2'].inputs.negative_text]).toEqual(['Primitive focused text', 'Edited after the load']);
|
||||
});
|
||||
|
||||
test('moving a linked text and dropping it on the editor body keeps it on text, never prepend_text (#343 review I2)', async ({ page }) => {
|
||||
test.setTimeout(90000);
|
||||
const ids = await addEditors(page);
|
||||
await page.evaluate(async ({ focused, pair, string }) => {
|
||||
const { app } = await import('/scripts/app.js');
|
||||
const source = app.graph.getNodeById(string);
|
||||
for (const id of [focused, pair]) { const node = app.graph.getNodeById(id); source.connect(0, node, node.inputs.findIndex(input => input.name === 'text')); }
|
||||
app.canvas.setDirty(true, true);
|
||||
}, ids);
|
||||
for (const [id, title] of [[ids.focused, 0], [ids.pair, 1]]) {
|
||||
// Grab the link at the text socket and release it over the editor's toolbar, away from any prompt box.
|
||||
const toolbar = page.locator('.pm-editor-toolbar strong').nth(title);
|
||||
await drag(page, await slotPosition(page, id, 'in', 'text'), center(await toolbar.boundingBox()));
|
||||
await expect.poll(() => linkedInputs(page, id)).toEqual(['text']);
|
||||
}
|
||||
});
|
||||
|
||||
test('paste, clone and undo keep the editor text a linked socket (#343 review M7)', async ({ page }) => {
|
||||
const ids = await addEditors(page);
|
||||
const state = await page.evaluate(async ({ pair, string }) => {
|
||||
const { app } = await import('/scripts/app.js');
|
||||
const node = app.graph.getNodeById(pair);
|
||||
app.graph.getNodeById(string).connect(0, node, node.inputs.findIndex(input => input.name === 'text'));
|
||||
const describe = target => ({ bound: target.inputs.filter(input => ['text', 'negative_text'].includes(input.name) && input.widget).length,
|
||||
linked: target.inputs.find(input => input.name === 'text').link != null });
|
||||
app.canvas.selectItems([node]);
|
||||
app.canvas.copyToClipboard();
|
||||
app.canvas.pasteFromClipboard({ connectInputs: true });
|
||||
const pasted = app.graph._nodes.find(item => item.type === 'PromptManagerPair' && item.id !== pair);
|
||||
const clone = node.clone();
|
||||
app.graph.add(clone);
|
||||
const output = (await app.graphToPrompt()).output;
|
||||
return { pasted: describe(pasted), clone: describe(clone), queued: output[String(pasted.id)].inputs.text, pastedId: pasted.id };
|
||||
}, ids);
|
||||
expect(state).toMatchObject({ pasted: { bound: 0, linked: true }, clone: { bound: 0, linked: false }, queued: [String(ids.string), 0] });
|
||||
// Undo restores the graph through a workflow load: the restored link is still a socket link.
|
||||
const undone = await page.evaluate(async ({ pair }) => {
|
||||
const { app } = await import('/scripts/app.js');
|
||||
const tracker = app.extensionManager?.workflow?.activeWorkflow?.changeTracker;
|
||||
if (!tracker) throw new Error('no change tracker on the active workflow');
|
||||
tracker.checkState();
|
||||
const node = app.graph.getNodeById(pair);
|
||||
node.disconnectInput(node.inputs.findIndex(input => input.name === 'text'));
|
||||
tracker.checkState();
|
||||
await tracker.undo();
|
||||
const restored = app.graph.getNodeById(pair);
|
||||
return { bound: restored.inputs.filter(input => ['text', 'negative_text'].includes(input.name) && input.widget).length,
|
||||
text: (await app.graphToPrompt()).output[String(pair)].inputs.text };
|
||||
}, ids);
|
||||
expect(undone).toEqual({ bound: 0, text: [String(ids.string), 0] });
|
||||
});
|
||||
|
||||
test('PromptSearchList text is a core widget socket and takes a dragged STRING link too', async ({ page }) => {
|
||||
const { id, sources } = await page.evaluate(async () => {
|
||||
const { app } = await import('/scripts/app.js');
|
||||
const node = LiteGraph.createNode('PromptSearchList');
|
||||
app.graph.add(node);
|
||||
node.pos = [500, 80];
|
||||
const source = LiteGraph.createNode('PrimitiveString');
|
||||
app.graph.add(source);
|
||||
source.pos = [40, 80];
|
||||
app.canvas.ds.scale = .8;
|
||||
app.canvas.ds.offset = [40, 40];
|
||||
app.canvas.setDirty(true, true);
|
||||
return { id: node.id, sources: [source.id] };
|
||||
});
|
||||
// A core widget's socket shows while a link is dragged; aim at where the frontend places it.
|
||||
await expect.poll(async () => Boolean(await slotPosition(page, id, 'in', 'text'))).toBe(true);
|
||||
await drag(page, await slotPosition(page, sources[0], 'out', 0), await slotPosition(page, id, 'in', 'text'));
|
||||
await expect.poll(() => linkedInputs(page, id)).toEqual(['text']);
|
||||
expect((await queued(page, id)).text).toEqual([String(sources[0]), 0]);
|
||||
});
|
||||
|
||||
test('workflows saved before #343 load unchanged and their linked text keeps working', async ({ page }) => {
|
||||
const workflow = JSON.parse(await readFile(new URL('./fixtures/workflow-before-343.json', import.meta.url), 'utf8'));
|
||||
const result = await page.evaluate(async workflow => {
|
||||
const { app } = await import('/scripts/app.js');
|
||||
await app.loadGraphData(structuredClone(workflow));
|
||||
const output = (await app.graphToPrompt()).output;
|
||||
const nodes = Object.fromEntries(app.graph._nodes.map(node => [node.id, node]));
|
||||
const reloaded = app.graph.serialize();
|
||||
return {
|
||||
values: Object.fromEntries(reloaded.nodes.map(node => [node.id, node.widgets_values])),
|
||||
links: Object.fromEntries(Object.entries(output).map(([id, node]) => [id, { text: node.inputs.text, negative_text: node.inputs.negative_text }])),
|
||||
widgetBound: Object.fromEntries(Object.values(nodes).filter(node => node.type.startsWith('PromptManager')).map(node => [node.id, node.inputs.filter(input => ['text', 'negative_text'].includes(input.name) && input.widget).map(input => input.name)])),
|
||||
};
|
||||
}, workflow);
|
||||
for (const node of workflow.nodes) {
|
||||
// Every saved widget value comes back as it was saved, except the typed text main lost under a link (null).
|
||||
if (node.widgets_values?.[0] !== null) expect(result.values[node.id], `${node.type} ${node.id}`).toEqual(node.widgets_values);
|
||||
}
|
||||
expect(result.links).toMatchObject({
|
||||
1: { text: 'Saved PromptManager text' }, 2: { text: 'Saved PromptManagerText text' }, 3: { text: 'Saved PromptManagerFocused text' },
|
||||
4: { text: 'Saved PromptManagerSplit text' }, 5: { text: 'Saved PromptManagerPair text', negative_text: 'Saved pair negative' },
|
||||
6: { text: ['7', 0] }, 8: { text: ['9', 0], negative_text: ['10', 0] }, 11: { text: ['12', 0] },
|
||||
});
|
||||
// The editor nodes' text inputs are sockets now; the Classic nodes keep their core widget sockets.
|
||||
expect(result.widgetBound).toEqual({ 1: ['text'], 2: ['text'], 3: [], 4: [], 5: [], 6: [], 8: [], 11: ['text'] });
|
||||
// The linked sockets are drawn where a user can see and re-drag them, in either renderer.
|
||||
for (const [node, name] of [[6, 'text'], [8, 'text'], [8, 'negative_text']]) {
|
||||
await page.evaluate(async node => { const { app } = await import('/scripts/app.js'); app.canvas.centerOnNode(app.graph.getNodeById(node)); app.canvas.setDirty(true, true); }, node);
|
||||
await expect.poll(async () => {
|
||||
const position = await slotPosition(page, node, 'in', name);
|
||||
return Boolean(position && position[0] > 0 && position[1] > 0 && position[0] < 1600 && position[1] < 1100);
|
||||
}, { message: `${node}.${name} socket on screen` }).toBe(true);
|
||||
}
|
||||
const unplugged = await page.evaluate(async () => {
|
||||
const { app } = await import('/scripts/app.js');
|
||||
for (const id of [6, 8]) { const node = app.graph.getNodeById(id); for (const input of [...node.inputs]) if (input.link != null) node.disconnectInput(node.inputs.indexOf(input)); }
|
||||
const output = (await app.graphToPrompt()).output;
|
||||
return { focused: output['6'].inputs.text, positive: output['8'].inputs.text, negative: output['8'].inputs.negative_text };
|
||||
});
|
||||
expect(unplugged).toEqual({ focused: 'Typed under focused link', positive: 'Typed pair positive', negative: 'Typed pair negative' });
|
||||
});
|
||||
+51
@@ -186,6 +186,57 @@
|
||||
"prompt": null,
|
||||
"workflow": "{\"nodes\": [{\"id\": 4, \"type\": \"CheckpointLoaderSimple\", \"mode\": 0, \"inputs\": [], \"widgets_values\": [\"example.safetensors\"]}, {\"id\": 6, \"type\": \"CLIPTextEncode\", \"mode\": 0, \"inputs\": [{\"name\": \"clip\", \"link\": 3}], \"widgets_values\": [\"a tidepool at noon\"]}, {\"id\": 7, \"type\": \"CLIPTextEncode\", \"mode\": 0, \"inputs\": [{\"name\": \"clip\", \"link\": 5}], \"widgets_values\": [\"noisy\"]}, {\"id\": 10, \"type\": \"Reroute\", \"mode\": 0, \"inputs\": [{\"name\": \"\", \"link\": 7}]}, {\"id\": 3, \"type\": \"KSampler\", \"mode\": 0, \"inputs\": [{\"name\": \"model\", \"link\": 1}, {\"name\": \"positive\", \"link\": 8}, {\"name\": \"negative\", \"link\": 4}, {\"name\": \"latent_image\", \"link\": 2}], \"widgets_values\": [314, \"randomize\", 25, 6.5, \"dpmpp_2m\", \"karras\", 1]}, {\"id\": 11, \"type\": \"CLIPTextEncode\", \"mode\": 2, \"inputs\": [], \"widgets_values\": [\"muted node text\"]}], \"links\": [[1, 4, 0, 3, 0, \"MODEL\"], [3, 4, 1, 6, 0, \"CLIP\"], [5, 4, 1, 7, 0, \"CLIP\"], [7, 6, 0, 10, 0, \"CONDITIONING\"], [8, 10, 0, 3, 1, \"CONDITIONING\"], [4, 7, 0, 3, 2, \"CONDITIONING\"], {\"id\": 2, \"origin_id\": 99, \"origin_slot\": 0, \"target_id\": 3, \"target_slot\": 3, \"type\": \"LATENT\"}]}"
|
||||
},
|
||||
{
|
||||
"expected": {
|
||||
"ok": true,
|
||||
"result": {
|
||||
"cfgScale": 5,
|
||||
"checkpoint": "example.safetensors",
|
||||
"negativePrompt": "plain negative",
|
||||
"positivePrompt": "a detached positive",
|
||||
"sampler": "euler",
|
||||
"seed": 7,
|
||||
"steps": 12
|
||||
}
|
||||
},
|
||||
"name": "case:workflow-only-detached-editor-text",
|
||||
"prompt": null,
|
||||
"workflow": "{\"nodes\": [{\"id\": 4, \"type\": \"CheckpointLoaderSimple\", \"mode\": 0, \"inputs\": [], \"widgets_values\": [\"example.safetensors\"]}, {\"id\": 6, \"type\": \"PromptManagerFocused\", \"mode\": 0, \"inputs\": [{\"name\": \"clip\", \"link\": 3}, {\"name\": \"prepend_text\", \"link\": null}, {\"name\": \"append_text\", \"link\": null}, {\"name\": \"text\", \"link\": null}, {\"name\": \"category\", \"widget\": {\"name\": \"category\"}, \"link\": null}, {\"name\": \"tags\", \"widget\": {\"name\": \"tags\"}, \"link\": null}, {\"name\": \"search_text\", \"widget\": {\"name\": \"search_text\"}, \"link\": null}], \"widgets_values\": [\"a detached positive\", \"a category\", \"a tag\", \"\", \"\", \"\"]}, {\"id\": 7, \"type\": \"CLIPTextEncode\", \"mode\": 0, \"inputs\": [{\"name\": \"clip\", \"link\": 5}], \"widgets_values\": [\"plain negative\"]}, {\"id\": 3, \"type\": \"KSampler\", \"mode\": 0, \"inputs\": [{\"name\": \"model\", \"link\": 1}, {\"name\": \"positive\", \"link\": 8}, {\"name\": \"negative\", \"link\": 4}], \"widgets_values\": [7, \"fixed\", 12, 5, \"euler\", \"normal\", 1]}], \"links\": [[1, 4, 0, 3, 0, \"MODEL\"], [3, 4, 1, 6, 0, \"CLIP\"], [5, 4, 1, 7, 0, \"CLIP\"], [8, 6, 0, 3, 1, \"CONDITIONING\"], [4, 7, 0, 3, 2, \"CONDITIONING\"]]}"
|
||||
},
|
||||
{
|
||||
"expected": {
|
||||
"ok": true,
|
||||
"result": {
|
||||
"cfgScale": 5,
|
||||
"checkpoint": "example.safetensors",
|
||||
"negativePrompt": "a detached negative",
|
||||
"positivePrompt": "a detached positive",
|
||||
"sampler": "euler",
|
||||
"seed": 7,
|
||||
"steps": 12
|
||||
}
|
||||
},
|
||||
"name": "case:workflow-only-detached-pair-text",
|
||||
"prompt": null,
|
||||
"workflow": "{\"nodes\": [{\"id\": 4, \"type\": \"CheckpointLoaderSimple\", \"mode\": 0, \"inputs\": [], \"widgets_values\": [\"example.safetensors\"]}, {\"id\": 6, \"type\": \"PromptManagerPair\", \"mode\": 0, \"inputs\": [{\"name\": \"clip\", \"link\": 3}, {\"name\": \"prepend_text\", \"link\": null}, {\"name\": \"append_text\", \"link\": null}, {\"name\": \"text\", \"link\": null}, {\"name\": \"negative_text\", \"link\": null}, {\"name\": \"category\", \"widget\": {\"name\": \"category\"}, \"link\": null}, {\"name\": \"tags\", \"widget\": {\"name\": \"tags\"}, \"link\": null}, {\"name\": \"search_text\", \"widget\": {\"name\": \"search_text\"}, \"link\": null}], \"widgets_values\": [\"a detached positive\", \"a detached negative\", \"a category\", \"a tag\", \"\", \"\", \"\"]}, {\"id\": 3, \"type\": \"KSampler\", \"mode\": 0, \"inputs\": [{\"name\": \"model\", \"link\": 1}, {\"name\": \"positive\", \"link\": 8}, {\"name\": \"negative\", \"link\": 4}], \"widgets_values\": [7, \"fixed\", 12, 5, \"euler\", \"normal\", 1]}], \"links\": [[1, 4, 0, 3, 0, \"MODEL\"], [3, 4, 1, 6, 0, \"CLIP\"], [5, 4, 1, 7, 0, \"CLIP\"], [8, 6, 0, 3, 1, \"CONDITIONING\"], [4, 6, 1, 3, 2, \"CONDITIONING\"]]}"
|
||||
},
|
||||
{
|
||||
"expected": {
|
||||
"ok": true,
|
||||
"result": {
|
||||
"cfgScale": 4.5,
|
||||
"checkpoint": "example.safetensors",
|
||||
"negativePrompt": "plain negative",
|
||||
"positivePrompt": "a detached positive",
|
||||
"sampler": "dpmpp_2m",
|
||||
"seed": 9,
|
||||
"steps": 30
|
||||
}
|
||||
},
|
||||
"name": "case:workflow-only-core-widget-input-out-of-order",
|
||||
"prompt": null,
|
||||
"workflow": "{\"nodes\": [{\"id\": 4, \"type\": \"CheckpointLoaderSimple\", \"mode\": 0, \"inputs\": [], \"widgets_values\": [\"example.safetensors\"]}, {\"id\": 6, \"type\": \"PromptManagerFocused\", \"mode\": 0, \"inputs\": [{\"name\": \"clip\", \"link\": 3}, {\"name\": \"prepend_text\", \"link\": null}, {\"name\": \"append_text\", \"link\": null}, {\"name\": \"text\", \"link\": null}, {\"name\": \"category\", \"widget\": {\"name\": \"category\"}, \"link\": null}, {\"name\": \"tags\", \"widget\": {\"name\": \"tags\"}, \"link\": null}, {\"name\": \"search_text\", \"widget\": {\"name\": \"search_text\"}, \"link\": null}], \"widgets_values\": [\"a detached positive\", \"a category\", \"a tag\", \"\", \"\", \"\"]}, {\"id\": 7, \"type\": \"CLIPTextEncode\", \"mode\": 0, \"inputs\": [{\"name\": \"clip\", \"link\": 5}], \"widgets_values\": [\"plain negative\"]}, {\"id\": 3, \"type\": \"KSampler\", \"mode\": 0, \"inputs\": [{\"name\": \"model\", \"link\": 1}, {\"name\": \"positive\", \"link\": 8}, {\"name\": \"negative\", \"link\": 4}, {\"name\": \"cfg\", \"widget\": {\"name\": \"cfg\"}, \"link\": null}], \"widgets_values\": [9, \"fixed\", 30, 4.5, \"dpmpp_2m\", \"karras\", 1]}], \"links\": [[1, 4, 0, 3, 0, \"MODEL\"], [3, 4, 1, 6, 0, \"CLIP\"], [5, 4, 1, 7, 0, \"CLIP\"], [8, 6, 0, 3, 1, \"CONDITIONING\"], [4, 7, 0, 3, 2, \"CONDITIONING\"]]}"
|
||||
},
|
||||
{
|
||||
"expected": {
|
||||
"ok": true,
|
||||
|
||||
@@ -244,6 +244,90 @@ def _workflow_only():
|
||||
}
|
||||
|
||||
|
||||
def _detached_editor_text(pair):
|
||||
"""#343: editor text inputs are sockets, not widget inputs; no named values."""
|
||||
names = ["text"] + (["negative_text"] if pair else [])
|
||||
sockets = [
|
||||
{"name": "clip", "link": 3},
|
||||
{"name": "prepend_text", "link": None},
|
||||
{"name": "append_text", "link": None},
|
||||
]
|
||||
sockets += [{"name": name, "link": None} for name in names]
|
||||
sockets += [
|
||||
{"name": name, "widget": {"name": name}, "link": None}
|
||||
for name in ("category", "tags", "search_text")
|
||||
]
|
||||
editor = {
|
||||
"id": 6,
|
||||
"type": "PromptManagerPair" if pair else "PromptManagerFocused",
|
||||
"mode": 0,
|
||||
"inputs": sockets,
|
||||
"widgets_values": ["a detached positive"]
|
||||
+ (["a detached negative"] if pair else [])
|
||||
+ ["a category", "a tag", "", "", ""],
|
||||
}
|
||||
negative = (
|
||||
[]
|
||||
if pair
|
||||
else [
|
||||
{
|
||||
"id": 7,
|
||||
"type": "CLIPTextEncode",
|
||||
"mode": 0,
|
||||
"inputs": [{"name": "clip", "link": 5}],
|
||||
"widgets_values": ["plain negative"],
|
||||
}
|
||||
]
|
||||
)
|
||||
return {
|
||||
"nodes": [
|
||||
{
|
||||
"id": 4,
|
||||
"type": "CheckpointLoaderSimple",
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"widgets_values": ["example.safetensors"],
|
||||
},
|
||||
editor,
|
||||
*negative,
|
||||
{
|
||||
"id": 3,
|
||||
"type": "KSampler",
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{"name": "model", "link": 1},
|
||||
{"name": "positive", "link": 8},
|
||||
{"name": "negative", "link": 4},
|
||||
],
|
||||
"widgets_values": [7, "fixed", 12, 5, "euler", "normal", 1],
|
||||
},
|
||||
],
|
||||
"links": [
|
||||
[1, 4, 0, 3, 0, "MODEL"],
|
||||
[3, 4, 1, 6, 0, "CLIP"],
|
||||
[5, 4, 1, 7, 0, "CLIP"],
|
||||
[8, 6, 0, 3, 1, "CONDITIONING"],
|
||||
[4, 6 if pair else 7, 1 if pair else 0, 3, 2, "CONDITIONING"],
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _core_widget_input_out_of_order():
|
||||
"""#343 review R4: a core KSampler whose only widget input is ``cfg``.
|
||||
|
||||
Older frontends saved only converted or linked widget inputs, so the list
|
||||
is not a prefix of the schema and carries no named values; the positional
|
||||
values must still land on the schema's names.
|
||||
"""
|
||||
workflow = _detached_editor_text(False)
|
||||
sampler = workflow["nodes"][-1]
|
||||
sampler["inputs"] = sampler["inputs"] + [
|
||||
{"name": "cfg", "widget": {"name": "cfg"}, "link": None}
|
||||
]
|
||||
sampler["widgets_values"] = [9, "fixed", 30, 4.5, "dpmpp_2m", "karras", 1]
|
||||
return workflow
|
||||
|
||||
|
||||
def _nested(depth):
|
||||
"""A value nested ``depth`` arrays deep, as a crafted file can hold."""
|
||||
value = ["leaf", 7]
|
||||
@@ -389,6 +473,11 @@ CASES = {
|
||||
}
|
||||
),
|
||||
"workflow-only": case(workflow=_workflow_only()),
|
||||
"workflow-only-detached-editor-text": case(workflow=_detached_editor_text(False)),
|
||||
"workflow-only-detached-pair-text": case(workflow=_detached_editor_text(True)),
|
||||
"workflow-only-core-widget-input-out-of-order": case(
|
||||
workflow=_core_widget_input_out_of_order()
|
||||
),
|
||||
"overwritten-prompt-manager-text": case(
|
||||
{
|
||||
"4": CHECKPOINT,
|
||||
|
||||
@@ -356,6 +356,21 @@ test("keepalive is forwarded only when asked, so a request can outlive its page"
|
||||
await Promise.all(writes);
|
||||
});
|
||||
|
||||
test("ApiError keeps the job that applied a used plan (plan-used, System S7)", () => {
|
||||
const body = { job: { id: "b".repeat(32), title: "Optimize database" } };
|
||||
const used = new ApiError(409, "This plan was already applied.", "plan-used", null, body);
|
||||
assert.deepEqual(used.job, { id: "b".repeat(32), title: "Optimize database" });
|
||||
assert.notEqual(used.job, body.job, "server objects must be copied");
|
||||
// The applying job may already be pruned: its title is null, its id still links Activity.
|
||||
assert.deepEqual(new ApiError(409, "used", "plan-used", null, { job: { id: "c".repeat(32), title: null } }).job,
|
||||
{ id: "c".repeat(32), title: null });
|
||||
assert.equal(new ApiError(409, "busy", "busy", null, body).job, null);
|
||||
for (const job of [null, "x", { id: 1, title: "t" }, { id: "a b", title: "t" }, { id: "a", title: 7 },
|
||||
{ id: "a", title: "x".repeat(201) }, { id: "a".repeat(65), title: "t" }]) {
|
||||
assert.equal(new ApiError(409, "used", "plan-used", null, { job }).job, null);
|
||||
}
|
||||
});
|
||||
|
||||
test("ApiError keeps a validated running job on busy and a validated source on invalid-source", () => {
|
||||
const body = { running: { id: "a".repeat(32), title: "Scan for metadata" }, source: "output/gone" };
|
||||
const busy = new ApiError(409, "busy", "busy", null, body);
|
||||
@@ -372,3 +387,12 @@ test("ApiError keeps a validated running job on busy and a validated source on i
|
||||
}
|
||||
assert.deepEqual([new ApiError(409, "old caller").running, new ApiError(409, "old caller").source], [null, null]);
|
||||
});
|
||||
|
||||
test("ApiError says whether the server itself refused (a parsed success:false body; S7 review I1)", () => {
|
||||
assert.equal(new ApiError(400, "bad", null, null, { success: false, error: "bad" }).refused, true);
|
||||
assert.equal(new ApiError(409, "busy", "busy", null, { success: false, code: "busy" }).refused, true);
|
||||
// A proxy's page, a body that was not JSON, or no body at all: nothing says the server refused.
|
||||
for (const body of [null, undefined, "<html>", { success: true }, [], { error: "x" }]) {
|
||||
assert.equal(new ApiError(504, "Gateway Timeout", null, null, body).refused, false);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
// #222: what "clear the cache" may remove from this browser. Only entries Prompt Manager itself writes as a cache count,
|
||||
// from an inventory of the real writers (never a guessed prefix): preferences (the workspace theme, the update notice,
|
||||
// the display defaults) and other tools' storage and caches on the shared ComfyUI origin always stay.
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
|
||||
const CacheOwnership = require('../../web/js/cache-ownership.js');
|
||||
|
||||
const WEB = path.join(__dirname, '../../web');
|
||||
|
||||
function memoryStorage(entries) {
|
||||
const map = new Map(Object.entries(entries));
|
||||
return {
|
||||
map,
|
||||
get length() { return map.size; },
|
||||
key: index => [...map.keys()][index] ?? null,
|
||||
getItem: key => (map.has(key) ? map.get(key) : null),
|
||||
setItem: (key, value) => map.set(key, String(value)),
|
||||
removeItem: key => map.delete(key),
|
||||
};
|
||||
}
|
||||
|
||||
function memoryCaches(names, { failDelete = new Set() } = {}) {
|
||||
const set = new Set(names);
|
||||
return {
|
||||
set,
|
||||
keys: async () => [...set],
|
||||
delete: async name => {
|
||||
if (failDelete.has(name)) throw new Error('denied');
|
||||
return set.delete(name);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const PREFERENCES = { gallerySettings: '{"viewMode":"grid"}', pm_workspace_theme: 'light', pm_update_notice_dismissed: '3.3.0' };
|
||||
const OWNED = Object.freeze({ storageKeys: ['pm-owned-test-entry'], cacheNames: ['pm-owned-test-cache'] });
|
||||
|
||||
test('keeps pm_workspace_theme, pm_update_notice_dismissed and gallerySettings', async () => {
|
||||
const storage = memoryStorage({ ...PREFERENCES, 'pm-owned-test-entry': 'x', pm_cached: '1', promptManager_x: '1' });
|
||||
const result = await CacheOwnership.clearOwned({ storage, cacheStorage: null, owned: OWNED });
|
||||
assert.deepEqual(result.removed, ['pm-owned-test-entry']);
|
||||
for (const [key, value] of Object.entries(PREFERENCES)) assert.equal(storage.getItem(key), value, key);
|
||||
// The old prefix rule's matches are not Prompt Manager caches either: nothing writes them.
|
||||
assert.equal(storage.getItem('pm_cached'), '1');
|
||||
assert.equal(storage.getItem('promptManager_x'), '1');
|
||||
for (const key of Object.keys(PREFERENCES)) assert.ok(CacheOwnership.PREFERENCE_KEYS.includes(key), key);
|
||||
});
|
||||
|
||||
test("keeps other tools' CacheStorage named gallery/thumbnail", async () => {
|
||||
const caches = memoryCaches(['other-gallery', 'thumbnail-cache', 'my-gallery-thumbnails', 'pm-owned-test-cache']);
|
||||
const result = await CacheOwnership.clearOwned({ storage: memoryStorage({}), cacheStorage: caches, owned: OWNED });
|
||||
assert.deepEqual([...caches.set].sort(), ['my-gallery-thumbnails', 'other-gallery', 'thumbnail-cache']);
|
||||
assert.deepEqual(result.removed, ['pm-owned-test-cache']);
|
||||
});
|
||||
|
||||
test('removes only inventoried owned keys', async () => {
|
||||
// The shipped inventory: no Prompt Manager writer stores a cache entry in this browser.
|
||||
assert.deepEqual(CacheOwnership.OWNED, { storageKeys: [], cacheNames: [] });
|
||||
assert.ok(Object.isFrozen(CacheOwnership.OWNED.storageKeys) && Object.isFrozen(CacheOwnership.OWNED.cacheNames));
|
||||
const storage = memoryStorage({ ...PREFERENCES, gallery_cache_1: 'a', metadata_cache_2: 'b', other: 'c' });
|
||||
const caches = memoryCaches(['gallery', 'thumbnail']);
|
||||
const result = await CacheOwnership.clearOwned({ storage, cacheStorage: caches });
|
||||
assert.deepEqual(result, { removed: [], failed: [], denied: false, partial: false });
|
||||
assert.equal(storage.map.size, 6);
|
||||
assert.equal(caches.set.size, 2);
|
||||
assert.equal(CacheOwnership.clearMessage(result), CacheOwnership.NOTHING_CACHED);
|
||||
});
|
||||
|
||||
/**
|
||||
* Every browser-storage write in one file's text: `{keys}` (setItem keys, as written) and `problems` (writes no key
|
||||
* can be read from: a bracket or property assignment on a Storage, a computed setItem key, an IndexedDB or
|
||||
* CacheStorage open or write). The scan is textual, so it errs on the side of reporting.
|
||||
*/
|
||||
function storageWrites(text) {
|
||||
const keys = [];
|
||||
const problems = [];
|
||||
for (const match of text.matchAll(/\.setItem\(\s*([^,)]+)/g)) {
|
||||
const key = match[1].trim();
|
||||
const literal = /^(?:'([^']+)'|"([^"]+)")$/.exec(key);
|
||||
if (literal) keys.push(literal[1] ?? literal[2]);
|
||||
else if (/^[A-Z_]+_KEY$/.test(key)) keys.push(`$${key}`);
|
||||
else problems.push(`a computed setItem key: ${key}`);
|
||||
}
|
||||
if (/\b\w*[sS]torage\s*\[[^\]]+\]\s*=(?!=)/.test(text)) problems.push('a bracket assignment on a Storage');
|
||||
if (/\b(?:local|session)Storage\s*\.\s*(?!getItem|setItem|removeItem|key\b|length\b|clear\b)\w+\s*=(?!=)/.test(text)) {
|
||||
problems.push('a property assignment on a Storage');
|
||||
}
|
||||
if (/\bcaches\s*\.\s*open\s*\(/.test(text)) problems.push('opens a CacheStorage cache');
|
||||
if (/\bindexedDB\s*\.\s*open\s*\(/.test(text)) problems.push('opens IndexedDB');
|
||||
return { keys, problems };
|
||||
}
|
||||
|
||||
test('the storage scan reports every way of writing browser storage (S7 review M5)', () => {
|
||||
assert.deepEqual(storageWrites("localStorage.setItem('gallerySettings', x); storage.setItem(NOTICE_KEY, v);").keys,
|
||||
['gallerySettings', '$NOTICE_KEY']);
|
||||
for (const text of ["localStorage['pm_cache'] = 1;", 'sessionStorage[key] = value;', 'storage[name] = x;',
|
||||
'localStorage.pm_cache = 1;', 'sessionStorage.someKey = "v";', 'localStorage.setItem(key, 1);',
|
||||
"caches.open('pm').then(cache => cache.put(url, response));", "indexedDB.open('pm-cache', 1);"]) {
|
||||
assert.equal(storageWrites(text).problems.length, 1, text);
|
||||
}
|
||||
for (const text of ["if (localStorage['pm'] === x) {}", 'const value = localStorage.getItem(KEY);',
|
||||
'localStorage.removeItem(KEY);', 'const n = localStorage.length;']) {
|
||||
assert.deepEqual(storageWrites(text), { keys: [], problems: [] }, text);
|
||||
}
|
||||
});
|
||||
|
||||
test('the inventory covers every browser storage writer in web/', () => {
|
||||
// Every key a Prompt Manager page writes is a preference or an inventoried cache entry; a new writer must be
|
||||
// classified here before a cache clear may remove it. Skipped: web/lib (third-party Viewer.js and Tailwind
|
||||
// output) and web/js/shared/toolkit (the pinned ComfyAssets Toolkit build, never hand-edited here and verified by
|
||||
// `npm run --prefix tools/vendor check`). Neither writes browser storage today; the Toolkit's own repository owns
|
||||
// its storage, and a new pin is reviewed as its own change.
|
||||
const files = [];
|
||||
const walk = dir => {
|
||||
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
const full = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) { if (!['lib', 'toolkit'].includes(entry.name)) walk(full); }
|
||||
else if (/\.(m?js|html)$/.test(entry.name)) files.push(full);
|
||||
}
|
||||
};
|
||||
walk(WEB);
|
||||
const written = new Set();
|
||||
const constants = {};
|
||||
for (const file of files) {
|
||||
const text = fs.readFileSync(file, 'utf8');
|
||||
for (const match of text.matchAll(/(?:const|export const)\s+([A-Z_]+_KEY)\s*=\s*'([^']+)'/g)) constants[match[1]] = match[2];
|
||||
const { keys, problems } = storageWrites(text);
|
||||
assert.deepEqual(problems, [], path.relative(WEB, file));
|
||||
for (const key of keys) written.add(key);
|
||||
}
|
||||
const keys = [...written].map(key => (key.startsWith('$') ? constants[key.slice(1)] : key));
|
||||
assert.ok(keys.length >= 3, 'the writers were found');
|
||||
for (const key of keys) {
|
||||
assert.ok(key, 'every written key resolves');
|
||||
assert.ok(CacheOwnership.PREFERENCE_KEYS.includes(key) || CacheOwnership.OWNED.storageKeys.includes(key), key);
|
||||
}
|
||||
});
|
||||
|
||||
test('storage denial reports partial', async () => {
|
||||
const denied = { get length() { throw new Error('SecurityError'); }, key() { throw new Error('SecurityError'); },
|
||||
getItem() { throw new Error('SecurityError'); }, removeItem() { throw new Error('SecurityError'); } };
|
||||
const result = await CacheOwnership.clearOwned({ storage: denied, cacheStorage: null, owned: OWNED });
|
||||
assert.equal(result.denied, true);
|
||||
assert.equal(result.partial, true);
|
||||
assert.equal(CacheOwnership.clearMessage(result), CacheOwnership.STORAGE_DENIED);
|
||||
|
||||
const caches = memoryCaches(['pm-owned-test-cache'], { failDelete: new Set(['pm-owned-test-cache']) });
|
||||
const storage = memoryStorage({ 'pm-owned-test-entry': 'x' });
|
||||
const partial = await CacheOwnership.clearOwned({ storage, cacheStorage: caches, owned: OWNED });
|
||||
assert.deepEqual(partial.removed, ['pm-owned-test-entry']);
|
||||
assert.deepEqual(partial.failed, ['pm-owned-test-cache']);
|
||||
assert.equal(partial.partial, true);
|
||||
assert.equal(CacheOwnership.clearMessage(partial), 'Cleared 1 of 2 cached entries; this browser refused the rest.');
|
||||
|
||||
const keysDenied = { keys: async () => { throw new Error('denied'); } };
|
||||
const unreadable = await CacheOwnership.clearOwned({ storage: memoryStorage({}), cacheStorage: keysDenied, owned: OWNED });
|
||||
assert.equal(unreadable.denied, true);
|
||||
});
|
||||
|
||||
test('server thumbnail clear stays a separate explicit call', async () => {
|
||||
const calls = [];
|
||||
const original = globalThis.fetch;
|
||||
globalThis.fetch = async (...args) => { calls.push(args); return { ok: true }; };
|
||||
try {
|
||||
const storage = memoryStorage({ 'pm-owned-test-entry': 'x' });
|
||||
const result = await CacheOwnership.clearOwned({ storage, cacheStorage: memoryCaches([]), owned: OWNED });
|
||||
assert.deepEqual(result.removed, ['pm-owned-test-entry']);
|
||||
assert.equal(CacheOwnership.clearMessage(result), 'Cleared 1 cached entry from this browser.');
|
||||
} finally {
|
||||
globalThis.fetch = original;
|
||||
}
|
||||
assert.deepEqual(calls, []);
|
||||
assert.equal(/fetch|clear-thumbnails|ApiClient/.test(fs.readFileSync(path.join(WEB, 'js/cache-ownership.js'), 'utf8')), false);
|
||||
});
|
||||
|
||||
test('ownedEntries lists what a clear would remove, reading nothing it cannot', async () => {
|
||||
const storage = memoryStorage({ ...PREFERENCES, 'pm-owned-test-entry': 'x' });
|
||||
const caches = memoryCaches(['pm-owned-test-cache', 'gallery']);
|
||||
assert.deepEqual(await CacheOwnership.ownedEntries({ storage, cacheStorage: caches, owned: OWNED }),
|
||||
{ keys: ['pm-owned-test-entry'], caches: ['pm-owned-test-cache'], denied: false });
|
||||
assert.deepEqual(await CacheOwnership.ownedEntries({ storage, cacheStorage: caches }),
|
||||
{ keys: [], caches: [], denied: false });
|
||||
assert.deepEqual(await CacheOwnership.ownedEntries({ storage: null, cacheStorage: undefined, owned: OWNED }),
|
||||
{ keys: [], caches: [], denied: false });
|
||||
assert.equal(CacheOwnership.clearMessage({ removed: ['a', 'b'], failed: [], denied: false, partial: false }),
|
||||
'Cleared 2 cached entries from this browser.');
|
||||
});
|
||||
@@ -32,8 +32,11 @@ function sseResponse(frames) {
|
||||
function load(file, className, fetchUrl, extra = {}, dom = elements()) {
|
||||
const timers = [];
|
||||
const notifications = [];
|
||||
const storage = new Map([['pm_cached', '1'], ['userData', 'keep']]);
|
||||
const storage = new Map([['pm_cached', '1'], ['userData', 'keep'], ['pm_workspace_theme', 'light'],
|
||||
['pm_update_notice_dismissed', '3.3.0']]);
|
||||
const localStorage = {
|
||||
get length() { return storage.size; },
|
||||
key: index => [...storage.keys()][index] ?? null,
|
||||
getItem: key => storage.get(key) ?? null,
|
||||
setItem: (key, value) => storage.set(key, String(value)),
|
||||
removeItem: key => storage.delete(key),
|
||||
@@ -45,11 +48,12 @@ function load(file, className, fetchUrl, extra = {}, dom = elements()) {
|
||||
setTimeout: callback => timers.push(callback),
|
||||
confirm: () => true,
|
||||
localStorage,
|
||||
Object: { ...Object, keys: target => (target === localStorage ? [...storage.keys()] : Object.keys(target)) },
|
||||
TextDecoder, TextEncoder, JSON,
|
||||
ApiClient: { fetchUrl },
|
||||
...extra,
|
||||
});
|
||||
// gallery.html loads cache-ownership.js before gallery.js (#222).
|
||||
vm.runInContext(fs.readFileSync(path.join(__dirname, '../../web/js/cache-ownership.js'), 'utf8'), context);
|
||||
const source = fs.readFileSync(path.join(__dirname, `../../web/js/${file}`), 'utf8');
|
||||
const Page = vm.runInContext(`${source}\n${className};`, context, { filename: path.join(__dirname, `../../web/js/${file}`) });
|
||||
const page = Object.create(Page.prototype);
|
||||
@@ -78,18 +82,68 @@ test('gallery cache clear reports a refused server clear and never claims succes
|
||||
assert.equal(storage.get('gallerySettings'), undefined); // thumbnailsGenerated is not reset
|
||||
});
|
||||
|
||||
test('gallery cache clear still says so when the server clears', async () => {
|
||||
const { page, dom, notifications, storage } = load('gallery.js', 'GalleryManager',
|
||||
async () => jsonResponse(200, { success: true, cleared_count: 3 }));
|
||||
test('gallery cache clear says what it cleared and keeps preferences and other data (#222)', async () => {
|
||||
const calls = [];
|
||||
const { page, dom, notifications, storage } = load('gallery.js', 'GalleryManager', async url => {
|
||||
calls.push(url);
|
||||
return jsonResponse(200, { success: true, cleared_count: 3 });
|
||||
});
|
||||
page.getSettings = () => ({ thumbnailsGenerated: true });
|
||||
|
||||
await page.clearCache();
|
||||
|
||||
assert.deepEqual(notifications, [['success', 'PromptManager cache cleared successfully! Original images untouched.']]);
|
||||
// The server's thumbnails are cleared by their own explicit call; nothing Prompt Manager owns is cached in this
|
||||
// browser, so no browser entry is removed and the toast says exactly that.
|
||||
assert.deepEqual(calls, ['/prompt_manager/images/clear-thumbnails']);
|
||||
assert.deepEqual(notifications, [['success',
|
||||
"Thumbnails cleared; original images untouched. Nothing of Prompt Manager's is cached in this browser."]]);
|
||||
assert.equal(dom.nodes.thumbnailStatus.textContent, 'Not generated');
|
||||
assert.equal(JSON.parse(storage.get('gallerySettings')).thumbnailsGenerated, false);
|
||||
assert.equal(storage.get('userData'), 'keep');
|
||||
assert.equal(storage.has('pm_cached'), false);
|
||||
assert.equal(storage.get('pm_cached'), '1');
|
||||
assert.equal(storage.get('pm_workspace_theme'), 'light');
|
||||
assert.equal(storage.get('pm_update_notice_dismissed'), '3.3.0');
|
||||
});
|
||||
|
||||
test("gallery cache clear never deletes another tool's gallery or thumbnail caches (#222)", async () => {
|
||||
const names = new Set(['comfy-gallery', 'thumbnail-cache']);
|
||||
const caches = { keys: async () => [...names], delete: async name => names.delete(name) };
|
||||
const { page, notifications } = load('gallery.js', 'GalleryManager',
|
||||
async () => jsonResponse(200, { success: true }), { caches, window: { caches } });
|
||||
page.getSettings = () => ({});
|
||||
|
||||
await page.clearCache();
|
||||
|
||||
assert.deepEqual([...names], ['comfy-gallery', 'thumbnail-cache']);
|
||||
assert.equal(notifications.at(-1)[0], 'success');
|
||||
});
|
||||
|
||||
test('gallery cache clear reports a lost server clear instead of success', async () => {
|
||||
const { page, dom, notifications, storage } = load('gallery.js', 'GalleryManager', async () => {
|
||||
throw new TypeError('Failed to fetch');
|
||||
});
|
||||
page.getSettings = () => ({ thumbnailsGenerated: true });
|
||||
dom.getElementById('thumbnailStatus').textContent = 'Generated';
|
||||
|
||||
await page.clearCache();
|
||||
|
||||
assert.deepEqual(notifications, [['error', 'Thumbnails were not cleared: the server could not be reached.']]);
|
||||
assert.equal(dom.nodes.thumbnailStatus.textContent, 'Generated');
|
||||
assert.equal(storage.get('gallerySettings'), undefined);
|
||||
});
|
||||
|
||||
test('gallery cache clear says so when browser storage is denied', async () => {
|
||||
const denied = { get length() { throw new Error('SecurityError'); }, key() { throw new Error('SecurityError'); },
|
||||
getItem() { throw new Error('SecurityError'); }, setItem() { throw new Error('SecurityError'); },
|
||||
removeItem() { throw new Error('SecurityError'); } };
|
||||
const { page, notifications } = load('gallery.js', 'GalleryManager',
|
||||
async () => jsonResponse(200, { success: true }), { localStorage: denied });
|
||||
page.getSettings = () => ({});
|
||||
|
||||
await page.clearCache();
|
||||
|
||||
assert.deepEqual(notifications, [['warning',
|
||||
'Thumbnails cleared; original images untouched. This browser refused access to its storage, so nothing there was cleared.']]);
|
||||
});
|
||||
|
||||
test('admin LoRA import shows a refusal frame and closes its modal', async () => {
|
||||
|
||||
@@ -17,6 +17,17 @@ test('lists only explicit PromptManager destinations and roles for all five sche
|
||||
assert.equal(targets[0].id,'0'); assert.equal(targets[0].label,'My prompts');
|
||||
assert.deepEqual(createPromptAdapter({}).listTargets(),[]);
|
||||
});
|
||||
test('each destination says which renderer its window draws nodes with (System › Integrations, S10)',async()=>{
|
||||
const {createPromptAdapter}=await source; const {app}=fixture();
|
||||
assert.equal(createPromptAdapter(app,{runtime:{LiteGraph:{vueNodesMode:true}}}).listTargets()[0].renderer,'nodes2');
|
||||
assert.equal(createPromptAdapter(app,{runtime:{LiteGraph:{vueNodesMode:false}}}).listTargets()[0].renderer,'classic');
|
||||
assert.equal(createPromptAdapter(app,{runtime:{LiteGraph:{}}}).listTargets()[0].renderer,'classic');
|
||||
assert.equal(Object.hasOwn(createPromptAdapter(app,{runtime:{}}).listTargets()[0],'renderer'),false);
|
||||
// Read at every listing: a window that switches renderer reports the new one.
|
||||
const runtime={LiteGraph:{vueNodesMode:false}}; const adapter=createPromptAdapter(app,{runtime});
|
||||
runtime.LiteGraph.vueNodesMode=true;
|
||||
assert.equal(adapter.listTargets()[0].renderer,'nodes2');
|
||||
});
|
||||
test('previews exact composition, updates requested visible/serialized values, preserves other fields',async()=>{
|
||||
const {createPromptAdapter}=await source; const {app,node,widget}=fixture(); const adapter=createPromptAdapter(app);
|
||||
const {preview,snapshot}=adapter.preview('7',{positive:'new'});
|
||||
|
||||
@@ -0,0 +1,201 @@
|
||||
// #343: a link can drive the editor nodes' text inputs (web/js/text-links.js).
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const source = import('../../web/js/text-links.js');
|
||||
|
||||
// The frontend's own slot order for the Focused/Split/Pair schemas: forceInput sockets first, then the widget sockets.
|
||||
function pairNode() {
|
||||
const inputs = [
|
||||
{name:'clip',type:'CLIP',link:null},
|
||||
{name:'prepend_text',type:'STRING',link:null},
|
||||
{name:'append_text',type:'STRING',link:null},
|
||||
{name:'text',type:'STRING',link:null,widget:{name:'text'},_widget:{name:'text'}},
|
||||
{name:'negative_text',type:'STRING',link:null,widget:{name:'negative_text'}},
|
||||
{name:'category',type:'STRING',link:null,widget:{name:'category'}},
|
||||
];
|
||||
return {inputs};
|
||||
}
|
||||
|
||||
test('the editor note says the value comes from a link and that typed text is kept', async () => {
|
||||
const {linkNoteText} = await source;
|
||||
assert.equal(linkNoteText('Negative prompt'), 'Negative prompt comes from a link. Your typed text is kept for when you disconnect it.');
|
||||
});
|
||||
|
||||
test('a linked widget stores its typed value; an unlinked one its live editor text', async () => {
|
||||
const {storedWidgetValue} = await source;
|
||||
const widget = {name:'text', value:'typed', inputEl:{value:'live'}};
|
||||
assert.equal(storedWidgetValue({inputs:[{name:'text',link:3}]}, widget), 'typed');
|
||||
assert.equal(storedWidgetValue({inputs:[{name:'text',link:null}]}, widget), 'live');
|
||||
assert.equal(storedWidgetValue({}, {name:'tags', value:'a, b'}), 'a, b');
|
||||
});
|
||||
|
||||
test('exposing a socket detaches only the named inputs from their hidden widgets', async () => {
|
||||
const {exposeTextSockets} = await source;
|
||||
const node = pairNode();
|
||||
assert.deepEqual(exposeTextSockets(node, ['text','negative_text']), ['text','negative_text']);
|
||||
const byName = Object.fromEntries(node.inputs.map(input => [input.name,input]));
|
||||
for (const name of ['text','negative_text']) {
|
||||
assert.equal('widget' in byName[name], false);
|
||||
assert.equal(byName[name]._widget, undefined);
|
||||
assert.equal(byName[name].type, 'STRING');
|
||||
}
|
||||
assert.deepEqual(byName.category.widget, {name:'category'});
|
||||
assert.deepEqual(exposeTextSockets(node, ['text','negative_text']), [], 'idempotent');
|
||||
assert.deepEqual(exposeTextSockets({}, ['text']), [], 'a node without inputs is left alone');
|
||||
});
|
||||
|
||||
test('exposing a socket keeps an existing link and the widget value', async () => {
|
||||
const {exposeTextSockets,isLinked} = await source;
|
||||
const node = pairNode();
|
||||
node.inputs[3].link = 12;
|
||||
exposeTextSockets(node, ['text']);
|
||||
assert.equal(node.inputs[3].link, 12);
|
||||
assert.equal(isLinked(node,'text'), true);
|
||||
assert.equal(isLinked(node,'negative_text'), false);
|
||||
assert.equal(isLinked({},'text'), false);
|
||||
});
|
||||
|
||||
test('a STRING link dropped on the editor prefers the first free text input over prepend', async () => {
|
||||
const {preferredTextSlot} = await source;
|
||||
const node = pairNode();
|
||||
assert.deepEqual(preferredTextSlot(node, 'STRING', ['text','negative_text']), {index:3,slot:node.inputs[3]});
|
||||
node.inputs[3].link = 4;
|
||||
assert.deepEqual(preferredTextSlot(node, 'STRING', ['text','negative_text']), {index:4,slot:node.inputs[4]});
|
||||
node.inputs[4].link = 5;
|
||||
assert.equal(preferredTextSlot(node, 'STRING', ['text','negative_text']), undefined, 'both linked: the frontend chooses');
|
||||
});
|
||||
|
||||
test('only exact STRING links are steered; wildcards keep the frontend choice (#343 review M2)', async () => {
|
||||
const {preferredTextSlot} = await source;
|
||||
const node = pairNode();
|
||||
for (const type of ['STRING','STRING,INT',' STRING ']) assert.equal(preferredTextSlot(node, type, ['text'])?.index, 3, type);
|
||||
for (const type of ['*','','CLIP','INT','IMAGE',undefined,null,0]) assert.equal(preferredTextSlot(node, type, ['text']), undefined, String(type));
|
||||
assert.equal(preferredTextSlot({}, 'STRING', ['text']), undefined);
|
||||
});
|
||||
|
||||
test('a link being moved off a text input leaves it free, as the frontend counts it (#343 review I2)', async () => {
|
||||
const {preferredTextSlot} = await source;
|
||||
const links = new Map([[7,{id:7,_dragging:true}],[8,{id:8}]]);
|
||||
const node = Object.assign(pairNode(), {graph:{getLink:id => links.get(id)}});
|
||||
node.inputs[3].link = 7;
|
||||
assert.equal(preferredTextSlot(node, 'STRING', ['text','negative_text']).slot.name, 'text', 'the dragged link does not block text');
|
||||
links.get(7)._dragging = false;
|
||||
assert.equal(preferredTextSlot(node, 'STRING', ['text','negative_text']).slot.name, 'negative_text');
|
||||
node.inputs[4].link = 8;
|
||||
assert.equal(preferredTextSlot(node, 'STRING', ['text','negative_text']), undefined);
|
||||
node.inputs[4].link = 99;
|
||||
assert.equal(preferredTextSlot(node, 'STRING', ['text','negative_text']).slot.name, 'negative_text', 'a link the graph no longer has is free');
|
||||
const legacy = Object.assign(pairNode(), {graph:{links:{5:{id:5,_dragging:true}}}});
|
||||
legacy.inputs[3].link = 5;
|
||||
assert.equal(preferredTextSlot(legacy, 'STRING', ['text']).slot.name, 'text', 'object-keyed link tables too');
|
||||
});
|
||||
|
||||
function primitiveGraph() {
|
||||
const nodes = new Map();
|
||||
const links = new Map();
|
||||
const graph = {getNodeById:id => nodes.get(id), getLink:id => links.get(id)};
|
||||
const add = (id, node) => { nodes.set(id, {id, graph, ...node}); return nodes.get(id); };
|
||||
return {graph, nodes, links, add};
|
||||
}
|
||||
|
||||
test('a legacy frontend Primitive feeding a detached text input supplies its value (#343 review I1)', async () => {
|
||||
const {primitiveTextValue} = await source;
|
||||
const {nodes, links, add} = primitiveGraph();
|
||||
const target = add(1, {inputs:[{name:'text',link:10},{name:'negative_text',link:11},{name:'prepend_text',link:12},{name:'category',link:null}]});
|
||||
add(2, {type:'PrimitiveNode', isVirtualNode:true, widgets:[{value:'From the primitive'}]});
|
||||
add(3, {type:'Reroute', isVirtualNode:true, inputs:[{link:13}]});
|
||||
add(4, {type:'PrimitiveNode', isVirtualNode:true, widgets:[{value:42}]});
|
||||
add(5, {type:'PrimitiveStringMultiline', widgets:[{value:'a real node'}]});
|
||||
links.set(10, {origin_id:2}); links.set(11, {origin_id:3}); links.set(13, {origin_id:4}); links.set(12, {origin_id:5});
|
||||
assert.equal(await primitiveTextValue(target, 'text'), 'From the primitive');
|
||||
assert.equal(await primitiveTextValue(target, 'negative_text'), '42', 'through a virtual reroute');
|
||||
assert.equal(await primitiveTextValue(target, 'prepend_text'), undefined, 'a real node is serialized as a link');
|
||||
assert.equal(await primitiveTextValue(target, 'category'), undefined, 'not linked');
|
||||
assert.equal(await primitiveTextValue(target, 'missing'), undefined);
|
||||
nodes.get(2).widgets = [];
|
||||
assert.equal(await primitiveTextValue(target, 'text'), undefined, 'a primitive without its widget yet');
|
||||
links.set(13, {origin_id:3});
|
||||
assert.equal(await primitiveTextValue(target, 'negative_text'), undefined, 'a reroute cycle ends');
|
||||
assert.equal(await primitiveTextValue({inputs:[{name:'text',link:1}]}, 'text'), undefined, 'no graph');
|
||||
});
|
||||
|
||||
test('an input still bound to its widget is left to the host, which applies the Primitive itself (#343 re-review R1)', async () => {
|
||||
const {primitiveTextValue} = await source;
|
||||
const {links, add} = primitiveGraph();
|
||||
const target = add(1, {inputs:[{name:'text',link:10,widget:{name:'text'}}], widgets:[{name:'text',value:'typed'}]});
|
||||
add(2, {type:'PrimitiveNode', isVirtualNode:true, widgets:[{value:'{red|blue}'}]});
|
||||
links.set(10, {origin_id:2});
|
||||
assert.equal(await primitiveTextValue(target, 'text'), undefined);
|
||||
});
|
||||
|
||||
test('a detached input runs the Primitive value the way the host runs a widget value: replace, then the widget serializer (#343 re-review R1)', async () => {
|
||||
const {primitiveTextValue} = await source;
|
||||
const {graph, links, add} = primitiveGraph();
|
||||
const calls = [];
|
||||
const widget = {name:'text', value:'typed under the link', serializeValue(...args) { calls.push([this.value, args.length]); return this.value.replace('{alpha|beta}', 'alpha'); }};
|
||||
const target = add(1, {inputs:[{name:'text',link:10},{name:'negative_text',link:11}], widgets:[widget]});
|
||||
const primitive = add(2, {type:'PrimitiveNode', isVirtualNode:true, properties:{'Run widget replace on values':false}, widgets:[{value:'a %year% {alpha|beta} car'}]});
|
||||
add(3, {type:'PrimitiveNode', isVirtualNode:true, properties:{'Run widget replace on values':true}, widgets:[{value:'no serializer %year%'}]});
|
||||
links.set(10, {origin_id:2}); links.set(11, {origin_id:3});
|
||||
const replaced = [];
|
||||
const replaceText = (from, value) => { replaced.push(from); return value.replace('%year%', '2026'); };
|
||||
assert.equal(await primitiveTextValue(target, 'text', {replaceText}), 'a %year% alpha car', 'replace stays off unless the Primitive asks for it');
|
||||
assert.deepEqual(calls, [['a %year% {alpha|beta} car', 0]], 'serialized once, without a node, so widgets_values is not touched');
|
||||
assert.equal(widget.value, 'typed under the link', 'the typed text is restored');
|
||||
primitive.properties['Run widget replace on values'] = true;
|
||||
assert.equal(await primitiveTextValue(target, 'text', {replaceText}), 'a 2026 alpha car');
|
||||
assert.deepEqual(replaced, [graph]);
|
||||
assert.equal(await primitiveTextValue(target, 'negative_text', {replaceText}), 'no serializer 2026', 'no widget serializer: the replaced value');
|
||||
assert.equal(await primitiveTextValue(target, 'negative_text'), 'no serializer %year%', 'no replacer: the value as is');
|
||||
const failing = () => { throw new Error('not this host'); };
|
||||
assert.equal(await primitiveTextValue(target, 'negative_text', {replaceText:failing}), 'no serializer %year%', 'a replacer that fails leaves the value');
|
||||
widget.serializeValue = () => { throw new Error('broken serializer'); };
|
||||
await assert.rejects(primitiveTextValue(target, 'text'), /broken serializer/);
|
||||
assert.equal(widget.value, 'typed under the link', 'restored even when the serializer throws');
|
||||
});
|
||||
|
||||
test('installing the preference wraps findInputByType and restores nothing it did not own', async () => {
|
||||
const {preferTextInputs} = await source;
|
||||
const calls = [];
|
||||
const prototype = {findInputByType(type) { calls.push(type); return {index:1,slot:this.inputs[1]}; }};
|
||||
preferTextInputs(prototype, ['text']);
|
||||
const node = Object.assign(Object.create(prototype), pairNode());
|
||||
assert.equal(node.findInputByType('STRING').index, 3);
|
||||
assert.deepEqual(calls, []);
|
||||
assert.equal(node.findInputByType('CLIP').index, 1);
|
||||
node.inputs[3].link = 9;
|
||||
assert.equal(node.findInputByType('STRING').index, 1, 'the main prompt is taken: fall back to the frontend');
|
||||
assert.deepEqual(calls, ['CLIP','STRING']);
|
||||
const bare = {};
|
||||
preferTextInputs(bare, ['text']);
|
||||
assert.equal(bare.findInputByType, undefined, 'an older frontend without findInputByType is left unchanged');
|
||||
});
|
||||
|
||||
test('a drop point over a visible prompt box or its note picks that box input (#343 review M1)', async () => {
|
||||
const {textInputAtPoint} = await source;
|
||||
const box = (left, top, width, height) => ({getBoundingClientRect:() => ({left, top, width, height, right:left + width, bottom:top + height})});
|
||||
const node = pairNode();
|
||||
const targets = [{name:'text',element:box(0,0,100,100)},{name:'negative_text',element:box(120,0,100,100)},{name:'text',element:box(0,0,0,0)},{name:'missing',element:box(300,0,50,50)}];
|
||||
assert.equal(textInputAtPoint(node, [150,50], targets).name, 'negative_text');
|
||||
assert.equal(textInputAtPoint(node, [10,10], targets).name, 'text');
|
||||
assert.equal(textInputAtPoint(node, [110,50], targets), undefined, 'between the boxes');
|
||||
assert.equal(textInputAtPoint(node, [320,10], targets), undefined, 'a target whose input the node lacks');
|
||||
assert.equal(textInputAtPoint(node, [0,0], [{name:'text',element:box(0,0,0,0)}]), undefined, 'a hidden target');
|
||||
});
|
||||
|
||||
test('a linked Classic text box gives its row back and gets its size back on disconnect (#343 review M5)', async () => {
|
||||
const {collapseLinkedWidget} = await source;
|
||||
const own = () => 120;
|
||||
const widget = {options:{getMinHeight:own}};
|
||||
collapseLinkedWidget(widget, true);
|
||||
assert.equal(widget.options.getMinHeight(), 0);
|
||||
assert.equal(widget.options.getMaxHeight(), 0);
|
||||
collapseLinkedWidget(widget, true);
|
||||
collapseLinkedWidget(widget, false);
|
||||
assert.equal(widget.options.getMinHeight, own);
|
||||
assert.equal('getMaxHeight' in widget.options, false, 'an option it did not have is removed again');
|
||||
collapseLinkedWidget(widget, false);
|
||||
assert.equal(widget.options.getMinHeight, own);
|
||||
collapseLinkedWidget(undefined, true);
|
||||
collapseLinkedWidget({}, true);
|
||||
});
|
||||
@@ -126,7 +126,7 @@ test('held legacy jobs read "(admin page)" with no outcomes', async () => {
|
||||
|
||||
test('unknown and generic kinds show no tiles of another kind', async () => {
|
||||
const { kindOf, viewOf } = await source;
|
||||
for (const kind of ['maintenance', '', 'constructor', '__proto__', 'health']) {
|
||||
for (const kind of ['backup', '', 'constructor', '__proto__', 'health']) {
|
||||
const view = kindOf(kind);
|
||||
assert.deepEqual([view.tiles, view.outcomes, [...view.openable]], [[], [], []], kind);
|
||||
assert.deepEqual([view.log, view.cancelLabel], ['Job log', 'Cancel job']);
|
||||
@@ -146,3 +146,82 @@ test('unknown and generic kinds show no tiles of another kind', async () => {
|
||||
assert.deepEqual([...partial.openable], ['added']);
|
||||
assert.equal(partial.log, 'Scan log');
|
||||
});
|
||||
|
||||
// System S7 (carry-over from S6): the maintenance jobs read as themselves in Activity, with their own counters, a
|
||||
// report like every job, and Cancel only when the server can stop them.
|
||||
test('a maintenance apply shows its title, changed / protected / errors and no Cancel', async () => {
|
||||
const { kindOf, viewOf, canCancel } = await source;
|
||||
const { finishToast, startToast, pillParts, outcomeLine } = await store;
|
||||
const apply = job({ kind: 'maintenance', title: 'Duplicate prompt text', progress: 0, processed: 0, total: 0,
|
||||
counters: { changed: 3, protected: 1, errors: 0 }, scope: { operation: 'duplicate_prompts' }, cancellable: false });
|
||||
const view = kindOf('maintenance');
|
||||
assert.equal(viewOf(apply), view);
|
||||
assert.deepEqual(view.tiles, [['changed', 'Changed', 'added'], ['protected', 'Protected', ''], ['errors', 'Errors', 'bad']]);
|
||||
assert.deepEqual(view.outcomes, [['changed', 'Changed'], ['errors', 'Errors'], ['protected', 'Protected']]);
|
||||
assert.deepEqual([...view.openable], []);
|
||||
assert.deepEqual([view.log, view.cancelLabel], ['Maintenance log', 'Cancel job']);
|
||||
assert.equal(canCancel(apply), false);
|
||||
// Indeterminate while it runs: the title alone, never a made-up percentage.
|
||||
assert.equal(pillParts(apply).text, 'Duplicate prompt text');
|
||||
assert.equal(startToast(apply), 'Duplicate prompt text started.');
|
||||
assert.equal(outcomeLine(apply), '3 changed · 0 errors');
|
||||
assert.equal(finishToast({ ...apply, state: 'finished' }), 'Duplicate prompt text finished · 3 changed');
|
||||
// A failed apply says why, so plan-changed asks for a new review.
|
||||
const changed = 'The library changed since the review; nothing was changed. Review the plan again.';
|
||||
assert.equal(finishToast({ ...apply, state: 'failed', status_line: changed, counters: { changed: 0, protected: 1, errors: 1 } }),
|
||||
`Duplicate prompt text failed · ${changed}`);
|
||||
assert.equal(finishToast({ ...apply, state: 'failed', status_line: '' }), 'Duplicate prompt text failed · 0 errors');
|
||||
// Store each workflow once is a cancellable maintenance job: Cancel shows while it runs.
|
||||
const migration = { ...apply, title: 'Store each workflow once', cancellable: true };
|
||||
assert.equal(canCancel(migration), true);
|
||||
assert.equal(finishToast({ ...migration, state: 'cancelled' }), 'Store each workflow once cancelled · 3 changed');
|
||||
});
|
||||
|
||||
test('a duplicate search counts groups, opens their first copy and cancels as a search', async () => {
|
||||
const { kindOf, viewOf, canCancel } = await source;
|
||||
const { finishToast, startToast, pillParts, outcomeLine } = await store;
|
||||
const find = job({ kind: 'duplicates', title: 'Find duplicate image files', processed: 5, total: 10, progress: 50,
|
||||
counters: { groups_found: 0 }, scope: { output_folders: 2 } });
|
||||
const view = kindOf('duplicates');
|
||||
assert.equal(viewOf(find), view);
|
||||
assert.deepEqual(view.tiles, [['groups_found', 'Groups found', '']]);
|
||||
assert.deepEqual(view.outcomes, [['groups_found', 'Groups found']]);
|
||||
assert.deepEqual([...view.openable], ['groups_found']);
|
||||
assert.deepEqual([view.log, view.cancelLabel], ['Search log', 'Cancel search']);
|
||||
assert.equal(canCancel(find), true);
|
||||
assert.equal(pillParts(find).text, 'Finding duplicates · 50% · 5 / 10');
|
||||
assert.equal(startToast(find), 'Duplicate search started.');
|
||||
assert.equal(view.scope(find), '2 output folders');
|
||||
const found = { ...find, state: 'finished', counters: { groups_found: 12 } };
|
||||
assert.equal(outcomeLine(found), '12 groups found');
|
||||
assert.equal(finishToast(found), 'Duplicate search finished · 12 groups found');
|
||||
assert.equal(finishToast({ ...find, state: 'cancelled' }), 'Duplicate search cancelled · 0 groups found');
|
||||
assert.equal(finishToast({ ...found, counters: { groups_found: 1 } }), 'Duplicate search finished · 1 group found');
|
||||
});
|
||||
|
||||
test('a duplicate cleanup counts removed, kept and errors and cannot be cancelled', async () => {
|
||||
const { kindOf, viewOf, canCancel } = await source;
|
||||
const { finishToast, pillParts, outcomeLine } = await store;
|
||||
const cleanup = job({ kind: 'duplicate_cleanup', title: 'Remove duplicate image files', processed: 1, total: 4,
|
||||
progress: 25, counters: { removed: 1, kept: 0, errors: 0 }, cancellable: false });
|
||||
const view = kindOf('duplicate_cleanup');
|
||||
assert.equal(viewOf(cleanup), view);
|
||||
assert.deepEqual(view.tiles, [['removed', 'Removed', 'added'], ['kept', 'Kept', ''], ['errors', 'Errors', 'bad']]);
|
||||
assert.deepEqual(view.outcomes, [['removed', 'Removed'], ['errors', 'Errors'], ['kept', 'Kept']]);
|
||||
assert.deepEqual([...view.openable], ['kept']);
|
||||
assert.deepEqual([view.log, view.cancelLabel], ['Cleanup log', 'Cancel job']);
|
||||
assert.equal(canCancel(cleanup), false);
|
||||
assert.equal(pillParts(cleanup).text, 'Removing duplicate files · 25% · 1 / 4');
|
||||
assert.equal(outcomeLine(cleanup), '1 removed · 0 errors');
|
||||
assert.equal(finishToast({ ...cleanup, state: 'finished', counters: { removed: 3, kept: 1, errors: 1 } }),
|
||||
'Duplicate cleanup finished · 3 removed · 1 error');
|
||||
});
|
||||
|
||||
test('a failed duplicate cleanup says the server reason (S7 review M3)', async () => {
|
||||
const { finishToast } = await store;
|
||||
const reason = 'The sources changed since the search. Search for duplicates again.';
|
||||
const failed = job({ kind: 'duplicate_cleanup', title: 'Remove duplicate image files', state: 'failed', status_line: reason,
|
||||
counters: { removed: 0, kept: 0, errors: 1 }, cancellable: false });
|
||||
assert.equal(finishToast(failed), `Duplicate cleanup failed · ${reason}`);
|
||||
assert.equal(finishToast({ ...failed, status_line: '' }), 'Duplicate cleanup failed · 0 removed · 1 error');
|
||||
});
|
||||
|
||||
@@ -344,7 +344,7 @@ test('unknown kinds get the generic view without the scan tiles', async () => {
|
||||
['no_metadata', 'No usable metadata'], ['known', 'Already known']]);
|
||||
assert.deepEqual([...scan.openable].sort(), ['added', 'known']);
|
||||
assert.deepEqual([scan.log, scan.cancel], ['Scan log', 'Cancel scan']);
|
||||
for (const kind of ['maintenance', '', 'constructor', '__proto__']) {
|
||||
for (const kind of ['backup', '', 'constructor', '__proto__']) {
|
||||
const view = jobView(kind);
|
||||
assert.deepEqual([view.tiles, view.outcomes, [...view.openable]], [[], [], []]);
|
||||
assert.deepEqual([view.log, view.cancel], ['Job log', 'Cancel job']);
|
||||
@@ -437,3 +437,16 @@ test('numbers group the way the server writes them, whatever the browser locale'
|
||||
Number.prototype.toLocaleString = original;
|
||||
}
|
||||
});
|
||||
|
||||
test('a busy start names the job the server named, not the first running one (S7 review M9)', async () => {
|
||||
const { createActivityStore } = await source;
|
||||
const { world, ...deps } = harness();
|
||||
const store = createActivityStore(deps);
|
||||
const holder = 'b'.repeat(32);
|
||||
// Two read jobs run side by side; the server says the second one holds the slot.
|
||||
world.list = [summary(), summary({ id: holder, kind: 'duplicates', title: 'Find duplicate image files' })];
|
||||
world.postError = Object.assign(apiError(409, 'busy', 'busy'),
|
||||
{ running: { id: holder, title: 'Find duplicate image files' } });
|
||||
const busy = await store.start('duplicates');
|
||||
assert.deepEqual([busy.busy.id, busy.busy.title], [holder, 'Find duplicate image files']);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,286 @@
|
||||
// System › Backup & export (#249 System S9 Task 24): the concept's copy (app.js 691–693), the status line from
|
||||
// GET /backup/status (D103), the restore review's rows from the S8 preview (D105), its upload and refusals, the export
|
||||
// list, the restore job's Activity copy and the reload after a finished restore.
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const { ApiError } = require('../../web/js/api-client.js');
|
||||
|
||||
const backups = import('../../web/js/workspace/system-backups.mjs');
|
||||
const kinds = import('../../web/js/workspace/activity-kinds.mjs');
|
||||
const scan = import('../../web/js/workspace/activity-scan.mjs');
|
||||
const dialogs = import('../../web/js/workspace/workspace-dialogs.mjs');
|
||||
|
||||
const CONCEPT = fs.readFileSync(path.join(__dirname, '../../docs/concepts/workspace/app.js'), 'utf8');
|
||||
// The module's code without its comments (which name the concept's prototype labels they replace).
|
||||
const SOURCE = fs.readFileSync(path.join(__dirname, '../../web/js/workspace/system-backups.mjs'), 'utf8')
|
||||
.replace(/\/\*[\s\S]*?\*\//g, '').replace(/^\s*\/\/.*$/gm, '');
|
||||
|
||||
const conceptSection = () => {
|
||||
const start = CONCEPT.indexOf('case "backups":');
|
||||
return CONCEPT.slice(start, CONCEPT.indexOf('break;', start));
|
||||
};
|
||||
|
||||
test('the section keeps the concept copy: surfaces, descriptions and the two kept labels', () => {
|
||||
const body = conceptSection();
|
||||
for (const text of ['Database backup', 'A complete recovery snapshot, including prompt pairs, metadata and image links. Media files are separate.',
|
||||
'Review restore…', 'Portable export', 'Export reusable recipes and tags as versioned JSON. This is distinct from a complete database backup.',
|
||||
'Import / inspect file', 'Recovery policy', 'Proposed', 'Create a safety backup before maintenance', 'Scheduled database backups',
|
||||
'Manual only', 'Daily', 'Weekly']) {
|
||||
assert.ok(body.includes(text), `concept lacks ${text}`);
|
||||
assert.ok(SOURCE.includes(text), `production lacks ${text}`);
|
||||
}
|
||||
// D103/D104: prototype copy is never shipped.
|
||||
// D104 (M2): the toggle's concept line ("Preview a verified backup…") misstates the policy and is replaced too.
|
||||
for (const sample of ['Simulate backup', 'Sample backup', 'Export sample JSON', 'Production feature proposal', 'A restore would be',
|
||||
'Preview a verified backup before applying operations that change records.']) {
|
||||
assert.ok(body.includes(sample));
|
||||
assert.ok(!SOURCE.includes(sample), `production ships ${sample}`);
|
||||
}
|
||||
});
|
||||
|
||||
test('status line shows only server-reported backups (D103)', async () => {
|
||||
const { backupStatusText, NO_BACKUP, STATUS_LOADING, STATUS_FAILED, whenText } = await backups;
|
||||
assert.equal(backupStatusText(undefined), STATUS_LOADING);
|
||||
assert.equal(backupStatusText(null), STATUS_FAILED);
|
||||
assert.equal(backupStatusText({ success: true, safety_backups: [], last_download: null }), NO_BACKUP);
|
||||
assert.equal(NO_BACKUP, 'No backup recorded.');
|
||||
// A time the server did not send in a readable form names nothing.
|
||||
assert.equal(backupStatusText({ safety_backups: [{ name: 'x', modified_at: 'soon' }], last_download: { at: 7, verified: true } }),
|
||||
NO_BACKUP);
|
||||
const at = '2026-10-25T09:30:00+00:00';
|
||||
assert.equal(backupStatusText({ safety_backups: [], last_download: { at, bytes: 10, verified: true } }),
|
||||
`Downloaded backup · ${whenText(at)} · verified.`);
|
||||
assert.equal(backupStatusText({ safety_backups: [], last_download: { at, bytes: 10, verified: false } }),
|
||||
`Downloaded backup · ${whenText(at)} · not verified.`);
|
||||
const safety = [{ name: 'prompts.db.backup_2', bytes: 1, modified_at: at }, { name: 'prompts.db.backup_1', bytes: 1, modified_at: at }];
|
||||
assert.equal(backupStatusText({ safety_backups: safety, last_download: null }), `Safety backup · ${whenText(at)} · 2 on disk.`);
|
||||
assert.equal(backupStatusText({ safety_backups: safety.slice(0, 1), last_download: { at, verified: true } }),
|
||||
`Downloaded backup · ${whenText(at)} · verified. Safety backup · ${whenText(at)} · 1 on disk.`);
|
||||
assert.equal(whenText(undefined), null);
|
||||
});
|
||||
|
||||
const PREVIEW = Object.freeze({
|
||||
success: true, token: 'r'.repeat(43), expires_at: '2026-10-25T12:15:00+00:00',
|
||||
summary: { prompts: 1234, positives: 900, pairs: 3, archived_pairs: 0, image_links: 56, runs: 1, saved_views: 2 },
|
||||
schema: { migrated: true, library_state: true }, integrity: 'ok', documents: { repaired: 0, cleared: 0, images: 0 },
|
||||
live: { prompts: 1, positives: 1, pairs: 0, archived_pairs: 0, image_links: 0, runs: 0, saved_views: 0 }, size: 4096,
|
||||
safety_backup: { destination_label: 'prompts.db.backup_<date and time>' },
|
||||
});
|
||||
|
||||
test('Review restore shows the real summary in the concept rows (D105)', async () => {
|
||||
const { restoreGrid, restoreNote, countsText, integrityText, snapshotText, whenText } = await backups;
|
||||
const file = { name: 'prompts_backup_20261025_093000.db', size: 3 * 2 ** 20, lastModified: Date.parse('2026-10-25T09:30:00Z') };
|
||||
assert.deepEqual(restoreGrid(PREVIEW, file), [
|
||||
['Snapshot', `prompts_backup_20261025_093000.db · 3.0 MB · ${whenText('2026-10-25T09:30:00.000Z')}`],
|
||||
['Integrity', 'Verified'],
|
||||
['Migration', 'Upgraded to the current schema'],
|
||||
['Safety backup', 'Created before replacing (prompts.db.backup_<date and time>)'],
|
||||
['In the backup', '1,234 prompts · 56 image links · 3 saved pairs · 1 run'],
|
||||
['In the library now', '1 prompt · 0 image links · 0 saved pairs · 0 runs'],
|
||||
]);
|
||||
assert.equal(restoreGrid({ ...PREVIEW, schema: { migrated: false } }, file)[2][1], 'Current schema');
|
||||
assert.equal(restoreGrid({ ...PREVIEW, safety_backup: null }, file)[3][1], 'Created before replacing');
|
||||
assert.equal(countsText({}), 'Not reported');
|
||||
assert.equal(snapshotText({ name: 'a.db' }), 'a.db');
|
||||
assert.equal(integrityText({ integrity: 'ok', documents: { repaired: 2, cleared: 1, images: 1 } }),
|
||||
'Verified · repaired 2 damaged workflow documents · could not repair 1 workflow document: 1 image shows no workflow');
|
||||
assert.equal(integrityText({ integrity: 'ok', documents: { repaired: 1, cleared: 2, images: 3 } }),
|
||||
'Verified · repaired 1 damaged workflow document · could not repair 2 workflow documents: 3 images show no workflow');
|
||||
assert.equal(integrityText({}), 'Not verified');
|
||||
const now = Date.parse('2026-10-25T12:00:00+00:00');
|
||||
assert.equal(restoreNote(PREVIEW, { now }), 'Restore replaces the whole library with this backup and runs alone in Activity. ' +
|
||||
'Nothing changes until you restore. This review holds for 15 minutes.');
|
||||
assert.equal(restoreNote({ expires_at: 'x' }, { now }), 'Restore replaces the whole library with this backup and runs alone ' +
|
||||
'in Activity. Nothing changes until you restore.');
|
||||
});
|
||||
|
||||
function response(status, body, headers = {}) {
|
||||
return { ok: status >= 200 && status < 300, status, json: async () => {
|
||||
if (body === undefined) throw new SyntaxError('not JSON');
|
||||
return body;
|
||||
}, headers: { get: name => headers[name] ?? null }, blob: async () => new Blob(['x']) };
|
||||
}
|
||||
|
||||
test('invalid file shows the validator message; a refusal names its job; a lost answer stages nothing', async () => {
|
||||
const { uploadRestore, reviewRefusal, REVIEW_FAILED, PREVIEW_URL } = await backups;
|
||||
const file = new File(['not a database'], 'bad.db');
|
||||
let seen;
|
||||
const refuse = async (url, init) => { seen = [url, init.method, init.cache]; return response(400,
|
||||
{ success: false, code: 'invalid-backup', error: 'Not a SQLite database file' }); };
|
||||
const invalid = await uploadRestore(file, { fetchUrl: refuse, ApiError }).catch(error => error);
|
||||
assert.deepEqual(seen, [PREVIEW_URL, 'POST', 'no-store']);
|
||||
assert.equal(reviewRefusal(invalid), "This file can't be restored: Not a SQLite database file.");
|
||||
const busy = await uploadRestore(file, { ApiError, fetchUrl: async () => response(409,
|
||||
{ success: false, code: 'busy', error: 'Busy', running: { id: 'j1', title: 'Metadata scan' } }) }).catch(error => error);
|
||||
assert.deepEqual(reviewRefusal(busy), ['Another job is running: ', { text: 'Metadata scan', href: '#/activity/j1' }, '.']);
|
||||
const full = await uploadRestore(file, { ApiError, fetchUrl: async () => response(429,
|
||||
{ success: false, code: 'too-large', error: 'Two restores are already being reviewed. Discard one first.' }) })
|
||||
.catch(error => error);
|
||||
// I1: the server's "Discard one first" names nothing this page can do; the toast says what the user can.
|
||||
assert.equal(reviewRefusal(full), (await backups).REVIEWS_FULL);
|
||||
const proxy = await uploadRestore(file, { ApiError, fetchUrl: async () => response(502, undefined) }).catch(error => error);
|
||||
assert.equal(reviewRefusal(proxy), REVIEW_FAILED);
|
||||
const odd = await uploadRestore(file, { ApiError, fetchUrl: async () => response(200, { success: true }) }).catch(error => error);
|
||||
assert.equal(reviewRefusal(odd), REVIEW_FAILED);
|
||||
assert.equal(reviewRefusal(new Error('offline')), REVIEW_FAILED);
|
||||
assert.deepEqual(await uploadRestore(file, { ApiError, fetchUrl: async () => response(200, PREVIEW) }), PREVIEW);
|
||||
});
|
||||
|
||||
test('Cancel discards the stage through the api client and never throws', async () => {
|
||||
const { discardRestore, PREVIEW_URL } = await backups;
|
||||
const calls = [];
|
||||
await discardRestore('a/b', { fetchUrl: async (url, init) => { calls.push([url, init.method]); return response(200, {}); } });
|
||||
await discardRestore('gone', { fetchUrl: async () => { throw new TypeError('offline'); } });
|
||||
assert.deepEqual(calls, [[`${PREVIEW_URL}/a%2Fb`, 'DELETE']]);
|
||||
});
|
||||
|
||||
test('Export JSON downloads /library/export; secondary buttons download /export and /pairs/export (D103)', async () => {
|
||||
const { EXPORTS } = await backups;
|
||||
assert.deepEqual(EXPORTS.map(([label, url]) => [label, url]), [
|
||||
['Export JSON', '/prompt_manager/library/export'],
|
||||
['Export all prompts JSON', '/prompt_manager/export'],
|
||||
['Export saved pairs JSON', '/prompt_manager/pairs/export'],
|
||||
]);
|
||||
});
|
||||
|
||||
test('a failed export toasts and saves nothing', async () => {
|
||||
const { downloadExport, EXPORTS, EXPORT_FAILED } = await backups;
|
||||
const toasts = [];
|
||||
assert.equal(await downloadExport(EXPORTS[1], { fetchUrl: async () => response(500, {}), notify: text => toasts.push(text) }), false);
|
||||
const aborted = AbortSignal.abort();
|
||||
assert.equal(await downloadExport(EXPORTS[1], { fetchUrl: async () => { throw new Error('x'); }, notify: text => toasts.push(text),
|
||||
signal: aborted }), false);
|
||||
assert.deepEqual(toasts, [EXPORT_FAILED]);
|
||||
});
|
||||
|
||||
test('Restore starts the job: Activity names it and ends with the server line', async () => {
|
||||
const { finishToast, outcomeLine, pillParts, canCancel, startToast } = await kinds;
|
||||
const job = { id: 'j', kind: 'restore', title: 'Database restore', state: 'running', cancellable: false, progress: 0, total: 0,
|
||||
counters: { errors: 0 }, status_line: 'Restoring the database' };
|
||||
assert.equal(startToast(job), 'Database restore started.');
|
||||
assert.equal(pillParts(job).label, 'Restoring the database');
|
||||
assert.equal(canCancel(job), false);
|
||||
assert.equal(outcomeLine(job), '');
|
||||
const done = { ...job, state: 'finished', status_line: 'Restored 3 prompts, 1 image link, 0 saved pairs and 0 runs' };
|
||||
assert.equal(finishToast(done), 'Database restore finished · Restored 3 prompts, 1 image link, 0 saved pairs and 0 runs');
|
||||
assert.equal(outcomeLine(done), done.status_line);
|
||||
const failed = { ...job, state: 'failed', status_line: 'The library changed since the review; nothing was changed. Review the restore again.' };
|
||||
assert.equal(finishToast(failed), `Database restore failed · ${failed.status_line}`);
|
||||
assert.equal(finishToast({ ...job, state: 'failed', status_line: '' }), 'Database restore failed.');
|
||||
});
|
||||
|
||||
test('views reload once after a restore this page saw finish; a failed restore changes nothing', async () => {
|
||||
const { onRestoreFinished } = await backups;
|
||||
let list = [];
|
||||
const listeners = new Set();
|
||||
const activity = { jobs: () => list, subscribe(listener) { listeners.add(listener); return () => listeners.delete(listener); } };
|
||||
const emit = next => { list = next; for (const listener of listeners) listener(); };
|
||||
let reloads = 0;
|
||||
const stop = onRestoreFinished(activity, () => { reloads += 1; });
|
||||
emit([{ id: 'old', kind: 'restore', state: 'finished' }]);
|
||||
assert.equal(reloads, 0, 'a restore that ended before this page saw it run');
|
||||
emit([{ id: 'a', kind: 'restore', state: 'running' }, { id: 's', kind: 'scan', state: 'running' }]);
|
||||
emit([{ id: 'a', kind: 'restore', state: 'finished' }, { id: 's', kind: 'scan', state: 'finished' }]);
|
||||
emit([{ id: 'a', kind: 'restore', state: 'finished' }]);
|
||||
assert.equal(reloads, 1);
|
||||
emit([{ id: 'b', kind: 'restore', state: 'running' }]);
|
||||
emit([{ id: 'b', kind: 'restore', state: 'failed' }]);
|
||||
assert.equal(reloads, 1);
|
||||
stop();
|
||||
assert.equal(listeners.size, 0);
|
||||
});
|
||||
|
||||
test('a downloaded backup the server could not verify says so', async () => {
|
||||
const { downloadDatabaseBackup, BACKUP_DONE, backupUnverified } = await scan;
|
||||
const toasts = [];
|
||||
const body = globalThis.document;
|
||||
globalThis.document = { createElement: () => ({ click() {}, remove() {} }), body: { append() {} } };
|
||||
try {
|
||||
await downloadDatabaseBackup({ notify: text => toasts.push(text),
|
||||
fetchUrl: async () => response(200, {}, { 'X-PromptManager-Verify': 'ok' }) });
|
||||
await downloadDatabaseBackup({ notify: text => toasts.push(text),
|
||||
fetchUrl: async () => response(200, {}, { 'X-PromptManager-Verify': 'Database does not contain a pair table' }) });
|
||||
} finally {
|
||||
globalThis.document = body;
|
||||
}
|
||||
assert.deepEqual(toasts, [BACKUP_DONE, backupUnverified('Database does not contain a pair table')]);
|
||||
assert.equal(toasts[1], 'Database backup downloaded, but it did not pass verification: Database does not contain a pair table');
|
||||
});
|
||||
|
||||
test('the Import dialog shares the rescan copy with Images (D15)', async () => {
|
||||
const { rescanFinished, RESCAN_STARTED, RESCAN_FAILED } = await dialogs;
|
||||
assert.equal(RESCAN_STARTED, 'Rescanning output folders…');
|
||||
assert.equal(RESCAN_FAILED, 'Rescan could not finish. Try again.');
|
||||
assert.equal(rescanFinished({ total: 1, roots: 2 }), 'Rescan finished. 1 file in 2 output folders.');
|
||||
assert.equal(rescanFinished({ total: 1200, roots: 1 }), 'Rescan finished. 1,200 files in 1 output folder.');
|
||||
assert.equal(rescanFinished(null), 'Rescan finished.');
|
||||
});
|
||||
|
||||
test('applyPlan follows a started job even after its section left, and reports the outcome (S9 N5, I1)', async () => {
|
||||
const { applyPlan } = await import('../../web/js/workspace/maintenance-review.mjs');
|
||||
const job = { id: 'a'.repeat(32), kind: 'restore', title: 'Database restore', state: 'running' };
|
||||
const tracked = [];
|
||||
const store = { track: raw => { tracked.push(raw.id); return raw; }, load() {} };
|
||||
const left = new AbortController();
|
||||
left.abort();
|
||||
const toasts = [], routes = [];
|
||||
const outcome = await applyPlan({ api: { post: async () => ({ success: true, job }) }, store, scope: { signal: left.signal },
|
||||
token: 't', notify: text => toasts.push(text), navigate: hash => routes.push(hash) });
|
||||
// The page's store follows it (a finished restore must reload the views); the left section neither toasts nor opens it.
|
||||
assert.deepEqual(tracked, [job.id]);
|
||||
assert.deepEqual([toasts, routes], [[], []]);
|
||||
assert.deepEqual(outcome, { job });
|
||||
const busy = new ApiError(409, 'Busy', 'busy', null, { success: false, code: 'busy', running: { id: 'j1', title: 'Scan' } });
|
||||
const live = { signal: new AbortController().signal };
|
||||
const refused = await applyPlan({ api: { post: async () => { throw busy; } }, store, scope: live, token: 't',
|
||||
notify: text => toasts.push(text), navigate: hash => routes.push(hash) });
|
||||
assert.deepEqual(refused, { refused: busy });
|
||||
assert.deepEqual(toasts, [['Another job is running: ', { text: 'Scan', href: '#/activity/j1' }, '.']]);
|
||||
});
|
||||
|
||||
test('after an apply: follow a started job, reopen a busy refusal, release the token otherwise (I1)', async () => {
|
||||
const { afterApply } = await backups;
|
||||
const refusal = code => ({ refused: new ApiError(409, 'x', code, null, { success: false, code }) });
|
||||
assert.equal(afterApply({ job: { id: 'j' } }), 'follow');
|
||||
assert.equal(afterApply(refusal('busy')), 'reopen');
|
||||
for (const outcome of [refusal('plan-used'), refusal('plan-expired'), refusal('plan-changed'), refusal('storage-changed'),
|
||||
{ refused: new TypeError('offline') }, { lost: true }, undefined]) {
|
||||
assert.equal(afterApply(outcome), 'release');
|
||||
}
|
||||
});
|
||||
|
||||
test('a restore job that hands its review back reopens it once in the page that started it; any other end forgets it (I1)', async () => {
|
||||
const { rememberRestore, onRestoreHandedBack } = await backups;
|
||||
let list = [];
|
||||
const listeners = new Set();
|
||||
const activity = { jobs: () => list, subscribe(listener) { listeners.add(listener); return () => listeners.delete(listener); } };
|
||||
const emit = next => { list = next; for (const listener of listeners) listener(); };
|
||||
const reopened = [];
|
||||
const stop = onRestoreHandedBack(activity, review => reopened.push(review.token));
|
||||
rememberRestore('kept', { token: 'k' });
|
||||
rememberRestore('done', { token: 'd' });
|
||||
rememberRestore('gone', { token: 'g' });
|
||||
emit([{ id: 'kept', kind: 'restore', state: 'running', scope: {} }]);
|
||||
emit([{ id: 'kept', kind: 'restore', state: 'failed', scope: { failure: 'busy', review_kept: true } },
|
||||
{ id: 'done', kind: 'restore', state: 'finished', scope: {} },
|
||||
{ id: 'gone', kind: 'restore', state: 'failed', scope: { failure: 'busy' } }]);
|
||||
emit(list);
|
||||
assert.deepEqual(reopened, ['k']);
|
||||
stop();
|
||||
assert.equal(listeners.size, 0);
|
||||
});
|
||||
|
||||
test('small files show KB; a full review store says what to do (N3, I1)', async () => {
|
||||
const { snapshotText, reviewRefusal, EXPORTS } = await backups;
|
||||
assert.equal(snapshotText({ name: 'a.db', size: 40 * 1024 }), 'a.db · 40 KB');
|
||||
assert.equal(snapshotText({ name: 'a.db', size: 10 }), 'a.db · 1 KB');
|
||||
assert.equal(snapshotText({ name: 'a.db', size: 3 * 2 ** 20 }), 'a.db · 3.0 MB');
|
||||
const full = new ApiError(429, 'Two restores are already being reviewed. Discard one first.', 'too-large', null,
|
||||
{ success: false, code: 'too-large' });
|
||||
assert.equal(reviewRefusal(full), 'Two restores are already being reviewed in another window. Discard one there, ' +
|
||||
'or try again when it expires (15 minutes after it was reviewed).');
|
||||
// N1: /export is the legacy list of every prompt.
|
||||
assert.deepEqual(EXPORTS[1].slice(0, 2), ['Export all prompts JSON', '/prompt_manager/export']);
|
||||
});
|
||||
@@ -0,0 +1,197 @@
|
||||
// System › Integrations (#249 System S10 Task 27): the host values and saves behind the LoRA rows (D114), the D106
|
||||
// renderer and detection lines, and the D107 import review built from the server's plan.
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
|
||||
const integrations = import('../../web/js/workspace/system-integrations.mjs');
|
||||
const kinds = import('../../web/js/workspace/activity-kinds.mjs');
|
||||
|
||||
const CONCEPT = fs.readFileSync(path.join(__dirname, '../../docs/concepts/workspace/app.js'), 'utf8');
|
||||
const SOURCE = fs.readFileSync(path.join(__dirname, '../../web/js/workspace/system-integrations.mjs'), 'utf8');
|
||||
|
||||
test('the section keeps the concept rows, labels and options in order', () => {
|
||||
const start = CONCEPT.indexOf('case "integrations":');
|
||||
const body = CONCEPT.slice(start, CONCEPT.indexOf('break;', start));
|
||||
const rows = [...body.matchAll(/setting\("([^"]+)", "([^"]+)", "([^"]+)"/g)].map(([, id, label]) => [id, label]);
|
||||
assert.deepEqual(rows.map(([id]) => id), ['send-destination', 'renderer', 'return-canvas', 'lora-enable', 'lora-triggers',
|
||||
'civitai-key']);
|
||||
for (const [id, label] of rows) {
|
||||
assert.ok(SOURCE.includes(`id: '${id}', label: '${label}'`), `${id} keeps the concept label "${label}"`);
|
||||
}
|
||||
for (const option of ['Ask each time', 'Selected Prompt Manager node', 'Create a Prompt Manager (+/-) node',
|
||||
'Follow host renderer', 'Classic', 'Nodes 2.0']) assert.ok(body.includes(option) && SOURCE.includes(`'${option}'`), option);
|
||||
for (const surface of ['ComfyUI', 'LoRA & resources']) assert.ok(SOURCE.includes(`'h2', '${surface}'`), surface);
|
||||
assert.ok(body.includes('"LoRA reference data", "Review and import resource metadata."'));
|
||||
// Prototype copy is never shipped (Global constraints): the sample detection line and the mock warning.
|
||||
assert.ok(!SOURCE.includes('Sample detection'));
|
||||
assert.ok(!SOURCE.includes('Do not enter a real key'));
|
||||
assert.ok(!SOURCE.includes('Simulate import'));
|
||||
});
|
||||
|
||||
test('LoRA status becomes host values: drafted flags and read-only facts kept apart', async () => {
|
||||
const { loraValues } = await integrations;
|
||||
assert.deepEqual(loraValues({ enabled: true, trigger_words_enabled: false, path: 'custom_nodes/ComfyUI-Lora-Manager',
|
||||
has_civitai_api_key: true, detected: true, detected_path: 'custom_nodes/ComfyUI-Lora-Manager', trigger_cache_loaded: true }), {
|
||||
lora_enabled: true, lora_trigger_words_enabled: false, clear_civitai_api_key: false,
|
||||
lora_path: 'custom_nodes/ComfyUI-Lora-Manager', lora_detected: true, lora_detected_path: 'custom_nodes/ComfyUI-Lora-Manager',
|
||||
lora_detected_usable: true, lora_has_key: true, lora_cache_loaded: true,
|
||||
});
|
||||
const empty = loraValues({});
|
||||
assert.equal(empty.lora_enabled, false);
|
||||
assert.equal(empty.lora_path, '');
|
||||
assert.equal(empty.lora_has_key, false);
|
||||
// The key itself is never part of what the page holds from the server.
|
||||
assert.ok(!Object.keys(loraValues({ civitai_api_key: 'secret' })).includes('civitai_api_key'));
|
||||
});
|
||||
|
||||
test('host changes split between /lora/enable and /settings', async () => {
|
||||
const { splitHostChanges } = await integrations;
|
||||
assert.deepEqual(splitHostChanges({ lora_enabled: true, result_timeout: 5 }),
|
||||
{ lora: { lora_enabled: true }, settings: { result_timeout: 5 } });
|
||||
assert.deepEqual(splitHostChanges({ result_timeout: 5 }), { lora: null, settings: { result_timeout: 5 } });
|
||||
assert.deepEqual(splitHostChanges({ civitai_api_key: 'k', clear_civitai_api_key: false }),
|
||||
{ lora: { civitai_api_key: 'k', clear_civitai_api_key: false }, settings: null });
|
||||
assert.deepEqual(splitHostChanges(undefined), { lora: null, settings: null });
|
||||
});
|
||||
|
||||
test('key never prefilled; blank keeps; Clear saved key clears; saved through Save preferences (D114)', async () => {
|
||||
const { loraPayload, keyText, KEY_COPY } = await integrations;
|
||||
const saved = { lora_enabled: true, lora_trigger_words_enabled: false, lora_path: 'custom_nodes/LM', lora_has_key: true };
|
||||
// The whole LoRA state goes with every save; the saved path is kept; no key unless one was typed.
|
||||
assert.deepEqual(loraPayload({ lora_trigger_words_enabled: true }, saved),
|
||||
{ enabled: true, trigger_words_enabled: true, path: 'custom_nodes/LM' });
|
||||
assert.deepEqual(loraPayload({ civitai_api_key: ' ' }, saved), { enabled: true, trigger_words_enabled: false, path: 'custom_nodes/LM' });
|
||||
assert.deepEqual(loraPayload({ civitai_api_key: ' new-key ' }, saved),
|
||||
{ enabled: true, trigger_words_enabled: false, path: 'custom_nodes/LM', civitai_api_key: 'new-key' });
|
||||
assert.deepEqual(loraPayload({ clear_civitai_api_key: true, civitai_api_key: 'ignored' }, saved),
|
||||
{ enabled: true, trigger_words_enabled: false, path: 'custom_nodes/LM', clear_civitai_api_key: true });
|
||||
assert.deepEqual(loraPayload({ lora_enabled: true }, {}), { enabled: true, trigger_words_enabled: false, path: '' });
|
||||
assert.equal(keyText({ lora_has_key: true }, {}), KEY_COPY.saved);
|
||||
assert.equal(keyText({ lora_has_key: false }, {}), KEY_COPY.none);
|
||||
assert.equal(keyText({ lora_has_key: true }, { clear_civitai_api_key: true }), KEY_COPY.clearing);
|
||||
assert.equal(keyText({ lora_has_key: false }, { civitai_api_key: 'k' }), KEY_COPY.replacing);
|
||||
assert.equal(keyText({}, {}), null);
|
||||
assert.deepEqual(KEY_COPY, { saved: 'A key is saved.', none: 'No key saved.', clearing: 'The saved key is removed when you save.',
|
||||
replacing: 'The new key replaces the saved one when you save.' });
|
||||
});
|
||||
|
||||
test('LoRA rows show detected / enabled / cache-loaded separately', async () => {
|
||||
const { detectionText, cacheText } = await integrations;
|
||||
assert.equal(detectionText({ lora_detected: true, lora_detected_path: 'custom_nodes/ComfyUI-Lora-Manager' }),
|
||||
'Detected: custom_nodes/ComfyUI-Lora-Manager.');
|
||||
assert.equal(detectionText({ lora_detected: true, lora_detected_path: '' }), 'Detected.');
|
||||
assert.equal(detectionText({ lora_detected: false }), 'Not detected in ComfyUI’s custom_nodes folders.');
|
||||
assert.equal(detectionText({}), null);
|
||||
// S10 review N5: a linked install outside custom_nodes is found but can't be enabled.
|
||||
assert.equal(detectionText({ lora_detected: true, lora_detected_path: 'LoRA-Manager', lora_detected_usable: false }),
|
||||
'Found at LoRA-Manager, but outside ComfyUI’s custom_nodes folders, so it can’t be enabled.');
|
||||
assert.equal((await integrations).loraValues({ detected: true, detected_usable: false }).lora_detected_usable, false);
|
||||
assert.equal(cacheText({ lora_cache_loaded: true }), 'Trigger words are loaded.');
|
||||
assert.equal(cacheText({ lora_cache_loaded: false }), 'Trigger words are not loaded.');
|
||||
assert.equal(cacheText({}), null);
|
||||
});
|
||||
|
||||
test('host compatibility shows the detected renderer as status (D106)', async () => {
|
||||
const { rendererText, RENDERER_COPY } = await integrations;
|
||||
const host = (...renderers) => ({ hostId: 'h', label: 'PromptManager — ComfyUI', targets: renderers.map((renderer, id) =>
|
||||
({ id: String(id), roles: ['positive'], renderer })) });
|
||||
assert.equal(rendererText([]), RENDERER_COPY.none);
|
||||
assert.equal(rendererText(null), RENDERER_COPY.none);
|
||||
assert.equal(rendererText([host()]), RENDERER_COPY.noNodes);
|
||||
assert.equal(rendererText([host('vue')]), RENDERER_COPY.noNodes);
|
||||
assert.equal(rendererText([host('classic')]), 'Detected: Classic (1 ComfyUI window).');
|
||||
assert.equal(rendererText([host('nodes2', 'nodes2')]), 'Detected: Nodes 2.0 (1 ComfyUI window).');
|
||||
assert.equal(rendererText([host('nodes2'), host('classic')]), 'Detected: Classic and Nodes 2.0 (2 ComfyUI windows).');
|
||||
});
|
||||
|
||||
test('the import review shows replacement, blocked items and network effects (D107)', async () => {
|
||||
const { importGrid } = await integrations;
|
||||
const plan = { lora_files: 3, with_metadata: 2, current_imports: 5, blocked_by_pairs: 0, planned_downloads: 4,
|
||||
scope: { count: 2, label: 'Replace 5 imported prompts with 2 LoRAs' }, protected: { count: 0, reasons: [] },
|
||||
backup: { policy: 'on', destination_label: 'prompts.db.backup_<date and time>' }, network: { planned_downloads: 4, api_key: true },
|
||||
media: { deletes_files: false, label: 'No files are removed; image links are replaced' } };
|
||||
assert.deepEqual(importGrid(plan), [
|
||||
['Scope', 'Replace 5 imported prompts with 2 LoRAs'],
|
||||
['Found', '3 metadata files · 2 readable'],
|
||||
['Protected', 'Nothing is protected'],
|
||||
['Network', 'Downloads up to 4 example images from CivitAI with your saved key'],
|
||||
['Backup', 'Create and verify before changes (prompts.db.backup_<date and time>)'],
|
||||
['Media on disk', 'No files are removed; image links are replaced'],
|
||||
]);
|
||||
const blocked = importGrid({ ...plan, planned_downloads: 1, network: { api_key: false }, protected: { count: 2,
|
||||
reasons: ['2 imported prompts used by saved pairs'] } });
|
||||
assert.deepEqual(blocked[2], ['Protected', '2 imported prompts used by saved pairs']);
|
||||
assert.deepEqual(blocked[3], ['Network', 'Downloads up to 1 example image from CivitAI']);
|
||||
assert.deepEqual(importGrid({})[0], ['Scope', '0 LoRAs']);
|
||||
assert.deepEqual(importGrid({})[3], ['Network', 'No downloads']);
|
||||
assert.deepEqual(importGrid({})[5], ['Media on disk', 'No files are removed']);
|
||||
});
|
||||
|
||||
test('a LoRA import job reads in Activity: tiles, outcomes and the kept collection on failure', async () => {
|
||||
const { kindOf, finishToast, outcomeLine, viewOf } = await kinds;
|
||||
const view = kindOf('lora_import');
|
||||
assert.equal(view.pillVerb, 'Importing LoRAs');
|
||||
assert.deepEqual(view.tiles.map(([group]) => group), ['imported', 'linked', 'skipped', 'downloaded', 'download_errors', 'errors']);
|
||||
const counters = { imported: 3, linked: 1, skipped: 0, downloaded: 2, download_errors: 0, errors: 0 };
|
||||
const job = { kind: 'lora_import', title: 'LoRA reference import', state: 'finished', cancellable: false, counters };
|
||||
assert.equal(outcomeLine(job), '3 imported · 0 errors');
|
||||
assert.equal(finishToast(job), 'LoRA import finished · 3 imported · 0 errors');
|
||||
assert.equal(finishToast({ ...job, state: 'cancelled' }), 'LoRA import cancelled · 3 imported');
|
||||
const failed = { ...job, state: 'failed', status_line: 'The import failed; the previous LoRA imports were kept.' };
|
||||
assert.equal(finishToast(failed), 'LoRA import failed · The import failed; the previous LoRA imports were kept.');
|
||||
assert.equal(finishToast({ ...failed, status_line: '' }), 'LoRA import failed · 3 imported · 0 errors');
|
||||
assert.equal(viewOf(job).tiles.length, 6);
|
||||
// The legacy page's held import (no counters) reads as the generic job.
|
||||
assert.equal(viewOf({ kind: 'lora_import', title: 'LoRA import (admin page)', cancellable: false, counters: {} }).tiles.length, 0);
|
||||
});
|
||||
|
||||
test('a failed import the files or saved pairs overtook is reviewed again in Integrations', async () => {
|
||||
const { reviewAgainOf } = await import('../../web/js/workspace/activity-job-view.mjs');
|
||||
const failed = failure => ({ kind: 'lora_import', state: 'failed', scope: { operation: 'lora_import', failure } });
|
||||
for (const failure of ['plan-changed', 'storage-changed', 'plan-expired', 'blocked-by-pairs']) {
|
||||
assert.deepEqual(reviewAgainOf(failed(failure)), { section: '#/system/integrations' }, failure);
|
||||
}
|
||||
assert.equal(reviewAgainOf(failed('import-failed')), null);
|
||||
assert.equal(reviewAgainOf({ ...failed('plan-changed'), state: 'cancelled' }), null);
|
||||
});
|
||||
|
||||
test('a host save in two parts reports exactly which part saved (S10 review I3)', async () => {
|
||||
const { writeHostInParts } = await integrations;
|
||||
const saved = { lora_enabled: true, lora_trigger_words_enabled: false, lora_path: 'custom_nodes/LM' };
|
||||
const calls = [];
|
||||
const ok = revision => async body => { calls.push(body); return { success: true, persisted: true, revision }; };
|
||||
const refuse = (code, message) => async body => { calls.push(body); throw Object.assign(new Error(message), { code }); };
|
||||
const run = (changes, postLora, postSettings) => writeHostInParts({ changes, saved, expected_revision: 'r0', postLora, postSettings });
|
||||
|
||||
// Both parts: the second carries the revision the first returned.
|
||||
assert.equal((await run({ lora_trigger_words_enabled: true, result_timeout: 5 }, ok('r1'), ok('r2'))).revision, 'r2');
|
||||
assert.deepEqual(calls.map(body => body.expected_revision), ['r0', 'r1']);
|
||||
// LoRA saved, settings refused: the LoRA keys and the new revision ride on the settings error.
|
||||
calls.length = 0;
|
||||
const later = await run({ lora_trigger_words_enabled: true, gallery_root_paths: ['x'] }, ok('r1'),
|
||||
refuse('invalid-source', 'x cannot be used')).catch(error => error);
|
||||
assert.equal(later.code, 'invalid-source');
|
||||
assert.deepEqual(later.partial, { saved: ['lora_trigger_words_enabled'], revision: 'r1' });
|
||||
// LoRA refused (not detected), settings still sent with the unchanged revision and saved.
|
||||
calls.length = 0;
|
||||
const first = await run({ lora_enabled: true, result_timeout: 5 }, refuse(null, 'LoraManager not found'), ok('r2'))
|
||||
.catch(error => error);
|
||||
assert.equal(first.message, 'LoraManager not found');
|
||||
assert.deepEqual(first.partial, { saved: ['result_timeout'], revision: 'r2' });
|
||||
assert.deepEqual(calls.map(body => body.expected_revision), ['r0', 'r0']);
|
||||
// A conflict on the first write stops there; a lone part fails as itself.
|
||||
calls.length = 0;
|
||||
const conflict = await run({ lora_enabled: false, result_timeout: 5 }, refuse('settings-changed', 'changed'), ok('r2'))
|
||||
.catch(error => error);
|
||||
assert.equal(conflict.code, 'settings-changed');
|
||||
assert.equal(conflict.partial, undefined);
|
||||
assert.equal(calls.length, 1);
|
||||
const alone = await run({ result_timeout: 5 }, ok('r1'), refuse('busy', 'scan running')).catch(error => error);
|
||||
assert.equal(alone.code, 'busy');
|
||||
assert.equal(alone.partial, undefined);
|
||||
const both = await run({ lora_enabled: true, result_timeout: 5 }, refuse(null, 'a'), refuse(null, 'b')).catch(error => error);
|
||||
assert.equal(both.message, 'b');
|
||||
assert.equal(both.partial, undefined);
|
||||
});
|
||||
@@ -0,0 +1,171 @@
|
||||
// System › Maintenance (#249 System S7 Task 19): the concept's rows (app.js 688–690), the D100 plan review built from
|
||||
// the server's Plan, the D101 duplicates review's keep entries and copy, and the D102 metadata cache row.
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
|
||||
const maintenance = import('../../web/js/workspace/system-maintenance.mjs');
|
||||
const review = import('../../web/js/workspace/maintenance-review.mjs');
|
||||
const jobView = import('../../web/js/workspace/activity-job-view.mjs');
|
||||
|
||||
const CONCEPT = fs.readFileSync(path.join(__dirname, '../../docs/concepts/workspace/app.js'), 'utf8');
|
||||
|
||||
test('the section lists the concept rows in its order, labels and danger marks', async () => {
|
||||
const { MAINTENANCE_SURFACES } = await maintenance;
|
||||
const start = CONCEPT.indexOf('case "maintenance":');
|
||||
const body = CONCEPT.slice(start, CONCEPT.indexOf('break;', start));
|
||||
const concept = [...body.matchAll(/operation\("([^"]+)", "([^"]+)", "[^"]+", "([^"]+)"(?:, '[^']*'|, "")?(, true)?\)/g)]
|
||||
.map(([, title, description, label, danger]) => [title, description, label, Boolean(danger)]);
|
||||
const surfaces = [...body.matchAll(/<section class="surface"><h2>([^<]+)<\/h2>/g)].map(match => match[1]);
|
||||
assert.deepEqual(MAINTENANCE_SURFACES.map(([heading]) => heading), surfaces.map(text => text.replace('&', '&')));
|
||||
const production = MAINTENANCE_SURFACES.flatMap(([, rows]) => rows.map(([title, description, label, , danger = false]) =>
|
||||
[title, description, label, danger]));
|
||||
assert.deepEqual(production, concept.map(([title, ...rest]) => [title.replace('&', '&'), ...rest]));
|
||||
assert.equal(production.length, 8);
|
||||
});
|
||||
|
||||
const PLAN = Object.freeze({
|
||||
success: true, token: 't'.repeat(43), operation: 'duplicate_prompts', expires_at: '2026-10-24T12:15:00+00:00',
|
||||
scope: { count: 3, label: '3 duplicate prompts in 2 groups' },
|
||||
protected: { count: 1, reasons: ['1 group belongs to a saved pair'] },
|
||||
backup: { policy: 'on', destination_label: 'prompts.db.backup_<date and time>', expected_bytes: 4096 },
|
||||
media: { deletes_files: false, label: 'No files are deleted' },
|
||||
disk: { free_bytes: 2 ** 30, required_bytes: 2 ** 24, sufficient: true },
|
||||
details: [{ label: 'a cat', detail: '2 copies merged into the oldest' }, { label: 'a dog', detail: '' }],
|
||||
});
|
||||
|
||||
test("plan dialog shows the server's scope, protected, backup policy and media lines (D100)", async () => {
|
||||
const { planGrid, planNote, detailLines, backupText, protectedText, OPERATION_TITLES } = await review;
|
||||
assert.deepEqual(planGrid(PLAN), [
|
||||
['Scope', '3 duplicate prompts in 2 groups'],
|
||||
['Protected', '1 group belongs to a saved pair'],
|
||||
['Backup', 'Create and verify before changes (prompts.db.backup_<date and time>)'],
|
||||
['Media on disk', 'No files are deleted'],
|
||||
]);
|
||||
assert.equal(backupText({ policy: 'off', destination_label: 'x' }), 'No safety backup (the policy is off)');
|
||||
assert.equal(backupText({ policy: 'required', destination_label: 'prompts.db.pre-workflow-storage_<date and time>' }),
|
||||
'Always created and verified first (prompts.db.pre-workflow-storage_<date and time>)');
|
||||
assert.equal(backupText(undefined), 'Create and verify before changes');
|
||||
assert.equal(protectedText({ count: 0, reasons: [] }), 'Nothing is protected');
|
||||
assert.equal(protectedText({ count: 2, reasons: [] }), '2 records kept');
|
||||
assert.equal(protectedText({ count: 3, reasons: ['1 saved pair', '2 titled parents'] }), '1 saved pair · 2 titled parents');
|
||||
assert.deepEqual(planGrid({ scope: { count: 1 } })[0], ['Scope', '1 record']);
|
||||
assert.deepEqual(planGrid({})[3], ['Media on disk', 'No files are removed']);
|
||||
const now = Date.parse('2026-10-24T12:00:00+00:00');
|
||||
assert.equal(planNote(PLAN, { now }), 'Nothing changes until you apply. This review holds for 15 minutes.');
|
||||
assert.equal(planNote({ ...PLAN, expires_at: 'soon' }, { now }), 'Nothing changes until you apply.');
|
||||
assert.equal(planNote({ ...PLAN, expires_at: '2026-10-24T12:00:20+00:00', disk: { free_bytes: 2 ** 20, required_bytes: 3 * 2 ** 20,
|
||||
sufficient: false } }, { now }), 'Nothing changes until you apply. This review holds for 1 minute. ' +
|
||||
'Not enough free disk space: about 3.0 MB is needed and 1.0 MB is free.');
|
||||
assert.deepEqual(detailLines(PLAN), ['a cat · 2 copies merged into the oldest', 'a dog']);
|
||||
assert.deepEqual(detailLines({ details: [null, { label: 7 }] }), []);
|
||||
assert.equal(OPERATION_TITLES.missing_links, 'Missing-file links');
|
||||
// S7 review M8: the workflow migration's plan reads its own title (its job is started outside Maintenance).
|
||||
assert.equal(OPERATION_TITLES.workflow_storage, 'Store each workflow once');
|
||||
});
|
||||
|
||||
test('a refused apply names its reason; a lost answer is not confirmed', async () => {
|
||||
const { applyRefusal, reviewRefusal, NOT_CONFIRMED, APPLY_FAILED, REVIEW_FAILED } = await review;
|
||||
const error = (status, code, message, extra = {}) => Object.assign(new Error(message), { status, code, ...extra });
|
||||
assert.deepEqual(applyRefusal(new TypeError('Failed to fetch')), { toast: NOT_CONFIRMED, lost: true });
|
||||
assert.deepEqual(NOT_CONFIRMED, ['Outcome not confirmed. ', { text: 'Check Activity', href: '#/activity' },
|
||||
' before applying it again.']);
|
||||
const busy = applyRefusal(error(409, 'busy', 'busy', { running: { id: 'a'.repeat(32), title: 'Scan for metadata' } }));
|
||||
assert.deepEqual(busy.toast, ['Another job is running: ', { text: 'Scan for metadata', href: `#/activity/${'a'.repeat(32)}` }, '.']);
|
||||
assert.deepEqual(applyRefusal(error(409, 'plan-used', 'used', { job: { id: 'b'.repeat(32), title: 'Optimize database' } })).toast,
|
||||
['This plan was already applied: ', { text: 'Optimize database', href: `#/activity/${'b'.repeat(32)}` }, '.']);
|
||||
assert.deepEqual(applyRefusal(error(409, 'plan-used', 'used', { job: { id: 'b'.repeat(32), title: null } })).toast[1].text, 'its job');
|
||||
assert.deepEqual(applyRefusal(error(409, 'plan-used', 'used', { job: null })), { toast: 'This plan was already applied.' });
|
||||
assert.deepEqual(applyRefusal(error(410, 'plan-expired', 'This review expired. Review the plan again.')),
|
||||
{ toast: 'This review expired. Review the plan again.', review: true });
|
||||
assert.deepEqual(applyRefusal(error(409, 'storage-changed', 'Replaced.')), { toast: 'Replaced.', review: true });
|
||||
assert.deepEqual(applyRefusal(error(507, 'insufficient-space', 'Not enough free disk space.')), { toast: 'Not enough free disk space.' });
|
||||
// S7 review I1: only our server's own refusal is definitive. A proxy's 5xx, a 2xx whose body could not be read,
|
||||
// or any answer without the server's refusal may have followed a started job: not confirmed, read Activity again.
|
||||
assert.deepEqual(applyRefusal(error(504, null, 'Gateway Timeout')), { toast: NOT_CONFIRMED, lost: true });
|
||||
assert.deepEqual(applyRefusal(error(500, null, 'Internal Server Error')), { toast: NOT_CONFIRMED, lost: true });
|
||||
assert.deepEqual(applyRefusal(error(202, null, 'Invalid JSON response')), { toast: NOT_CONFIRMED, lost: true });
|
||||
assert.deepEqual(applyRefusal(error(502, null, 'Bad Gateway', { refused: false })), { toast: NOT_CONFIRMED, lost: true });
|
||||
assert.deepEqual(applyRefusal(error(400, null, 'token must be the plan token', { refused: true })), { toast: APPLY_FAILED });
|
||||
assert.deepEqual(applyRefusal(error(500, 'apply-failed', 'Maintenance failed; nothing was changed.', { refused: true })),
|
||||
{ toast: 'Maintenance failed; nothing was changed.' });
|
||||
assert.equal(reviewRefusal(error(413, 'too-large', 'This plan is too large to review at once.')),
|
||||
'This plan is too large to review at once.');
|
||||
assert.equal(reviewRefusal(error(500, null, 'boom')), REVIEW_FAILED);
|
||||
// S7 re-review N1: a duplicate review whose sources changed since the search keeps the server's "search again".
|
||||
const changed = 'The sources changed since the search. Search for duplicates again.';
|
||||
assert.equal(reviewRefusal(error(409, 'sources-changed', changed)), changed);
|
||||
assert.equal(reviewRefusal(new TypeError('x')), REVIEW_FAILED);
|
||||
});
|
||||
|
||||
test('keep entries name each copy by group, source index and path, never a bare path', async () => {
|
||||
const { keepEntries } = await review;
|
||||
// Two sources hold a file at the same relative path: only the source index tells them apart.
|
||||
const output = { path: 'same.png', root: 'output', root_index: 0 };
|
||||
const input = { path: 'same.png', root: 'input', root_index: 1 };
|
||||
const chosen = new Map([['hash-a', input], ['hash-b', output]]);
|
||||
assert.deepEqual(keepEntries(chosen), [
|
||||
{ group_id: 'hash-a', root_index: 1, path: 'same.png' },
|
||||
{ group_id: 'hash-b', root_index: 0, path: 'same.png' },
|
||||
]);
|
||||
});
|
||||
|
||||
test('the duplicates dialog says what removing a copy does, History included', async () => {
|
||||
const { DUPLICATES_NOTE, pageText, groupText, copyLabel } = await review;
|
||||
assert.match(DUPLICATES_NOTE, /History keeps the removed paths in its run records\./);
|
||||
assert.match(DUPLICATES_NOTE, /deletes its file from disk/);
|
||||
assert.equal(pageText(0, 10, 24), 'Groups 1–10 of 24');
|
||||
assert.equal(pageText(20, 4, 24), 'Groups 21–24 of 24');
|
||||
assert.equal(pageText(0, 0, 0), 'No duplicate files were found.');
|
||||
assert.equal(groupText({ count: 3, size: 120 * 1024, files: [] }), '3 identical files · 120 KB each');
|
||||
assert.equal(groupText({ size: 3 * 2 ** 20, files: [{}, {}] }), '2 identical files · 3.0 MB each');
|
||||
assert.equal(groupText({ count: 2, size: 10, files: [] }), '2 identical files · 1 KB each');
|
||||
assert.equal(copyLabel({ root: 'output', path: 'a/b.png' }), 'Keep output/a/b.png');
|
||||
assert.equal(copyLabel({ root: '', path: 'b.png', missing: true }), 'Keep b.png (no longer on disk)');
|
||||
});
|
||||
|
||||
test('Metadata cache follows Q5: disabled with its reason while nothing is owned (D102)', async () => {
|
||||
const { metadataCacheDescription, METADATA_CACHE_DESCRIPTION, NOTHING_CACHED_REASON } = await maintenance;
|
||||
assert.equal(METADATA_CACHE_DESCRIPTION, 'Clear only Prompt Manager metadata cached by this browser.');
|
||||
assert.equal(metadataCacheDescription({ keys: [], caches: [] }), `${METADATA_CACHE_DESCRIPTION} ${NOTHING_CACHED_REASON}`);
|
||||
assert.equal(metadataCacheDescription({ keys: ['k'], caches: [] }), METADATA_CACHE_DESCRIPTION);
|
||||
// S7 review M6: a browser that refused the listing is not claimed empty.
|
||||
assert.equal(metadataCacheDescription({ keys: [], caches: [], denied: true }),
|
||||
`${METADATA_CACHE_DESCRIPTION} This browser did not allow checking its storage.`);
|
||||
});
|
||||
|
||||
test('only a finished duplicate search with groups offers Review groups', async () => {
|
||||
const { hasDuplicateGroups } = await jobView;
|
||||
const job = (state, groups, kind = 'duplicates') => ({ kind, state, counters: { groups_found: groups } });
|
||||
assert.equal(hasDuplicateGroups(job('finished', 2)), true);
|
||||
assert.equal(hasDuplicateGroups(job('finished', 0)), false);
|
||||
assert.equal(hasDuplicateGroups(job('running', 2)), false);
|
||||
assert.equal(hasDuplicateGroups(job('finished', 2, 'scan')), false);
|
||||
assert.equal(hasDuplicateGroups({ kind: 'duplicates', state: 'finished', counters: {} }), false);
|
||||
});
|
||||
|
||||
test('a failed apply the library overtook offers Review again, back to its review (S7 review M2)', async () => {
|
||||
const { reviewAgainOf } = await jobView;
|
||||
const failed = (kind, failure, operation) => ({ kind, state: 'failed', scope: { operation, failure } });
|
||||
assert.deepEqual(reviewAgainOf(failed('maintenance', 'plan-changed', 'duplicate_prompts')),
|
||||
{ operation: 'duplicate_prompts' });
|
||||
assert.deepEqual(reviewAgainOf(failed('maintenance', 'storage-changed', 'optimize')), { operation: 'optimize' });
|
||||
// A duplicate cleanup reviewed against a search the sources overtook: search again in Maintenance.
|
||||
assert.deepEqual(reviewAgainOf(failed('duplicate_cleanup', 'sources-changed', 'duplicate_files')),
|
||||
{ section: '#/system/maintenance' });
|
||||
assert.equal(reviewAgainOf(failed('maintenance', 'backup-failed', 'optimize')), null);
|
||||
assert.equal(reviewAgainOf(failed('maintenance', 'plan-changed', 'not_an_operation')), null);
|
||||
assert.equal(reviewAgainOf({ ...failed('maintenance', 'plan-changed', 'optimize'), state: 'finished' }), null);
|
||||
assert.equal(reviewAgainOf(failed('scan', 'plan-changed', 'optimize')), null);
|
||||
});
|
||||
|
||||
test('a plan with nothing to do offers no Apply (S7 live test)', async () => {
|
||||
const { hasWork, NOTHING_TO_APPLY } = await review;
|
||||
assert.equal(hasWork(PLAN), true);
|
||||
assert.equal(hasWork({ ...PLAN, scope: { count: 0, label: 'No duplicate prompt text' } }), false);
|
||||
assert.equal(hasWork({ ...PLAN, operation: 'optimize', scope: { count: 1, label: 'The whole database' } }), true);
|
||||
// A plan whose count is unknown is not assumed empty: the server decides at apply.
|
||||
assert.equal(hasWork({ ...PLAN, scope: {} }), true);
|
||||
assert.equal(NOTHING_TO_APPLY, 'There is nothing to apply; nothing was changed.');
|
||||
});
|
||||
@@ -34,14 +34,17 @@ test('metric 4 is the last health result', async () => {
|
||||
});
|
||||
|
||||
test('library health rows never invent a review count, a backup or thumbnails', async () => {
|
||||
const { ambiguousTitle, backupText } = await overviewModule;
|
||||
const { ambiguousTitle } = await overviewModule;
|
||||
// S9: the Backup status row is Backup & export's status line (system-backups.mjs, tested there).
|
||||
const { backupStatusText } = await import('../../web/js/workspace/system-backups.mjs');
|
||||
assert.equal(ambiguousTitle({ pending: null }), 'Ambiguous image links');
|
||||
assert.equal(ambiguousTitle(undefined), 'Ambiguous image links');
|
||||
assert.equal(ambiguousTitle({ pending: 1 }), '1 ambiguous image link');
|
||||
assert.equal(ambiguousTitle({ pending: 3 }), '3 ambiguous image links');
|
||||
assert.equal(backupText(null), 'No retained backup record.');
|
||||
assert.equal(backupText({ verified_at: 'not a time' }), 'No retained backup record.');
|
||||
assert.match(backupText({ verified_at: '2026-10-22T09:30:00Z' }), /^Last verified backup .+\.$/);
|
||||
assert.equal(backupStatusText({ safety_backups: [], last_download: null }), 'No backup recorded.');
|
||||
assert.equal(backupStatusText({ safety_backups: [{ modified_at: 'not a time' }], last_download: null }), 'No backup recorded.');
|
||||
assert.match(backupStatusText({ safety_backups: [], last_download: { at: '2026-10-22T09:30:00Z', verified: true } }),
|
||||
/^Downloaded backup · .+ · verified\.$/);
|
||||
});
|
||||
|
||||
test('the thumbnail row is its own read of the thumbnail status', async () => {
|
||||
|
||||
@@ -190,6 +190,7 @@ test('status text per outcome', async () => {
|
||||
host: 'Saved to the server configuration.',
|
||||
runtime: 'Applied until the server restarts.',
|
||||
partial: 'Some changes were not saved. Review the highlighted settings.',
|
||||
partialSaved: 'Some changes were saved to the server configuration; the highlighted ones were not.',
|
||||
tabOnly: 'Saved for this tab only; browser storage is unavailable.',
|
||||
conflict: 'Settings changed elsewhere. Reload this section to see them.',
|
||||
restart: 'Restart ComfyUI to apply:',
|
||||
@@ -364,3 +365,42 @@ test('a host save refused for one source says the server reason and names it unt
|
||||
assert.equal(drafts.invalidSource(), null);
|
||||
assert.equal(drafts.statusText(), 'Unsaved changes.');
|
||||
});
|
||||
|
||||
test('a partial host save keeps what saved, adopts its revision and fails only the rest (S10 review I3)', async () => {
|
||||
const { SETTINGS_STATUS } = await source;
|
||||
const domains = adapters({ host: { lora_trigger_words_enabled: false, gallery_root_paths: ['a'] } });
|
||||
const drafts = await loaded(domains);
|
||||
// The LoRA part is written (config.json moves to sha256:lora), then /settings refuses one source.
|
||||
domains.outcome.host = () => {
|
||||
throw Object.assign(new Error('missing can\'t be used: not found. Remove it or reconnect it, then save.'), {
|
||||
code: 'invalid-source', source: 'missing', partial: { saved: ['lora_trigger_words_enabled'], revision: 'sha256:lora' } });
|
||||
};
|
||||
drafts.set('host', 'lora_trigger_words_enabled', true);
|
||||
drafts.set('host', 'gallery_root_paths', ['a', 'missing']);
|
||||
const first = await drafts.save();
|
||||
assert.equal(first.host, 'partial');
|
||||
assert.deepEqual([...drafts.failed('host')], ['gallery_root_paths']);
|
||||
assert.equal(drafts.saved('host').lora_trigger_words_enabled, true);
|
||||
assert.deepEqual(drafts.changes('host'), { gallery_root_paths: ['a', 'missing'] });
|
||||
assert.equal(drafts.invalidSource(), 'missing');
|
||||
assert.equal(drafts.statusText(), 'missing can\'t be used: not found. Remove it or reconnect it, then save.');
|
||||
// The next save carries the revision the saved part produced: no false "changed elsewhere".
|
||||
domains.outcome.host = null;
|
||||
drafts.set('host', 'gallery_root_paths', ['a']);
|
||||
const second = await drafts.save();
|
||||
assert.equal(second.host, 'saved');
|
||||
assert.equal(domains.calls.at(-1)[2].expected_revision, 'sha256:lora');
|
||||
assert.deepEqual(domains.calls.at(-1)[1], { gallery_root_paths: ['a'] });
|
||||
// Without a server reason, the status says some were saved and the highlighted ones were not.
|
||||
domains.outcome.host = () => { throw Object.assign(new Error('boom'), { partial: { saved: ['x'], revision: 'sha256:three' } }); };
|
||||
drafts.set('host', 'x', 1);
|
||||
drafts.set('host', 'y', 2);
|
||||
assert.equal((await drafts.save()).host, 'partial');
|
||||
assert.equal(drafts.statusText(), SETTINGS_STATUS.partialSaved);
|
||||
assert.deepEqual([...drafts.failed('host')], ['y']);
|
||||
// A partial result that saved nothing is a plain failure.
|
||||
domains.outcome.host = () => { throw Object.assign(new Error('boom'), { partial: { saved: [] } }); };
|
||||
drafts.set('host', 'z', 3);
|
||||
assert.equal((await drafts.save()).host, 'failed');
|
||||
assert.equal(drafts.statusText(), SETTINGS_STATUS.partial);
|
||||
});
|
||||
|
||||
@@ -889,7 +889,7 @@ test('review dialog backup box keeps the dialog open and toasts the outcome', as
|
||||
assert.equal(action.getAttribute('aria-disabled'), 'true');
|
||||
// A second press while the copy downloads is ignored.
|
||||
action.click();
|
||||
answer({ ok: true, blob: async () => new Blob(['db']), headers: { get: () => 'attachment; filename="prompts_backup_1.db"' } });
|
||||
answer({ ok: true, blob: async () => new Blob(['db']), headers: { get: name => (name === 'Content-Disposition' ? 'attachment; filename="prompts_backup_1.db"' : null) } });
|
||||
await tick();
|
||||
assert.deepEqual(toasts, [BACKUP_DONE]);
|
||||
assert.equal(modal(dom).open, true);
|
||||
@@ -933,3 +933,37 @@ test('the scan review is a thin openReviewDialog caller with its scope switch',
|
||||
dom.restore();
|
||||
}
|
||||
});
|
||||
|
||||
test('a review dialog reports one dismissal for Cancel, ×, Escape, a replacing dialog and the page leaving, never after its primary (S9 M1)', async () => {
|
||||
const { openReviewDialog, openContentDialog, closeWorkspaceDialog } = await dialogsModule;
|
||||
const { createScope } = await coreModule;
|
||||
const dom = fakeDOM();
|
||||
const scope = createScope();
|
||||
const dismissed = [];
|
||||
const open = (name, primary = 'Restore') => openReviewDialog({ scope, title: 'Restore preview', grid: [['Integrity', 'Verified']],
|
||||
cancelLabel: 'Cancel', primary: { label: primary, danger: true }, onStart() {}, onDismiss: () => dismissed.push(name) });
|
||||
try {
|
||||
open('cancel');
|
||||
button(byId(dom, 'modal-body'), 'Cancel').click();
|
||||
open('close');
|
||||
byId(dom, 'modal-close').click();
|
||||
open('escape');
|
||||
modal(dom).dispatchEvent(new Event('cancel', { cancelable: true }));
|
||||
open('replaced');
|
||||
openContentDialog({ scope, title: 'Other', nodes: [] });
|
||||
closeWorkspaceDialog();
|
||||
open('started');
|
||||
button(byId(dom, 'modal-body'), 'Restore').click();
|
||||
await tick();
|
||||
open('left');
|
||||
scope.dispose();
|
||||
await tick();
|
||||
assert.deepEqual(dismissed, ['cancel', 'close', 'escape', 'replaced', 'left']);
|
||||
assert.throws(() => openReviewDialog({ scope: createScope(), title: 'x', grid: [['a', 'b']], cancelLabel: 'Cancel',
|
||||
primary: { label: 'Go' }, onStart() {}, onDismiss: 'no' }), TypeError);
|
||||
} finally {
|
||||
scope.dispose();
|
||||
closeWorkspaceDialog();
|
||||
dom.restore();
|
||||
}
|
||||
});
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
"""Run the Python suite with a hang watchdog: ``python -m tests.run_suite``.
|
||||
|
||||
Why: on PR #349 a macOS job stopped inside the test suite and sat silent until
|
||||
GitHub cancelled it after six hours, with no stack to read. This runner does
|
||||
the same discovery as ``python -m unittest discover -s tests`` (same
|
||||
``sys.path`` when started with ``-m`` from the repo root, same warning
|
||||
filters, and the exit codes of Python 3.12+ on every version: 0, 1, and 5
|
||||
when no tests ran), and adds a watchdog built on
|
||||
``faulthandler.dump_traceback_later``: when the timer is not re-armed in time,
|
||||
every thread's stack goes to stderr and the process exits non-zero. It works
|
||||
the same on Linux, macOS and Windows.
|
||||
|
||||
The timer is armed before discovery, re-armed at the start of every test with
|
||||
PM_TEST_HANG_LIMIT seconds (default 120), and re-armed when the run ends with
|
||||
PM_TEST_SHUTDOWN_LIMIT seconds (default 120) to cover interpreter exit
|
||||
(non-daemon threads, atexit handlers, coverage's save). It is never
|
||||
cancelled. Class and module fixtures (setUpClass, tearDownModule, ...) run
|
||||
between two ``start:`` lines, so they count toward the test before them.
|
||||
120 s stays below asyncio's 300 s wait for default-executor threads on Python
|
||||
3.12 and 3.13 (3.10 and 3.11 wait without limit), so a hang in an
|
||||
``IsolatedAsyncioTestCase`` loop shutdown is dumped inside the test that hung.
|
||||
|
||||
Reading a dump: the last ``start: <test id>`` line names the test. If its
|
||||
"ok" line comes after it, the hang is after the test body: its teardown or
|
||||
event-loop shutdown, or a class/module fixture before the next test (the
|
||||
stack shows which). A dump after the summary is a hang at interpreter exit
|
||||
(a non-daemon thread or an atexit handler).
|
||||
|
||||
A limit is a measurement, not a dial: change it only with new measurements
|
||||
written next to it (TESTING.md "Time limits and hangs").
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import faulthandler
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
DEFAULT_PER_TEST_LIMIT = 120
|
||||
DEFAULT_SHUTDOWN_LIMIT = 120
|
||||
NO_TESTS_EXIT = 5
|
||||
USAGE_EXIT = 2
|
||||
|
||||
|
||||
class LimitError(ValueError):
|
||||
"""An environment limit is not a positive integer."""
|
||||
|
||||
|
||||
def read_limit(name, default):
|
||||
"""Return the positive integer in env var ``name``, or ``default``."""
|
||||
raw = os.environ.get(name)
|
||||
if raw is None:
|
||||
return default
|
||||
text = raw.strip()
|
||||
if not (text.isascii() and text.isdigit()) or int(text) <= 0:
|
||||
raise LimitError(f"run_suite: {name} must be a positive integer, got {raw!r}")
|
||||
return int(text)
|
||||
|
||||
|
||||
def make_result_class(stream, per_test_limit, shutdown_limit):
|
||||
"""Build a TextTestResult that names each test and re-arms the watchdog."""
|
||||
|
||||
def arm(seconds):
|
||||
faulthandler.dump_traceback_later(seconds, exit=True, file=stream)
|
||||
|
||||
class WatchdogResult(unittest.TextTestResult):
|
||||
def startTest(self, test):
|
||||
# At default verbosity the progress dots share a line; start ours.
|
||||
prefix = "\n" if self.dots else ""
|
||||
stream.write(f"{prefix}start: {test.id()}\n")
|
||||
stream.flush()
|
||||
super().startTest(test)
|
||||
arm(per_test_limit)
|
||||
|
||||
def stopTestRun(self):
|
||||
super().stopTestRun()
|
||||
arm(shutdown_limit)
|
||||
|
||||
return WatchdogResult
|
||||
|
||||
|
||||
def parse_args(argv):
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="python -m tests.run_suite",
|
||||
description="Discover and run the tests with a hang watchdog.",
|
||||
)
|
||||
# As in unittest, -v and -q share one setting and the last one wins.
|
||||
parser.set_defaults(verbosity=1)
|
||||
parser.add_argument(
|
||||
"-v",
|
||||
"--verbose",
|
||||
dest="verbosity",
|
||||
action="store_const",
|
||||
const=2,
|
||||
help="verbose output",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-q",
|
||||
"--quiet",
|
||||
dest="verbosity",
|
||||
action="store_const",
|
||||
const=0,
|
||||
help="quiet output",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-s",
|
||||
"--start-directory",
|
||||
default=str(Path(__file__).resolve().parent),
|
||||
help="directory to start discovery (default: this tests folder)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-p", "--pattern", default="test*.py", help="pattern to match test files"
|
||||
)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
args = parse_args(sys.argv[1:] if argv is None else argv)
|
||||
# Keep the real stderr: a test may replace sys.stderr, and the watchdog
|
||||
# needs a stream with a file descriptor.
|
||||
stream = sys.stderr
|
||||
try:
|
||||
per_test_limit = read_limit("PM_TEST_HANG_LIMIT", DEFAULT_PER_TEST_LIMIT)
|
||||
shutdown_limit = read_limit("PM_TEST_SHUTDOWN_LIMIT", DEFAULT_SHUTDOWN_LIMIT)
|
||||
except LimitError as error:
|
||||
stream.write(f"{error}\n")
|
||||
return USAGE_EXIT
|
||||
|
||||
faulthandler.dump_traceback_later(per_test_limit, exit=True, file=stream)
|
||||
start = os.path.abspath(args.start_directory)
|
||||
suite = unittest.TestLoader().discover(start, args.pattern, top_level_dir=start)
|
||||
runner = unittest.TextTestRunner(
|
||||
stream=stream,
|
||||
verbosity=args.verbosity,
|
||||
# As unittest.main does: show warnings unless -W/PYTHONWARNINGS set them.
|
||||
warnings=None if sys.warnoptions else "default",
|
||||
resultclass=make_result_class(stream, per_test_limit, shutdown_limit),
|
||||
)
|
||||
result = runner.run(suite)
|
||||
if result.testsRun == 0 and not result.skipped:
|
||||
return NO_TESTS_EXIT
|
||||
return 0 if result.wasSuccessful() else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+58
-4
@@ -112,6 +112,16 @@ class AdminAPITestCase(AioHTTPTestCase):
|
||||
if api is not None:
|
||||
# A scan of an ambiguous file captures it for Needs review.
|
||||
api.close_annotation_captures()
|
||||
# A scan's job hands its missing previews to the automatic worker
|
||||
# as it ends, which can be after the test read the scan's last
|
||||
# event and after tearDownAsync closed the worker: that hand-off
|
||||
# starts a new one. Close it for good now that no job runs, so
|
||||
# no preview write (an open ``.part`` file) outlives the test:
|
||||
# this API refuses new workers, and the close joins the worker.
|
||||
# Not ``final=True``, which also shuts the process-wide encoders
|
||||
# (#319) that later tests use.
|
||||
api._auto_thumbnails_shut = True
|
||||
api.close_auto_thumbnails()
|
||||
PromptModel.close_all_instances() # restore may have swapped api.db
|
||||
assert_closed(self, self.tmpdir)
|
||||
shutil.rmtree(self.tmpdir, ignore_errors=True)
|
||||
@@ -548,6 +558,48 @@ class TestDuplicateScanSyncBody(AdminAPITestCase):
|
||||
)
|
||||
self.assertNotIn(self.tmpdir, json.dumps(groups))
|
||||
|
||||
def test_view_full_names_the_gallery_folder_source(self):
|
||||
"""Carry-over (System S7): a group's View Full opens the copy in the gallery's
|
||||
own source; without ``?root=`` the serve route took the first source that
|
||||
has a file at that path, another source's file when two hold one."""
|
||||
renders = self.output_dir / "renders"
|
||||
renders.mkdir()
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [
|
||||
self._root(renders),
|
||||
self._root(self.output_dir),
|
||||
]
|
||||
(renders / "same.png").write_bytes(b"other content")
|
||||
(self.output_dir / "same.png").write_bytes(b"same")
|
||||
(self.output_dir / "a#1.png").write_bytes(b"same")
|
||||
|
||||
groups = self._scan()
|
||||
|
||||
self.assertEqual(len(groups), 1)
|
||||
self.assertEqual(
|
||||
sorted(img["url"] for img in groups[0]["images"]),
|
||||
[
|
||||
"/prompt_manager/images/serve/a%231.png?root=1",
|
||||
"/prompt_manager/images/serve/same.png?root=1",
|
||||
],
|
||||
)
|
||||
|
||||
def test_view_full_without_a_listed_source_keeps_its_plain_url(self):
|
||||
renders = self.output_dir / "renders"
|
||||
renders.mkdir()
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(renders)]
|
||||
(self.output_dir / "a.png").write_bytes(b"same")
|
||||
(self.output_dir / "b.png").write_bytes(b"same")
|
||||
|
||||
groups = self._scan()
|
||||
|
||||
self.assertEqual(
|
||||
sorted(img["url"] for img in groups[0]["images"]),
|
||||
[
|
||||
"/prompt_manager/images/serve/a.png",
|
||||
"/prompt_manager/images/serve/b.png",
|
||||
],
|
||||
)
|
||||
|
||||
def test_thumbnails_dir_is_skipped_and_thumbnail_url_reported(self):
|
||||
(self.output_dir / "a.png").write_bytes(b"same")
|
||||
(self.output_dir / "b.png").write_bytes(b"same")
|
||||
@@ -1963,7 +2015,7 @@ class TestBackupRestoreContract(AdminAPITestCase):
|
||||
self.assertFalse((await invalid.json())["success"])
|
||||
compatible = self._snapshot_upload()
|
||||
with patch(
|
||||
"database.models.rotate_history_epoch",
|
||||
"database.restore_stage.rotate_history_epoch",
|
||||
side_effect=sqlite3.OperationalError("staging failed before safety backup"),
|
||||
):
|
||||
staged = await self._restore("database_file", compatible)
|
||||
@@ -2057,7 +2109,8 @@ class TestBackupRestoreContract(AdminAPITestCase):
|
||||
resp = await self._restore("database_file", upload)
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertIn("verify_database_file", calls)
|
||||
self.assertIn("restore_from_file", calls)
|
||||
self.assertIn("stage_restore", calls)
|
||||
self.assertIn("publish_staged", calls)
|
||||
|
||||
async def test_restore_keeps_at_most_ten_safety_backups(self):
|
||||
import time
|
||||
@@ -2098,12 +2151,12 @@ class TestBackupRestoreContract(AdminAPITestCase):
|
||||
self.api.db.save_prompt(text="restored", prompt_hash="h1")
|
||||
upload = self._snapshot_upload()
|
||||
|
||||
def boom(src_path):
|
||||
def boom(staged):
|
||||
raise PermissionError(
|
||||
13, "Permission denied", os.path.join(self.tmpdir, "prompts.db")
|
||||
)
|
||||
|
||||
self.api.db.model.restore_from_file = boom
|
||||
self.api.db.model.publish_staged = boom
|
||||
resp = await self._restore("database_file", upload)
|
||||
body = await resp.text()
|
||||
self.assertEqual(resp.status, 500)
|
||||
@@ -2413,6 +2466,7 @@ class TestActivityJobs(AdminAPITestCase):
|
||||
ambiguous=None,
|
||||
untraced=None,
|
||||
records=None,
|
||||
history=None,
|
||||
):
|
||||
key = Path(media_file).relative_to(self.output_dir).as_posix()
|
||||
outcome = names[key]
|
||||
|
||||
@@ -38,6 +38,13 @@ from py.api import PromptManagerAPI, _public_path
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
|
||||
def _is_upload(path):
|
||||
"""Not the safety backup's temp copy (database/safety_backup.py)."""
|
||||
from database.safety_backup import SAFETY_TEMP_PREFIX
|
||||
|
||||
return not os.path.basename(path).startswith(SAFETY_TEMP_PREFIX)
|
||||
|
||||
|
||||
def _texts(path):
|
||||
conn = sqlite3.connect(path)
|
||||
try:
|
||||
@@ -114,7 +121,7 @@ class TestBackupRoute(BackupRestoreTestCase):
|
||||
created.append(result[1])
|
||||
return result
|
||||
|
||||
with patch("py.api.admin.tempfile.mkstemp", spy_mkstemp):
|
||||
with patch("tempfile.mkstemp", spy_mkstemp):
|
||||
resp = await self.client.get("/prompt_manager/backup")
|
||||
|
||||
self.assertEqual(resp.status, 200)
|
||||
@@ -125,6 +132,10 @@ class TestBackupRoute(BackupRestoreTestCase):
|
||||
self.assertTrue(disposition.rstrip('"').endswith(".db"))
|
||||
|
||||
body = await resp.read()
|
||||
# The handler removes its temp file after the last byte is sent,
|
||||
# which can be just after the client holds it: wait on that state.
|
||||
while self.api._jobs.active("backup") is not None or os.path.exists(created[0]):
|
||||
await asyncio.sleep(0)
|
||||
downloaded = self._write("downloaded.db", body)
|
||||
self.assertEqual(
|
||||
_texts(downloaded), ["backup me 0", "backup me 1", "backup me 2"]
|
||||
@@ -263,13 +274,18 @@ class TestRestoreRoute(BackupRestoreTestCase):
|
||||
created.append(result[1])
|
||||
return result
|
||||
|
||||
with patch("py.api.admin.tempfile.mkstemp", spy_mkstemp):
|
||||
with patch("tempfile.mkstemp", spy_mkstemp):
|
||||
resp = await self._restore(upload)
|
||||
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertEqual(len(created), 1)
|
||||
for suffix in ("", "-wal", "-shm", "-journal"):
|
||||
self.assertFalse(os.path.exists(created[0] + suffix))
|
||||
# One upload, in the library's folder; it and the safety backup's
|
||||
# temp copy are gone afterwards.
|
||||
uploads = [p for p in created if _is_upload(p)]
|
||||
self.assertEqual(len(uploads), 1)
|
||||
self.assertEqual(os.path.dirname(uploads[0]), self.tmpdir)
|
||||
for path in created:
|
||||
for suffix in ("", "-wal", "-shm", "-journal"):
|
||||
self.assertFalse(os.path.exists(path + suffix))
|
||||
|
||||
async def test_text_file_is_rejected_and_database_untouched(self):
|
||||
self._save("keep me")
|
||||
@@ -398,7 +414,7 @@ class TestLargeRestores(BackupRestoreTestCase):
|
||||
created.append(result[1])
|
||||
return result
|
||||
|
||||
return patch("py.api.admin.tempfile.mkstemp", spy), created
|
||||
return patch("tempfile.mkstemp", spy), created
|
||||
|
||||
def _assert_removed(self, created):
|
||||
for path in created:
|
||||
@@ -417,7 +433,7 @@ class TestLargeRestores(BackupRestoreTestCase):
|
||||
self.assertEqual(
|
||||
[p["text"] for p in self.api.db.search_prompts()], ["restored"]
|
||||
)
|
||||
self.assertEqual(len(created), 1)
|
||||
self.assertEqual(len([p for p in created if _is_upload(p)]), 1)
|
||||
self._assert_removed(created)
|
||||
|
||||
async def test_a_short_disk_is_refused_up_front(self):
|
||||
@@ -455,7 +471,7 @@ class TestLargeRestores(BackupRestoreTestCase):
|
||||
self.assertEqual(data["code"], "insufficient-space")
|
||||
self.assertTrue(data["error"].endswith("Nothing was changed."))
|
||||
self.assertEqual([p["text"] for p in self.api.db.search_prompts()], ["keep me"])
|
||||
self.assertEqual(len(created), 1)
|
||||
self.assertEqual(len([p for p in created if _is_upload(p)]), 1)
|
||||
self._assert_removed(created)
|
||||
|
||||
|
||||
|
||||
+64
-4
@@ -36,7 +36,7 @@ from aiohttp.test_utils import AioHTTPTestCase
|
||||
|
||||
from database.operations import PromptDatabase
|
||||
from py.api import PromptManagerAPI
|
||||
from py import config_store, lora_utils
|
||||
from py import config_store, lora_import, lora_utils
|
||||
from py.config import IntegrationConfig, PromptManagerConfig
|
||||
from py.api import settings_routes as settings_module
|
||||
from py.api import lora_integration as lora_module
|
||||
@@ -252,6 +252,21 @@ class TestLoraStatusRoute(LoraAPITestCase):
|
||||
self.assertFalse(os.path.isabs(public))
|
||||
self.assertNotIn(str(self.tmp_root), body)
|
||||
self.assertFalse(data["trigger_cache_loaded"])
|
||||
self.assertTrue(data["detected_usable"])
|
||||
|
||||
async def test_status_says_a_detected_install_outside_custom_nodes_is_unusable(
|
||||
self,
|
||||
):
|
||||
# S10 review N5: found (a linked install), but enabling it is refused.
|
||||
outside = _make_lora_manager(self.tmp_root)
|
||||
self.detect_mock.return_value = str(outside)
|
||||
resp = await self.client.request("GET", "/prompt_manager/lora/status")
|
||||
data = await resp.json()
|
||||
self.assertEqual((data["detected"], data["detected_usable"]), (True, False))
|
||||
|
||||
self.detect_mock.return_value = None
|
||||
resp = await self.client.request("GET", "/prompt_manager/lora/status")
|
||||
self.assertFalse((await resp.json())["detected_usable"])
|
||||
|
||||
async def test_status_error_is_500(self):
|
||||
with patch("py.lora_utils.get_trigger_cache", side_effect=RuntimeError("x")):
|
||||
@@ -528,6 +543,48 @@ class TestLoraEnableRoute(LoraAPITestCase):
|
||||
self.assertEqual(self.config_file.read_bytes(), b"{not json")
|
||||
self.assertFalse(IntegrationConfig.LORA_TRIGGER_WORDS_ENABLED)
|
||||
|
||||
async def test_enable_with_a_stale_revision_changes_nothing(self):
|
||||
# System S10: the Save row sends the revision it read; a file changed
|
||||
# since is a settings-changed conflict and nothing is written.
|
||||
self.config_file.write_text(json.dumps({"web_ui": {"result_timeout": 9}}))
|
||||
resp = await self.client.request("GET", "/prompt_manager/settings")
|
||||
revision = (await resp.json())["revision"]
|
||||
self.config_file.write_text(json.dumps({"web_ui": {"result_timeout": 10}}))
|
||||
before = self.config_file.read_bytes()
|
||||
|
||||
resp = await self._post_json(
|
||||
"/prompt_manager/lora/enable",
|
||||
{
|
||||
"enabled": False,
|
||||
"trigger_words_enabled": True,
|
||||
"expected_revision": revision,
|
||||
},
|
||||
)
|
||||
body = await resp.json()
|
||||
|
||||
self.assertEqual((resp.status, body["code"]), (409, "settings-changed"))
|
||||
self.assertEqual(self.config_file.read_bytes(), before)
|
||||
self.assertFalse(IntegrationConfig.LORA_TRIGGER_WORDS_ENABLED)
|
||||
|
||||
resp = await self._post_json(
|
||||
"/prompt_manager/lora/enable",
|
||||
{
|
||||
"enabled": False,
|
||||
"trigger_words_enabled": True,
|
||||
"expected_revision": body["revision"],
|
||||
},
|
||||
)
|
||||
saved = await resp.json()
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertTrue(saved["revision"].startswith("sha256:"))
|
||||
self.assertTrue(IntegrationConfig.LORA_TRIGGER_WORDS_ENABLED)
|
||||
|
||||
resp = await self._post_json(
|
||||
"/prompt_manager/lora/enable",
|
||||
{"enabled": False, "expected_revision": 7},
|
||||
)
|
||||
self.assertEqual(resp.status, 400)
|
||||
|
||||
async def test_enable_requires_boolean_flags_and_body_cap(self):
|
||||
for body in (
|
||||
{"enabled": "yes"},
|
||||
@@ -762,9 +819,9 @@ class TestLoraScanRoute(LoraAPITestCase):
|
||||
async def test_scan_error_hides_the_server_path(self):
|
||||
self._enable()
|
||||
err = PermissionError(13, "Permission denied", str(self.tmp_root / "x.db"))
|
||||
with patch.object(self.api.db, "delete_prompts_by_category", side_effect=err):
|
||||
with patch.object(self.api.db, "replace_category_prompts", side_effect=err):
|
||||
resp = await self.client.request("POST", "/prompt_manager/lora/scan")
|
||||
body = await resp.text()
|
||||
body = await resp.text() # the stream ends inside the patch
|
||||
self.assertNotIn(str(self.tmp_root), body)
|
||||
self.assertIn("x.db", body)
|
||||
# The admin page's modal shows the failure as its status and closes.
|
||||
@@ -862,7 +919,10 @@ class TestLoraScanRoute(LoraAPITestCase):
|
||||
loras.mkdir()
|
||||
_write_metadata(loras, "neon", words=["glow"])
|
||||
|
||||
with patch.object(self.api.db, "save_prompt", side_effect=RuntimeError("db")):
|
||||
# Preparing one LoRA's record fails: it is skipped, the import goes on.
|
||||
with patch.object(
|
||||
lora_import, "build_record", side_effect=RuntimeError("bad record")
|
||||
):
|
||||
resp, events, _ = await self._scan(loras)
|
||||
|
||||
self.assertEqual(events[-1]["imported"], 0)
|
||||
|
||||
@@ -23,6 +23,7 @@ sys.modules.setdefault("server", _mock_server)
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from database import safety_backup # noqa: E402
|
||||
from database.operations import PromptDatabase # noqa: E402
|
||||
from py.api import PromptManagerAPI # noqa: E402
|
||||
from py.api.maintenance_plans import PlanExpired # noqa: E402
|
||||
@@ -562,7 +563,7 @@ class TestSafetyBackup(ApplyTestCase):
|
||||
|
||||
frozen = datetime.datetime(2026, 10, 23, 12, 0, 0, 123456)
|
||||
model = self.api.db.model
|
||||
with patch.object(maintenance, "_now", return_value=frozen):
|
||||
with patch.object(safety_backup, "_now", return_value=frozen):
|
||||
names = [maintenance.take_safety_backup(model) for _ in range(3)]
|
||||
self.assertEqual(
|
||||
names,
|
||||
@@ -949,6 +950,37 @@ class TestFinalReviewFixes(ApplyTestCase):
|
||||
self.assertEqual(reply["code"], "plan-used")
|
||||
self.assertEqual(self.api._jobs.list(), [])
|
||||
|
||||
async def test_an_apply_that_never_started_says_so_with_its_code(self):
|
||||
"""S7 review I1: the server's own pre-start failure is definitive, so
|
||||
it carries a code the workspace can tell from a proxy's 5xx."""
|
||||
from py.api import maintenance
|
||||
|
||||
self.save("Orphan")
|
||||
plan = await self.preview("unused_components")
|
||||
with patch.object(maintenance, "disk_plan", side_effect=OSError("disk")):
|
||||
reply = await self.apply(plan["token"], 500)
|
||||
self.assertEqual(reply["code"], "apply-failed")
|
||||
self.assertEqual(reply["error"], maintenance.APPLY_FAILED)
|
||||
self.assertEqual(self.api._jobs.list(), [])
|
||||
|
||||
async def test_a_failed_apply_names_its_failure_in_its_scope(self):
|
||||
"""S7 review M2: Activity offers Review again from the job's own record."""
|
||||
orphan, soon = self.save("Orphan"), self.save("Soon a recipe")
|
||||
plan = await self.preview("unused_components")
|
||||
with self.api.db.model.get_connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO prompt_library_state(prompt_id,title) VALUES(?,?)",
|
||||
(soon, "Recipe"),
|
||||
)
|
||||
body = await self.apply(plan["token"])
|
||||
job = self.api._jobs.get(body["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(job.task), WAIT)
|
||||
self.assertEqual(
|
||||
job.summary()["scope"],
|
||||
{"operation": "unused_components", "failure": "plan-changed"},
|
||||
)
|
||||
self.assertIsNotNone(self.api.db.get_prompt_by_id(orphan))
|
||||
|
||||
async def test_the_backup_decision_is_frozen_when_the_job_starts(self):
|
||||
PromptManagerConfig.SAFETY_BACKUP = False
|
||||
self.save("Orphan")
|
||||
@@ -987,7 +1019,7 @@ class TestFinalReviewFixes(ApplyTestCase):
|
||||
with (
|
||||
patch.object(maintenance.os, "link", side_effect=link),
|
||||
patch.object(
|
||||
maintenance, "_reserve_backup_name", side_effect=AssertionError
|
||||
safety_backup, "reserve_backup_name", side_effect=AssertionError
|
||||
),
|
||||
):
|
||||
name = maintenance.take_safety_backup(self.api.db.model)
|
||||
|
||||
@@ -161,46 +161,48 @@ class TestAdminEndpointsUseConfigPath(unittest.TestCase):
|
||||
self.assertNotIn("/nonexistent/custom", message)
|
||||
|
||||
def test_backup_uses_model_db_path(self):
|
||||
"""backup_database should read from self.db.model.db_path."""
|
||||
"""The backup snapshot is taken of self.db.model.db_path."""
|
||||
if "server" not in sys.modules:
|
||||
sys.modules["server"] = MagicMock()
|
||||
|
||||
from py.api.admin import AdminRoutesMixin
|
||||
from py.api.backup_restore import write_backup_snapshot
|
||||
|
||||
from database.operations import PromptDatabase
|
||||
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as f:
|
||||
db_file = f.name
|
||||
|
||||
folder = tempfile.mkdtemp()
|
||||
db_file = os.path.join(folder, "custom.db")
|
||||
# A WAL-safe backup copies pages through sqlite itself, so the
|
||||
# configured path has to hold a real database, not arbitrary bytes.
|
||||
real_db = PromptDatabase(db_file)
|
||||
try:
|
||||
stub = self._make_api_stub(db_file)
|
||||
stub.db.model = real_db.model
|
||||
result = self._run_async(
|
||||
AdminRoutesMixin.backup_database(stub, MagicMock())
|
||||
)
|
||||
|
||||
self.assertEqual(result.content_type, "application/octet-stream")
|
||||
self.assertGreater(len(result.body), 0)
|
||||
self.assertTrue(result.body.startswith(b"SQLite format 3\x00"))
|
||||
snapshot, warning = write_backup_snapshot(stub.db.model)
|
||||
try:
|
||||
self.assertIsNone(warning)
|
||||
self.assertEqual(os.path.dirname(snapshot), folder)
|
||||
with open(snapshot, "rb") as handle:
|
||||
self.assertTrue(handle.read(16) == b"SQLite format 3\x00")
|
||||
finally:
|
||||
os.unlink(snapshot)
|
||||
finally:
|
||||
real_db.close_all()
|
||||
for suffix in ("", "-wal", "-shm"):
|
||||
if os.path.exists(db_file + suffix):
|
||||
os.unlink(db_file + suffix)
|
||||
for name in os.listdir(folder):
|
||||
os.unlink(os.path.join(folder, name))
|
||||
os.rmdir(folder)
|
||||
|
||||
def test_backup_reports_missing_custom_path(self):
|
||||
"""backup_database should return 404 when configured path doesn't exist."""
|
||||
"""The backup answers 404 when the configured path doesn't exist."""
|
||||
if "server" not in sys.modules:
|
||||
sys.modules["server"] = MagicMock()
|
||||
|
||||
from py.api.admin import AdminRoutesMixin
|
||||
from py.api.backup_restore import BackupRestoreRoutesMixin
|
||||
|
||||
stub = self._make_api_stub("/nonexistent/custom/prompts.db")
|
||||
|
||||
result = self._run_async(AdminRoutesMixin.backup_database(stub, MagicMock()))
|
||||
result = self._run_async(
|
||||
BackupRestoreRoutesMixin._stream_backup(stub, MagicMock())
|
||||
)
|
||||
body = json.loads(result.body)
|
||||
self.assertFalse(body["success"])
|
||||
self.assertEqual(result.status, 404)
|
||||
|
||||
@@ -512,7 +512,7 @@ class HistoryRestoreTests(unittest.TestCase):
|
||||
subscribe_history_restore(self.path, lambda: notices.append(True))
|
||||
)
|
||||
with patch(
|
||||
"database.models.rotate_history_epoch",
|
||||
"database.restore_stage.rotate_history_epoch",
|
||||
side_effect=sqlite3.OperationalError("injected staging failure"),
|
||||
):
|
||||
with self.assertRaisesRegex(sqlite3.OperationalError, "injected staging"):
|
||||
|
||||
@@ -24,6 +24,13 @@ from utils.execution_history_events import (
|
||||
)
|
||||
from utils.execution_history_writer import HistoryWriter
|
||||
|
||||
# These calls must return while another thread holds the writer in a step that
|
||||
# waits up to 3 s (release.wait(3)). 1 s still tells "returned at once" from
|
||||
# "waited for the release" without failing on a loaded shared runner (#242:
|
||||
# generous ceilings, never a short wall clock). A 0.1 s ceiling failed at
|
||||
# 0.19 s on macOS py3.13 (PR #351).
|
||||
NON_BLOCKING_CEILING = 1.0
|
||||
|
||||
|
||||
class ManualClock:
|
||||
def __init__(self):
|
||||
@@ -590,10 +597,10 @@ class HistoryWriterTests(unittest.TestCase):
|
||||
before = time.monotonic()
|
||||
self.assertTrue(writer.status().accepting)
|
||||
self.assertTrue(writer.submit(self.finish_command(start)))
|
||||
self.assertLess(time.monotonic() - before, 0.1)
|
||||
self.assertLess(time.monotonic() - before, NON_BLOCKING_CEILING)
|
||||
before = time.monotonic()
|
||||
self.assertFalse(writer.close(0))
|
||||
self.assertLess(time.monotonic() - before, 0.1)
|
||||
self.assertLess(time.monotonic() - before, NON_BLOCKING_CEILING)
|
||||
self.assertGreater(writer.context_budget.charged_bytes, 0)
|
||||
self.assertEqual(writer.status().pending, 1)
|
||||
finally:
|
||||
@@ -663,11 +670,11 @@ class HistoryWriterTests(unittest.TestCase):
|
||||
before = time.monotonic()
|
||||
self.assertFalse(writer.submit(self.start_command()))
|
||||
self.assertIsNone(writer.status().epoch)
|
||||
self.assertLess(time.monotonic() - before, 0.1)
|
||||
self.assertLess(time.monotonic() - before, NON_BLOCKING_CEILING)
|
||||
self.assertIsNone(self.db.get_execution_run(self.start_command().run_id))
|
||||
before = time.monotonic()
|
||||
self.assertFalse(writer.close(0.03))
|
||||
self.assertLess(time.monotonic() - before, 0.2)
|
||||
self.assertLess(time.monotonic() - before, NON_BLOCKING_CEILING)
|
||||
release.set()
|
||||
self.wait_for(lambda: not writer.status().alive)
|
||||
self.assertFalse(writer.status().accepting)
|
||||
@@ -958,7 +965,7 @@ class HistoryWriterTests(unittest.TestCase):
|
||||
self.assertTrue(writer.submit(queued))
|
||||
before = time.monotonic()
|
||||
self.assertFalse(writer.close(0.03))
|
||||
self.assertLess(time.monotonic() - before, 0.2)
|
||||
self.assertLess(time.monotonic() - before, NON_BLOCKING_CEILING)
|
||||
release.set()
|
||||
self.wait_for(lambda: not writer.status().alive)
|
||||
self.assertIsNone(self.db.get_execution_run(queued.run_id))
|
||||
|
||||
@@ -82,6 +82,26 @@ class TracesPrompts(unittest.TestCase):
|
||||
self.assertEqual(result["positivePrompt"], "the real positive")
|
||||
self.assertEqual(result["negativePrompt"], "the real negative")
|
||||
|
||||
def test_core_sampler_with_an_out_of_order_widget_input_keeps_positions(self):
|
||||
# #343 review R4: only ``cfg`` is widget-bound, so the bound names are not
|
||||
# a schema prefix; the values follow the KSampler schema positions.
|
||||
result = run("workflow-only-core-widget-input-out-of-order")
|
||||
self.assertEqual(
|
||||
(result["seed"], result["steps"], result["cfgScale"], result["sampler"]),
|
||||
(9, 30, 4.5, "dpmpp_2m"),
|
||||
)
|
||||
|
||||
def test_editor_text_inputs_without_widget_bindings_keep_their_positions(self):
|
||||
# #343: Focused/Split/Pair text inputs are plain sockets. The widget-bound
|
||||
# inputs that remain (category, tags, search_text) must not shift the
|
||||
# positional widget values onto the wrong names.
|
||||
result = run("workflow-only-detached-editor-text")
|
||||
self.assertEqual(result["positivePrompt"], "a detached positive")
|
||||
self.assertEqual(result["negativePrompt"], "plain negative")
|
||||
result = run("workflow-only-detached-pair-text")
|
||||
self.assertEqual(result["positivePrompt"], "a detached positive")
|
||||
self.assertEqual(result["negativePrompt"], "a detached negative")
|
||||
|
||||
|
||||
class ChoosesTheSampler(unittest.TestCase):
|
||||
def test_first_sampler_follows_javascript_key_order(self):
|
||||
|
||||
+17
-13
@@ -12,6 +12,7 @@ from database.operations import PromptDatabase
|
||||
from utils.hashing import generate_prompt_hash
|
||||
from utils.image_monitor import ImageGenerationHandler
|
||||
from utils.prompt_tracker import PromptTracker
|
||||
from utils.run_keys import prompt_node_key
|
||||
|
||||
LOADER = {
|
||||
"class_type": "CheckpointLoaderSimple",
|
||||
@@ -103,8 +104,11 @@ class TestMetadataLinking(LinkingTestCase):
|
||||
batch_graph = graph()
|
||||
batch_graph["134"]["inputs"]["text"] = ["7", 0]
|
||||
batch_graph["7"] = {"class_type": "PromptSearchList", "inputs": {}}
|
||||
self.tracker.set_current_prompt("item one", {"prompt_id": first})
|
||||
self.tracker.set_current_prompt("item two", {"prompt_id": second})
|
||||
key = prompt_node_key(batch_graph, "134")
|
||||
for text, prompt_id in (("item one", first), ("item two", second)):
|
||||
self.tracker.set_current_prompt(
|
||||
text, {"prompt_id": prompt_id}, run_key=key, run_id="run-1"
|
||||
)
|
||||
|
||||
self.assertEqual(self._process({"prompt": batch_graph}), first)
|
||||
self.assertEqual(self._process({"prompt": batch_graph}), second)
|
||||
@@ -122,9 +126,10 @@ class TestMetadataLinking(LinkingTestCase):
|
||||
self.assertEqual(
|
||||
self.handler._find_prompt_from_metadata(metadata), ("unknown", None)
|
||||
)
|
||||
latest = self._save("most recent prompt") # recency fallback picks this
|
||||
self.assertEqual(self._process(metadata), latest)
|
||||
self.assertNotEqual(latest, negative)
|
||||
self._save("most recent prompt") # never picked by recency (#342)
|
||||
self.assertIsNone(self._process(metadata))
|
||||
self.assertEqual(self.linked, [])
|
||||
self.assertIsNotNone(negative)
|
||||
|
||||
def test_workflow_fallback_still_used_without_an_api_graph(self):
|
||||
positive = self._save("from workflow only")
|
||||
@@ -200,12 +205,14 @@ class TestForeignWorkflows(LinkingTestCase):
|
||||
self.handler._find_prompt_from_metadata(metadata), ("foreign", None)
|
||||
)
|
||||
|
||||
def test_missing_metadata_is_unknown_and_uses_the_queue(self):
|
||||
def test_missing_metadata_is_unknown_and_left_unlinked(self):
|
||||
first = self._queue_batch_item("item one")
|
||||
self.assertEqual(
|
||||
self.handler._find_prompt_from_metadata(None), ("unknown", None)
|
||||
)
|
||||
self.assertEqual(self._process(None), first)
|
||||
# Nothing proves which run made it (#342): unlinked, queue untouched
|
||||
self.assertIsNone(self._process(None))
|
||||
self.assertEqual(self.tracker.pop_next_prompt()["id"], first)
|
||||
|
||||
def test_tri_state_for_linked_unknown_and_foreign(self):
|
||||
positive = self._save("a cat")
|
||||
@@ -224,14 +231,11 @@ class TestForeignWorkflows(LinkingTestCase):
|
||||
("unknown", None),
|
||||
)
|
||||
|
||||
def test_unknown_image_without_any_context_falls_back_to_the_latest_prompt(self):
|
||||
def test_unknown_image_is_never_given_the_latest_prompt(self):
|
||||
self._save("older")
|
||||
latest = self._save("latest")
|
||||
self.assertEqual(self._process(None), latest)
|
||||
|
||||
def test_recent_prompt_fallback_handles_an_empty_database(self):
|
||||
self.assertIsNone(self.handler._get_fallback_prompt())
|
||||
self._save("latest")
|
||||
self.assertIsNone(self._process(None))
|
||||
self.assertIsNone(self._process({"prompt": graph("not saved")}))
|
||||
|
||||
def test_metadata_lookup_errors_are_unknown(self):
|
||||
self._save("a cat")
|
||||
|
||||
@@ -326,7 +326,7 @@ class TestHandlerScheduling(unittest.TestCase):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.tmp.cleanup)
|
||||
self.tracker = MagicMock()
|
||||
self.tracker.get_current_prompt.return_value = {"id": 7}
|
||||
self.tracker.keyed_snapshot.return_value = {"key": ({"id": 7}, "run", 1)}
|
||||
self.handler = im_mod.ImageGenerationHandler(MagicMock(), self.tracker)
|
||||
self.handler.processing_delay = 0
|
||||
self.handler.settle_interval = 0.01
|
||||
@@ -354,7 +354,14 @@ class TestHandlerScheduling(unittest.TestCase):
|
||||
self.handler.on_created(FakeEvent(os.path.join(self.tmp.name, "c.txt")))
|
||||
self.handler.on_created(FakeEvent(self.tmp.name, is_directory=True))
|
||||
self.assertTrue(wait_for(lambda: self.handler.pending_count() == 0))
|
||||
self.assertEqual(self.processed, [(path, {"id": 7})])
|
||||
self.assertEqual(self.processed, [(path, {"key": ({"id": 7}, "run", 1)})])
|
||||
|
||||
def test_a_failed_snapshot_still_schedules_the_image(self):
|
||||
self.tracker.keyed_snapshot.side_effect = RuntimeError("tracker gone")
|
||||
path = self._image("d.png")
|
||||
self.handler.on_created(FakeEvent(path))
|
||||
self.assertTrue(wait_for(lambda: self.handler.pending_count() == 0))
|
||||
self.assertEqual(self.processed, [(path, None)])
|
||||
|
||||
def test_images_are_processed_in_creation_order(self):
|
||||
paths = [self._image(f"{i}.png") for i in range(5)]
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
"""Registry leases: short request-scoped work versus a database switch (#249 S8).
|
||||
|
||||
A lease is refused while an ``exclusive`` job (restore, Move library,
|
||||
Optimize) runs, never refuses another lease or any job, is not listed, and an
|
||||
exclusive job can await the leases already running. Ordering is driven by
|
||||
events only.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from py.api.jobs import JobBusy, JobRegistry # noqa: E402
|
||||
|
||||
|
||||
class TestLeases(unittest.IsolatedAsyncioTestCase):
|
||||
def setUp(self):
|
||||
self.registry = JobRegistry()
|
||||
|
||||
def start(self, kind, admission):
|
||||
gate = asyncio.Event()
|
||||
|
||||
async def runner(job):
|
||||
await gate.wait()
|
||||
|
||||
job = self.registry.start(
|
||||
kind, kind.title(), admission=admission, scope={}, runner=runner
|
||||
)
|
||||
return job, gate
|
||||
|
||||
async def test_leases_share_and_are_not_listed(self):
|
||||
scan, gate = self.start("scan", "records")
|
||||
async with self.registry.lease("Needs review decision"):
|
||||
async with self.registry.lease("Needs review decision"):
|
||||
self.assertEqual(self.registry.leases, {"Needs review decision": 2})
|
||||
# A lease never refuses a job, even an exclusive one.
|
||||
self.assertIsNone(self.registry.blocker("health", admission="read"))
|
||||
self.assertEqual(self.registry.leases, {})
|
||||
self.assertEqual([job.kind for job in self.registry.list()], ["scan"])
|
||||
gate.set()
|
||||
await scan.task
|
||||
|
||||
async def test_lease_refused_while_exclusive_job_runs(self):
|
||||
restore, gate = self.start("restore", "exclusive")
|
||||
with self.assertRaises(JobBusy) as refused:
|
||||
async with self.registry.lease("Maintenance preview"):
|
||||
self.fail("admitted beside a restore")
|
||||
self.assertIs(refused.exception.running, restore)
|
||||
self.assertEqual(self.registry.leases, {})
|
||||
gate.set()
|
||||
await restore.task
|
||||
async with self.registry.lease("Maintenance preview"):
|
||||
pass
|
||||
|
||||
async def test_exclusive_job_awaits_running_leases(self):
|
||||
entered, release = asyncio.Event(), asyncio.Event()
|
||||
order = []
|
||||
|
||||
async def decision():
|
||||
async with self.registry.lease("Needs review decision"):
|
||||
entered.set()
|
||||
await release.wait()
|
||||
order.append("decision done")
|
||||
|
||||
task = asyncio.ensure_future(decision())
|
||||
await entered.wait()
|
||||
drained = asyncio.ensure_future(self.registry.drain_leases())
|
||||
await asyncio.sleep(0)
|
||||
self.assertFalse(drained.done())
|
||||
release.set()
|
||||
self.assertTrue(await drained)
|
||||
order.append("drained")
|
||||
await task
|
||||
self.assertEqual(order, ["decision done", "drained"])
|
||||
self.assertTrue(await self.registry.drain_leases()) # none: at once
|
||||
|
||||
async def test_drain_reports_a_lease_that_outlives_the_ceiling(self):
|
||||
hold = self.registry.lease("Stuck preview")
|
||||
await hold.__aenter__()
|
||||
try:
|
||||
with self.assertLogs("py.api.jobs", "WARNING") as logs:
|
||||
self.assertFalse(await self.registry.drain_leases(timeout=0))
|
||||
self.assertIn("Stuck preview", "\n".join(logs.output))
|
||||
self.assertEqual(self.registry._drain_waiters, [])
|
||||
finally:
|
||||
await hold.__aexit__(None, None, None)
|
||||
self.assertEqual(self.registry.leases, {})
|
||||
|
||||
async def test_a_failed_body_still_releases_the_lease(self):
|
||||
with self.assertRaises(RuntimeError):
|
||||
async with self.registry.lease("Needs review decision"):
|
||||
raise RuntimeError("boom")
|
||||
self.assertEqual(self.registry.leases, {})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
File diff suppressed because it is too large
Load Diff
+699
-8
@@ -12,8 +12,10 @@ import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import types
|
||||
import unittest
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import urllib.response
|
||||
@@ -294,12 +296,9 @@ class TestCivitaiUrlPolicy(unittest.TestCase):
|
||||
self.assertEqual(
|
||||
CIVITAI_HOSTS,
|
||||
frozenset(
|
||||
{
|
||||
"civitai.com",
|
||||
"www.civitai.com",
|
||||
"api.civitai.com",
|
||||
"image.civitai.com",
|
||||
}
|
||||
f"{prefix}{domain}"
|
||||
for domain in ("civitai.com", "civitai.red", "civitai.green")
|
||||
for prefix in ("", "www.", "api.", "image.")
|
||||
),
|
||||
)
|
||||
self.assertEqual(MAX_CIVITAI_DOWNLOAD_BYTES, 50 * 1024 * 1024)
|
||||
@@ -332,6 +331,36 @@ class TestCivitaiUrlPolicy(unittest.TestCase):
|
||||
def test_explicit_default_port_is_allowed(self):
|
||||
self.assertTrue(is_civitai_url("https://civitai.com:443/x"))
|
||||
|
||||
def test_red_and_green_starting_hosts_are_allowed(self):
|
||||
# #340: CivitAI's own .red and .green domains serve the same content.
|
||||
for url in (
|
||||
"https://image.civitai.red/xG1/abc/width=450/1.jpeg",
|
||||
"https://civitai.red/api/download/1",
|
||||
"https://civitai.green/api/download/1",
|
||||
"https://image.civitai.green/xG1/abc/1.jpeg",
|
||||
"https://IMAGE.CIVITAI.RED/x",
|
||||
"https://civitai.green:443/x",
|
||||
):
|
||||
self.assertTrue(is_civitai_url(url), url)
|
||||
|
||||
def test_red_and_green_lookalikes_are_refused(self):
|
||||
for url in (
|
||||
"http://image.civitai.red/x",
|
||||
"http://civitai.green/x",
|
||||
"https://civitai.red.evil.example/x",
|
||||
"https://civitai.green.evil.example/x",
|
||||
"https://evilcivitai.red/x",
|
||||
"https://evilcivitai.green/x",
|
||||
"https://civitai.redx/x",
|
||||
"https://civitai.greenx/x",
|
||||
"https://civitai.red@evil.example/x",
|
||||
"https://user:pw@civitai.red/x",
|
||||
"https://civitai.green:8443/x",
|
||||
"https://blobs-b2.civitai.red/x", # starting hosts stay a fixed list
|
||||
"https://evil.example/civitai.red",
|
||||
):
|
||||
self.assertFalse(is_civitai_url(url), url)
|
||||
|
||||
|
||||
class TestLoraManagerPathPolicy(unittest.TestCase):
|
||||
"""Only a direct child of a ComfyUI custom_nodes dir named *lora* qualifies."""
|
||||
@@ -524,6 +553,114 @@ class TestDownloadOne(unittest.TestCase):
|
||||
self.assertFalse(self.target.exists())
|
||||
|
||||
|
||||
class TestDownloadIsAtomic(unittest.TestCase):
|
||||
"""I2 (S10 review): a cached image is never seen half written."""
|
||||
|
||||
def setUp(self):
|
||||
self.tmpdir = Path(tempfile.mkdtemp())
|
||||
self.addCleanup(shutil.rmtree, self.tmpdir, True)
|
||||
self.target = self.tmpdir / "out.jpg"
|
||||
|
||||
def test_the_image_appears_only_once_complete(self):
|
||||
seen = []
|
||||
real_replace = os.replace
|
||||
|
||||
def replace(source, destination):
|
||||
# At the moment the name appears, the temp copy is whole and closed.
|
||||
seen.append((Path(source).parent, self.target.exists()))
|
||||
with open(source, "rb") as handle:
|
||||
self.assertEqual(handle.read(2), b"\xff\xd8")
|
||||
return real_replace(source, destination)
|
||||
|
||||
opener = _FakeOpener(_FakeResponse(_png_bytes()))
|
||||
with (
|
||||
patch.object(lora_utils, "_open_url", opener),
|
||||
patch.object(lora_utils.os, "replace", side_effect=replace),
|
||||
):
|
||||
result = lora_utils._download_one(
|
||||
"https://civitai.com/api/download/1", self.target, ""
|
||||
)
|
||||
self.assertEqual(seen, [(self.tmpdir, False)])
|
||||
self.assertEqual(os.path.realpath(result), os.path.realpath(self.target))
|
||||
self.assertEqual(sorted(p.name for p in self.tmpdir.iterdir()), ["out.jpg"])
|
||||
|
||||
def test_a_failed_write_leaves_neither_the_image_nor_a_temp_file(self):
|
||||
opener = _FakeOpener(_FakeResponse(_png_bytes()))
|
||||
with (
|
||||
patch.object(lora_utils, "_open_url", opener),
|
||||
patch.object(lora_utils.os, "replace", side_effect=OSError("disk full")),
|
||||
):
|
||||
result = lora_utils._download_one(
|
||||
"https://civitai.com/api/download/1", self.target, ""
|
||||
)
|
||||
self.assertIsNone(result)
|
||||
self.assertEqual(list(self.tmpdir.iterdir()), [])
|
||||
|
||||
def test_an_unreadable_image_leaves_no_temp_file(self):
|
||||
opener = _FakeOpener(_FakeResponse(b"not an image"))
|
||||
with patch.object(lora_utils, "_open_url", opener):
|
||||
result = lora_utils._download_one(
|
||||
"https://civitai.com/api/download/1", self.target, ""
|
||||
)
|
||||
self.assertIsNone(result)
|
||||
self.assertEqual(list(self.tmpdir.iterdir()), [])
|
||||
|
||||
@unittest.skipIf(os.name == "nt", "POSIX file modes")
|
||||
def test_the_image_gets_the_permissions_a_plain_save_gives(self):
|
||||
# S10 re-review N7: the cached JPEG honours the umask like img.save did.
|
||||
previous = os.umask(0o022)
|
||||
self.addCleanup(os.umask, previous)
|
||||
opener = _FakeOpener(_FakeResponse(_png_bytes()))
|
||||
with patch.object(lora_utils, "_open_url", opener):
|
||||
lora_utils._download_one(
|
||||
"https://civitai.com/api/download/1", self.target, ""
|
||||
)
|
||||
self.assertEqual(self.target.stat().st_mode & 0o777, 0o644)
|
||||
|
||||
|
||||
class TestStaleDownloadSweep(unittest.TestCase):
|
||||
"""S10 re-review N7: a killed download's ``.pm-*.part`` is swept later."""
|
||||
|
||||
def setUp(self):
|
||||
self.cache = Path(tempfile.mkdtemp())
|
||||
self.addCleanup(shutil.rmtree, self.cache, True)
|
||||
self.lora = self.cache / "my_lora"
|
||||
self.lora.mkdir()
|
||||
|
||||
def _file(self, path, age_seconds):
|
||||
path.write_bytes(b"x")
|
||||
stamp = time.time() - age_seconds
|
||||
os.utime(path, (stamp, stamp))
|
||||
return path
|
||||
|
||||
def test_only_old_temp_downloads_are_removed(self):
|
||||
old = self._file(self.lora / ".pm-abc123.part", 2 * 3600)
|
||||
old_top = self._file(self.cache / ".pm-top.part", 2 * 3600)
|
||||
fresh = self._file(self.lora / ".pm-new.part", 60)
|
||||
image = self._file(self.lora / "abc.jpg", 2 * 3600)
|
||||
other = self._file(self.lora / "keep.part", 2 * 3600)
|
||||
prefixed = self._file(self.lora / ".pm-keep.jpg", 2 * 3600)
|
||||
deep = self.lora / "nested"
|
||||
deep.mkdir()
|
||||
nested = self._file(deep / ".pm-deep.part", 2 * 3600)
|
||||
|
||||
lora_utils.sweep_stale_downloads(self.cache)
|
||||
|
||||
self.assertFalse(old.exists())
|
||||
self.assertFalse(old_top.exists())
|
||||
for kept in (fresh, image, other, prefixed, nested):
|
||||
self.assertTrue(kept.exists(), kept.name)
|
||||
|
||||
def test_a_missing_cache_is_fine(self):
|
||||
lora_utils.sweep_stale_downloads(self.cache / "absent")
|
||||
|
||||
def test_a_file_that_cannot_be_removed_is_left(self):
|
||||
old = self._file(self.lora / ".pm-abc.part", 2 * 3600)
|
||||
with patch.object(lora_utils.os, "remove", side_effect=PermissionError):
|
||||
lora_utils.sweep_stale_downloads(self.cache)
|
||||
self.assertTrue(old.exists())
|
||||
|
||||
|
||||
class _FakeHTTPSHandler(urllib.request.HTTPSHandler):
|
||||
"""Serves canned responses through a real urllib opener, recording requests.
|
||||
|
||||
@@ -577,7 +714,7 @@ class TestRedirectsAreRefused(unittest.TestCase):
|
||||
if host not in CIVITAI_HOSTS:
|
||||
self.assertIsNone(req.get_header("Authorization"))
|
||||
|
||||
def test_redirect_to_civitai_is_not_followed_either(self):
|
||||
def test_redirect_to_civitai_is_followed_without_the_key(self):
|
||||
handler = _FakeHTTPSHandler("https://image.civitai.com/other.png")
|
||||
opener = lora_utils._build_opener(handler)
|
||||
|
||||
@@ -586,9 +723,477 @@ class TestRedirectsAreRefused(unittest.TestCase):
|
||||
"https://civitai.com/api/download/1", self.tmpdir / "x.jpg", "key"
|
||||
)
|
||||
|
||||
# image.civitai.com redirects again (the fake handler always does),
|
||||
# so the hop limit ends this chain; the first request alone had the key.
|
||||
self.assertIsNone(result)
|
||||
self.assertEqual(len(handler.requests), lora_utils.MAX_CIVITAI_REDIRECTS + 1)
|
||||
self.assertEqual(handler.requests[0].get_header("Authorization"), "Bearer key")
|
||||
for req in handler.requests[1:]:
|
||||
self.assertIsNone(req.get_header("Authorization"))
|
||||
|
||||
|
||||
class _ChainHTTPSHandler(urllib.request.HTTPSHandler):
|
||||
"""Serves ``routes`` (url -> (status, Location or None)) through a real
|
||||
urllib opener, recording every request; an unknown URL answers 404."""
|
||||
|
||||
def __init__(self, routes):
|
||||
super().__init__()
|
||||
self.routes = routes
|
||||
self.requests = []
|
||||
|
||||
def https_open(self, req):
|
||||
self.requests.append(req)
|
||||
status, location = self.routes.get(req.full_url, (404, None))
|
||||
if location is not None:
|
||||
headers = email.message_from_string(f"Location: {location}\n")
|
||||
resp = urllib.response.addinfourl(
|
||||
BytesIO(b""), headers, req.full_url, status
|
||||
)
|
||||
resp.msg = "Redirect"
|
||||
return resp
|
||||
body = _png_bytes() if status == 200 else b""
|
||||
headers = email.message_from_string("Content-Type: image/png\n")
|
||||
resp = urllib.response.addinfourl(BytesIO(body), headers, req.full_url, status)
|
||||
resp.msg = "OK" if status == 200 else "Not Found"
|
||||
return resp
|
||||
|
||||
def http_open(self, req): # a plain-HTTP hop must never be requested
|
||||
self.requests.append(req)
|
||||
raise AssertionError(f"plain HTTP requested: {req.full_url}")
|
||||
|
||||
|
||||
START = "https://image.civitai.com/xG1/abc/width=450/1.jpeg"
|
||||
BLOB = "https://blobs-b2.civitai.com/file/civitai-media-cache/1/original"
|
||||
|
||||
|
||||
class TestCivitaiRedirects(unittest.TestCase):
|
||||
"""#337: image.civitai.com now redirects to blobs-b2.civitai.com.
|
||||
|
||||
A redirect is followed only over HTTPS to civitai.com or a subdomain of
|
||||
it, at most MAX_CIVITAI_REDIRECTS hops, and never with the API key.
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
self.tmpdir = Path(tempfile.mkdtemp())
|
||||
self.addCleanup(shutil.rmtree, self.tmpdir, True)
|
||||
self.target = self.tmpdir / "x.jpg"
|
||||
|
||||
def _download(self, routes, url=START, api_key="secret-key"):
|
||||
handler = _ChainHTTPSHandler(routes)
|
||||
opener = lora_utils._build_opener(handler)
|
||||
with patch.object(lora_utils, "_opener", opener):
|
||||
result = lora_utils._download_one(url, self.target, api_key)
|
||||
return result, handler.requests
|
||||
|
||||
def _refused(self, location, status=302):
|
||||
result, requests = self._download({START: (status, location)})
|
||||
self.assertIsNone(result)
|
||||
self.assertEqual([r.full_url for r in requests], [START])
|
||||
self.assertFalse(self.target.exists())
|
||||
|
||||
def test_redirect_to_an_allowed_host_succeeds_without_the_key(self):
|
||||
for status in (301, 302, 303, 307, 308):
|
||||
with self.subTest(status=status):
|
||||
if self.target.exists():
|
||||
os.remove(self.target)
|
||||
result, requests = self._download(
|
||||
{START: (status, BLOB), BLOB: (200, None)}
|
||||
)
|
||||
|
||||
self.assertIsNotNone(result)
|
||||
self.assertTrue(self.target.exists())
|
||||
self.assertEqual([r.full_url for r in requests], [START, BLOB])
|
||||
self.assertEqual(
|
||||
requests[0].get_header("Authorization"), "Bearer secret-key"
|
||||
)
|
||||
self.assertIsNone(requests[1].get_header("Authorization"))
|
||||
# urllib adds Host itself; nothing else rides along.
|
||||
self.assertEqual(
|
||||
[h for h in requests[1].header_items() if h[0] != "Host"],
|
||||
[("User-agent", "ComfyUI-PromptManager/1.0")],
|
||||
)
|
||||
|
||||
def test_a_relative_location_resolves_on_the_same_host(self):
|
||||
follow = "https://image.civitai.com/elsewhere/1.jpeg"
|
||||
result, requests = self._download(
|
||||
{START: (302, "/elsewhere/1.jpeg"), follow: (200, None)}
|
||||
)
|
||||
|
||||
self.assertIsNotNone(result)
|
||||
self.assertEqual([r.full_url for r in requests], [START, follow])
|
||||
self.assertIsNone(requests[1].get_header("Authorization"))
|
||||
|
||||
def test_a_plain_http_redirect_is_refused(self):
|
||||
self._refused("http://blobs-b2.civitai.com/file/1")
|
||||
|
||||
def test_a_redirect_to_another_host_is_refused(self):
|
||||
self._refused("https://evil.example/steal")
|
||||
|
||||
def test_lookalike_hosts_are_refused(self):
|
||||
for location in (
|
||||
"https://civitai.com.evil.example/x",
|
||||
"https://evilcivitai.com/x",
|
||||
"https://civitai.com@evil.example/x",
|
||||
"https://user:pw@blobs-b2.civitai.com/x",
|
||||
"https://blobs-b2.civitai.com:8443/x",
|
||||
"https://civitai.co/x",
|
||||
"//evil.example/x",
|
||||
"ftp://blobs-b2.civitai.com/x",
|
||||
"https://[::1]/x",
|
||||
):
|
||||
with self.subTest(location=location):
|
||||
self._refused(location)
|
||||
|
||||
def test_a_redirect_without_a_location_is_refused(self):
|
||||
result, requests = self._download({START: (302, "")})
|
||||
self.assertIsNone(result)
|
||||
self.assertEqual(len(requests), 1)
|
||||
|
||||
def test_the_hop_limit_is_enforced(self):
|
||||
limit = lora_utils.MAX_CIVITAI_REDIRECTS
|
||||
self.assertEqual(limit, 3)
|
||||
hops = [f"https://blobs-b2.civitai.com/hop/{i}" for i in range(limit + 1)]
|
||||
|
||||
# Exactly ``limit`` redirects: followed.
|
||||
routes = {START: (302, hops[0])}
|
||||
for i in range(limit - 1):
|
||||
routes[hops[i]] = (302, hops[i + 1])
|
||||
routes[hops[limit - 1]] = (200, None)
|
||||
result, requests = self._download(routes)
|
||||
self.assertIsNotNone(result)
|
||||
self.assertEqual(len(requests), limit + 1)
|
||||
self.assertTrue(
|
||||
all(r.get_header("Authorization") is None for r in requests[1:])
|
||||
)
|
||||
|
||||
# One more redirect: refused, and the next hop is never requested.
|
||||
os.remove(self.target)
|
||||
routes[hops[limit - 1]] = (302, hops[limit])
|
||||
routes[hops[limit]] = (200, None)
|
||||
result, requests = self._download(routes)
|
||||
self.assertIsNone(result)
|
||||
self.assertEqual(len(requests), limit + 1)
|
||||
self.assertNotIn(hops[limit], [r.full_url for r in requests])
|
||||
self.assertFalse(self.target.exists())
|
||||
|
||||
def test_a_redirect_loop_ends_at_the_hop_limit(self):
|
||||
result, requests = self._download({START: (302, START)})
|
||||
self.assertIsNone(result)
|
||||
self.assertEqual(len(requests), lora_utils.MAX_CIVITAI_REDIRECTS + 1)
|
||||
|
||||
def test_size_cap_applies_after_a_redirect(self):
|
||||
with patch.object(lora_utils, "MAX_CIVITAI_DOWNLOAD_BYTES", 10):
|
||||
result, requests = self._download({START: (302, BLOB), BLOB: (200, None)})
|
||||
self.assertIsNone(result)
|
||||
self.assertEqual(len(requests), 2)
|
||||
self.assertFalse(self.target.exists())
|
||||
|
||||
def test_an_error_after_a_redirect_fails_cleanly(self):
|
||||
result, requests = self._download({START: (302, BLOB)}) # BLOB is 404
|
||||
self.assertIsNone(result)
|
||||
self.assertEqual(len(requests), 2)
|
||||
self.assertFalse(self.target.exists())
|
||||
|
||||
def test_the_opener_itself_still_refuses_every_redirect(self):
|
||||
handler = _ChainHTTPSHandler({START: (302, BLOB), BLOB: (200, None)})
|
||||
opener = lora_utils._build_opener(handler)
|
||||
req = urllib.request.Request(START, headers={"Authorization": "Bearer k"})
|
||||
|
||||
with self.assertRaises(urllib.error.HTTPError) as caught:
|
||||
opener.open(req, timeout=5)
|
||||
caught.exception.close()
|
||||
self.assertEqual(len(handler.requests), 1)
|
||||
|
||||
def test_redirect_logs_never_carry_the_key_or_the_query(self):
|
||||
signed = BLOB + "?X-Amz-Signature=sig123"
|
||||
with self.assertLogs("prompt_manager.lora_utils", level="DEBUG") as logs:
|
||||
self._download(
|
||||
{START: (302, "https://evil.example/x?token=t0k3n")},
|
||||
api_key="secret-key",
|
||||
)
|
||||
self._download({START: (302, signed), signed: (200, None)})
|
||||
text = "\n".join(logs.output)
|
||||
self.assertNotIn("secret-key", text)
|
||||
self.assertNotIn("t0k3n", text)
|
||||
self.assertNotIn("sig123", text)
|
||||
|
||||
|
||||
RED_BLOB = "https://blobs-b2.civitai.red/file/civitai-media-cache/1/original"
|
||||
GREEN_BLOB = "https://media.civitai.green/file/1/original"
|
||||
|
||||
|
||||
class TestCivitaiRedAndGreenDomains(unittest.TestCase):
|
||||
"""#340: civitai.red and civitai.green are CivitAI's own domains.
|
||||
|
||||
They are redirect targets under the civitai.com rules (HTTPS, default
|
||||
port, no credentials, parsed host name, 3 hops, never the key) and
|
||||
starting hosts whose first request may carry the key.
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
self.tmpdir = Path(tempfile.mkdtemp())
|
||||
self.addCleanup(shutil.rmtree, self.tmpdir, True)
|
||||
self.target = self.tmpdir / "x.jpg"
|
||||
|
||||
def _download(self, routes, url=START, api_key="secret-key"):
|
||||
handler = _ChainHTTPSHandler(routes)
|
||||
opener = lora_utils._build_opener(handler)
|
||||
with patch.object(lora_utils, "_opener", opener):
|
||||
result = lora_utils._download_one(url, self.target, api_key)
|
||||
return result, handler.requests
|
||||
|
||||
def test_redirect_targets_on_red_and_green_are_allowed(self):
|
||||
for url in (
|
||||
"https://civitai.red/x",
|
||||
"https://civitai.green/x",
|
||||
RED_BLOB,
|
||||
GREEN_BLOB,
|
||||
"https://a.b.civitai.red:443/x",
|
||||
"https://CIVITAI.GREEN/x",
|
||||
):
|
||||
self.assertTrue(lora_utils.is_civitai_redirect_target(url), url)
|
||||
|
||||
def test_redirect_target_lookalikes_are_refused(self):
|
||||
for url in (
|
||||
"https://civitai.red.evil.example/x",
|
||||
"https://civitai.green.evil.example/x",
|
||||
"https://evilcivitai.red/x",
|
||||
"https://evilcivitai.green/x",
|
||||
"https://civitai.redx/x",
|
||||
"https://civitai.greenx/x",
|
||||
"https://civitai.red@evil.example/x",
|
||||
"https://user:pw@blobs-b2.civitai.red/x",
|
||||
"https://user@civitai.green/x",
|
||||
"https://blobs-b2.civitai.red:8443/x",
|
||||
"http://blobs-b2.civitai.red/x",
|
||||
"http://civitai.green/x",
|
||||
"ftp://civitai.red/x",
|
||||
"https://evil.example/civitai.red",
|
||||
"https://evil.example/?h=civitai.green",
|
||||
"https://red/x",
|
||||
"https://.civitai.red.evil/x",
|
||||
):
|
||||
self.assertFalse(lora_utils.is_civitai_redirect_target(url), url)
|
||||
|
||||
def test_a_redirect_to_red_or_green_is_followed_without_the_key(self):
|
||||
for blob in (RED_BLOB, GREEN_BLOB):
|
||||
with self.subTest(blob=blob):
|
||||
if self.target.exists():
|
||||
os.remove(self.target)
|
||||
result, requests = self._download(
|
||||
{START: (302, blob), blob: (200, None)}
|
||||
)
|
||||
|
||||
self.assertIsNotNone(result)
|
||||
self.assertEqual([r.full_url for r in requests], [START, blob])
|
||||
self.assertEqual(
|
||||
requests[0].get_header("Authorization"), "Bearer secret-key"
|
||||
)
|
||||
self.assertIsNone(requests[1].get_header("Authorization"))
|
||||
|
||||
def test_lookalike_and_plain_http_redirects_are_refused(self):
|
||||
for location in (
|
||||
"https://civitai.red.evil.example/x",
|
||||
"https://evilcivitai.red/x",
|
||||
"https://civitai.redx/x",
|
||||
"https://civitai.green.evil.example/x",
|
||||
"https://evilcivitai.green/x",
|
||||
"https://civitai.greenx/x",
|
||||
"http://blobs-b2.civitai.red/x",
|
||||
"http://civitai.green/x",
|
||||
):
|
||||
with self.subTest(location=location):
|
||||
result, requests = self._download({START: (302, location)})
|
||||
self.assertIsNone(result)
|
||||
self.assertEqual([r.full_url for r in requests], [START])
|
||||
self.assertFalse(self.target.exists())
|
||||
|
||||
def test_a_red_starting_host_gets_the_key_and_its_redirect_does_not(self):
|
||||
start = "https://image.civitai.red/xG1/abc/width=450/1.jpeg"
|
||||
result, requests = self._download(
|
||||
{start: (302, BLOB), BLOB: (200, None)}, url=start
|
||||
)
|
||||
|
||||
self.assertIsNotNone(result)
|
||||
self.assertEqual([r.full_url for r in requests], [start, BLOB])
|
||||
self.assertEqual(requests[0].get_header("Authorization"), "Bearer secret-key")
|
||||
self.assertIsNone(requests[1].get_header("Authorization"))
|
||||
|
||||
def test_a_green_starting_host_downloads_directly(self):
|
||||
start = "https://civitai.green/api/download/1"
|
||||
result, requests = self._download({start: (200, None)}, url=start)
|
||||
|
||||
self.assertIsNotNone(result)
|
||||
self.assertEqual(requests[0].get_header("Authorization"), "Bearer secret-key")
|
||||
|
||||
def test_the_hop_limit_spans_domains(self):
|
||||
hops = [RED_BLOB, GREEN_BLOB, BLOB, "https://civitai.red/4"]
|
||||
routes = {START: (302, hops[0])}
|
||||
for here, there in zip(hops, hops[1:]):
|
||||
routes[here] = (302, there)
|
||||
routes[hops[-1]] = (200, None)
|
||||
|
||||
result, requests = self._download(routes)
|
||||
|
||||
self.assertIsNone(result)
|
||||
self.assertEqual(len(requests), lora_utils.MAX_CIVITAI_REDIRECTS + 1)
|
||||
self.assertNotIn(hops[-1], [r.full_url for r in requests])
|
||||
for req in requests[1:]:
|
||||
self.assertIsNone(req.get_header("Authorization"))
|
||||
|
||||
|
||||
class _RaisingOpener:
|
||||
"""An ``_open_url`` stand-in that raises ``error`` for every request."""
|
||||
|
||||
def __init__(self, error):
|
||||
self.error = error
|
||||
self.requests = []
|
||||
|
||||
def __call__(self, req, timeout=None):
|
||||
self.requests.append(req)
|
||||
raise self.error
|
||||
|
||||
|
||||
class TestDownloadFailureWarnings(unittest.TestCase):
|
||||
"""#340: each failed download logs one warning line with a short reason,
|
||||
the URL without its query or credentials, and never the key."""
|
||||
|
||||
LOGGER = "prompt_manager.lora_utils"
|
||||
SIGNED = START + "?token=t0k3n"
|
||||
|
||||
def setUp(self):
|
||||
self.tmpdir = Path(tempfile.mkdtemp())
|
||||
self.addCleanup(shutil.rmtree, self.tmpdir, True)
|
||||
self.target = self.tmpdir / "x.jpg"
|
||||
|
||||
def _warning(self, run):
|
||||
"""Run ``run`` and return its one WARNING-or-above record's text."""
|
||||
with self.assertLogs(self.LOGGER, level="WARNING") as logs:
|
||||
result = run()
|
||||
self.assertIsNone(result)
|
||||
self.assertFalse(self.target.exists())
|
||||
self.assertEqual(len(logs.records), 1, logs.output)
|
||||
record = logs.records[0]
|
||||
self.assertEqual(record.levelname, "WARNING")
|
||||
text = record.getMessage()
|
||||
self.assertNotIn("secret-key", text)
|
||||
self.assertNotIn("t0k3n", text)
|
||||
self.assertNotIn("sig123", text)
|
||||
self.assertIn(START.split("?")[0], text)
|
||||
return text
|
||||
|
||||
def _chain(self, routes, url=SIGNED):
|
||||
handler = _ChainHTTPSHandler(routes)
|
||||
opener = lora_utils._build_opener(handler)
|
||||
|
||||
def run():
|
||||
with patch.object(lora_utils, "_opener", opener):
|
||||
return lora_utils._download_one(url, self.target, "secret-key")
|
||||
|
||||
return run
|
||||
|
||||
def _fake(self, opener, url=SIGNED):
|
||||
def run():
|
||||
with patch.object(lora_utils, "_open_url", opener):
|
||||
return lora_utils._download_one(url, self.target, "secret-key")
|
||||
|
||||
return run
|
||||
|
||||
def test_http_status(self):
|
||||
text = self._warning(self._chain({})) # unknown URL answers 404
|
||||
self.assertIn("HTTP 404", text)
|
||||
|
||||
def test_http_status_after_a_redirect(self):
|
||||
text = self._warning(self._chain({self.SIGNED: (302, BLOB)}))
|
||||
self.assertIn("HTTP 404", text)
|
||||
|
||||
def test_refused_redirect(self):
|
||||
text = self._warning(
|
||||
self._chain({self.SIGNED: (302, "https://evil.example/x?sig123=1")})
|
||||
)
|
||||
self.assertIn("redirect refused", text)
|
||||
self.assertIn("https://evil.example/x", text)
|
||||
|
||||
def test_hop_limit(self):
|
||||
text = self._warning(self._chain({self.SIGNED: (302, self.SIGNED)}))
|
||||
self.assertIn("more than 3 redirects", text)
|
||||
|
||||
def test_size_cap(self):
|
||||
opener = _FakeOpener(
|
||||
_FakeResponse(_png_bytes(), content_length=MAX_CIVITAI_DOWNLOAD_BYTES + 1)
|
||||
)
|
||||
text = self._warning(self._fake(opener))
|
||||
self.assertIn("larger than the 50 MB cap", text)
|
||||
|
||||
def test_decode_error(self):
|
||||
text = self._warning(self._fake(_FakeOpener(_FakeResponse(b"not an image"))))
|
||||
self.assertIn("image could not be decoded", text)
|
||||
|
||||
def test_timeout(self):
|
||||
for error in (
|
||||
TimeoutError("timed out"),
|
||||
urllib.error.URLError(TimeoutError("timed out")),
|
||||
):
|
||||
with self.subTest(error=type(error).__name__):
|
||||
text = self._warning(self._fake(_RaisingOpener(error)))
|
||||
self.assertIn("timed out", text)
|
||||
|
||||
def test_network_error(self):
|
||||
for error in (
|
||||
urllib.error.URLError("[Errno -2] Name or service not known"),
|
||||
ConnectionResetError(104, "Connection reset by peer"),
|
||||
):
|
||||
with self.subTest(error=type(error).__name__):
|
||||
text = self._warning(self._fake(_RaisingOpener(error)))
|
||||
self.assertIn("network error", text)
|
||||
|
||||
def test_a_timeout_while_reading_the_body(self):
|
||||
class SlowResponse(_FakeResponse):
|
||||
def read(self, amt=-1):
|
||||
raise TimeoutError("timed out")
|
||||
|
||||
text = self._warning(self._fake(_FakeOpener(SlowResponse(b""))))
|
||||
self.assertIn("timed out", text)
|
||||
|
||||
def test_a_failed_save(self):
|
||||
opener = _FakeOpener(_FakeResponse(_png_bytes()))
|
||||
|
||||
def run():
|
||||
with patch.object(
|
||||
lora_utils.os, "replace", side_effect=OSError(28, "No space left")
|
||||
):
|
||||
return self._fake(opener)()
|
||||
|
||||
text = self._warning(run)
|
||||
self.assertIn("could not be saved", text)
|
||||
|
||||
def test_a_refused_starting_url(self):
|
||||
def run():
|
||||
return lora_utils._download_one(
|
||||
"http://image.civitai.com/x?token=t0k3n", self.target, "secret-key"
|
||||
)
|
||||
|
||||
with self.assertLogs(self.LOGGER, level="WARNING") as logs:
|
||||
self.assertIsNone(run())
|
||||
self.assertEqual(len(logs.records), 1)
|
||||
text = logs.records[0].getMessage()
|
||||
self.assertIn("not an allowed CivitAI URL", text)
|
||||
self.assertNotIn("t0k3n", text)
|
||||
|
||||
def test_an_unexpected_error_names_only_its_type(self):
|
||||
error = ValueError("unknown url type: https://x?token=t0k3n")
|
||||
text = self._warning(self._fake(_RaisingOpener(error)))
|
||||
self.assertIn("unexpected ValueError", text)
|
||||
|
||||
def test_the_line_names_the_download(self):
|
||||
text = self._warning(self._chain({}))
|
||||
self.assertTrue(text.startswith("CivitAI image download failed"), text)
|
||||
|
||||
def test_a_successful_download_logs_no_warning(self):
|
||||
with self.assertNoLogs(self.LOGGER, level="WARNING"):
|
||||
result = self._fake(_FakeOpener(_FakeResponse(_png_bytes())))()
|
||||
self.assertIsNotNone(result)
|
||||
|
||||
|
||||
class TestDownloadCivitaiImages(unittest.TestCase):
|
||||
|
||||
@@ -817,9 +1422,95 @@ class TestDetectLoraManager(unittest.TestCase):
|
||||
self.assertTrue(lora_utils._looks_like_lora_manager(lm))
|
||||
|
||||
def test_nothing_found(self):
|
||||
with patch.object(lora_utils, "find_comfyui_root", return_value=None):
|
||||
with (
|
||||
patch.object(lora_utils, "find_comfyui_root", return_value=None),
|
||||
patch.object(lora_utils, "custom_nodes_directories", return_value=[]),
|
||||
):
|
||||
self.assertIsNone(lora_utils.detect_lora_manager())
|
||||
|
||||
def test_auto_detects_in_an_extra_custom_nodes_folder(self):
|
||||
# #144: ComfyUI's main custom_nodes has no LoraManager; an extra
|
||||
# custom_nodes path registered through folder_paths (extra model
|
||||
# paths, --base-directory) has it.
|
||||
root = _make_comfy_root(self.tmp)
|
||||
extra = self.tmp / "shared_nodes"
|
||||
extra.mkdir()
|
||||
lm = _make_lora_manager(extra, "ComfyUI-Lora-Manager")
|
||||
fake = types.SimpleNamespace(
|
||||
base_path=str(root),
|
||||
get_folder_paths=lambda name: (
|
||||
[str(root / "custom_nodes"), str(extra)]
|
||||
if name == "custom_nodes"
|
||||
else []
|
||||
),
|
||||
)
|
||||
with (
|
||||
patch.dict(sys.modules, {"folder_paths": fake}),
|
||||
patch.object(lora_utils, "find_comfyui_root", return_value=root),
|
||||
):
|
||||
self.assertEqual(lora_utils.detect_lora_manager(), str(lm.resolve()))
|
||||
|
||||
def test_main_custom_nodes_wins_over_an_extra_folder(self):
|
||||
root = _make_comfy_root(self.tmp)
|
||||
main = _make_lora_manager(root / "custom_nodes")
|
||||
extra = self.tmp / "shared_nodes"
|
||||
extra.mkdir()
|
||||
_make_lora_manager(extra)
|
||||
with (
|
||||
patch.object(lora_utils, "find_comfyui_root", return_value=root),
|
||||
patch.object(
|
||||
lora_utils,
|
||||
"custom_nodes_directories",
|
||||
return_value=[(root / "custom_nodes").resolve(), extra.resolve()],
|
||||
),
|
||||
):
|
||||
self.assertEqual(lora_utils.detect_lora_manager(), str(main.resolve()))
|
||||
|
||||
def test_an_entry_that_cannot_be_checked_is_skipped(self):
|
||||
# S10 review N5: a permission error on one entry, not a missing folder.
|
||||
root = _make_comfy_root(self.tmp)
|
||||
blocked = root / "custom_nodes" / "a-lora-manager"
|
||||
blocked.mkdir()
|
||||
lm = _make_lora_manager(root / "custom_nodes", "b-Lora-Manager")
|
||||
real_is_dir = Path.is_dir
|
||||
|
||||
def is_dir(path):
|
||||
if path == blocked:
|
||||
raise PermissionError(13, "Permission denied", str(path))
|
||||
return real_is_dir(path)
|
||||
|
||||
with (
|
||||
patch.object(lora_utils, "find_comfyui_root", return_value=root),
|
||||
patch.object(lora_utils, "custom_nodes_directories", return_value=[]),
|
||||
patch.object(Path, "is_dir", is_dir),
|
||||
):
|
||||
self.assertEqual(lora_utils.detect_lora_manager(), str(lm.resolve()))
|
||||
|
||||
def test_a_folder_that_cannot_be_listed_is_skipped(self):
|
||||
root = _make_comfy_root(self.tmp)
|
||||
with (
|
||||
patch.object(lora_utils, "find_comfyui_root", return_value=root),
|
||||
patch.object(lora_utils, "custom_nodes_directories", return_value=[]),
|
||||
patch.object(Path, "iterdir", side_effect=PermissionError(13, "denied")),
|
||||
):
|
||||
self.assertIsNone(lora_utils.detect_lora_manager())
|
||||
|
||||
def test_an_unreadable_extra_folder_is_skipped(self):
|
||||
root = _make_comfy_root(self.tmp)
|
||||
missing = self.tmp / "gone"
|
||||
extra = self.tmp / "shared_nodes"
|
||||
extra.mkdir()
|
||||
lm = _make_lora_manager(extra)
|
||||
with (
|
||||
patch.object(lora_utils, "find_comfyui_root", return_value=root),
|
||||
patch.object(
|
||||
lora_utils,
|
||||
"custom_nodes_directories",
|
||||
return_value=[missing, extra.resolve()],
|
||||
),
|
||||
):
|
||||
self.assertEqual(lora_utils.detect_lora_manager(), str(lm.resolve()))
|
||||
|
||||
|
||||
class TestFindLoraDirectories(unittest.TestCase):
|
||||
|
||||
|
||||
@@ -0,0 +1,662 @@
|
||||
"""#342: an image is linked only to a prompt that provably produced it.
|
||||
|
||||
The maintainer's run: an LLM writes the prompt into a PromptManagerPair's
|
||||
``prepend_text`` while its ``text`` editor is empty. Prepend and append text are
|
||||
never stored as the prompt (product decision), so nothing is saved, and the
|
||||
image must stay unlinked instead of taking the newest prompt in the database or
|
||||
a stale tracker entry.
|
||||
"""
|
||||
|
||||
import copy
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
try:
|
||||
from tests.open_handles import assert_closed
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from open_handles import assert_closed
|
||||
|
||||
from database.operations import PromptDatabase
|
||||
from prompt_manager import PromptManager
|
||||
from prompt_manager_pair import PromptManagerPair
|
||||
from prompt_manager_text import PromptManagerText
|
||||
from utils.hashing import generate_prompt_hash
|
||||
from utils.image_monitor import ImageGenerationHandler
|
||||
from utils.prompt_tracker import PromptTracker
|
||||
from utils.run_keys import image_run_keys, prompt_node_key
|
||||
from utils.usage_tracking import (
|
||||
PendingFirstUse,
|
||||
queued_pair_runs,
|
||||
queued_prompt_runs,
|
||||
)
|
||||
|
||||
LLM_TEXT = "Photorealistic aerial landscape of misty mountains at dawn"
|
||||
EVIDENCE_KEY = "comfyassets_prompt_pairs"
|
||||
|
||||
|
||||
def issue_graph(text="", seed=1):
|
||||
"""The workflow from #342 (subgraph ids kept), PromptManagerPair positive."""
|
||||
return {
|
||||
"18:35": {
|
||||
"class_type": "PromptManagerPair",
|
||||
"inputs": {
|
||||
"text": text,
|
||||
"negative_text": "",
|
||||
"prepend_text": ["18:13", 0],
|
||||
"append_text": "",
|
||||
"category": "",
|
||||
"tags": "",
|
||||
"clip": ["4", 1],
|
||||
},
|
||||
},
|
||||
"18:13": {"class_type": "PreviewAny", "inputs": {"source": ["18:16", 0]}},
|
||||
"18:16": {
|
||||
"class_type": "ComfySwitchNode",
|
||||
"inputs": {
|
||||
"switch": True,
|
||||
"on_false": ["18:17", 0],
|
||||
"on_true": ["18:11", 0],
|
||||
},
|
||||
},
|
||||
"18:11": {
|
||||
"class_type": "TextGenerate",
|
||||
"inputs": {"prompt": ["18:17", 0], "seed": 7},
|
||||
},
|
||||
"18:17": {
|
||||
"class_type": "PrimitiveStringMultiline",
|
||||
"inputs": {"value": "Describe an aerial mountain landscape"},
|
||||
},
|
||||
"18:5": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": {"text": "blurry", "clip": ["4", 1]},
|
||||
},
|
||||
"4": {
|
||||
"class_type": "CheckpointLoaderSimple",
|
||||
"inputs": {"ckpt_name": "moss.safetensors"},
|
||||
},
|
||||
"3": {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {
|
||||
"model": ["4", 0],
|
||||
"positive": ["18:35", 0],
|
||||
"negative": ["18:5", 0],
|
||||
"latent_image": ["6", 0],
|
||||
"seed": seed,
|
||||
},
|
||||
},
|
||||
"6": {"class_type": "EmptyLatentImage", "inputs": {"batch_size": 1}},
|
||||
}
|
||||
|
||||
|
||||
def llm_text_graph(seed=1):
|
||||
"""An LLM feeding a PromptManager's ``text``: stored, but not in the metadata."""
|
||||
graph = issue_graph(seed=seed)
|
||||
graph["18:35"] = {
|
||||
"class_type": "PromptManager",
|
||||
"inputs": {"text": ["18:13", 0], "clip": ["4", 1]},
|
||||
}
|
||||
return graph
|
||||
|
||||
|
||||
def pair_evidence(text="", prepend=LLM_TEXT):
|
||||
"""The execution record PromptManagerPair embeds in the saved image."""
|
||||
return {
|
||||
"version": 1,
|
||||
"nodes": {
|
||||
"18:35": {
|
||||
"records": [
|
||||
{
|
||||
"text": text,
|
||||
"negative_text": "",
|
||||
"positive_text": " ".join(p for p in (prepend, text) if p),
|
||||
"negative_effective_text": "",
|
||||
"prepend_text": prepend,
|
||||
"append_text": "",
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def make_tracker(db):
|
||||
"""A real tracker without its cleanup thread or config import."""
|
||||
with mock.patch.object(PromptTracker, "__init__", lambda self, db: None):
|
||||
tracker = PromptTracker.__new__(PromptTracker)
|
||||
tracker.logger = logging.getLogger("test.prompt_attribution.tracker")
|
||||
tracker.db_manager = db
|
||||
tracker._local = threading.local()
|
||||
tracker.active_prompts = {}
|
||||
tracker.lock = threading.Lock()
|
||||
tracker._prompt_queue = []
|
||||
tracker._queue_lock = threading.Lock()
|
||||
tracker.cleanup_interval = 300
|
||||
tracker.prompt_timeout = 600
|
||||
return tracker
|
||||
|
||||
|
||||
class DatabaseCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.tmp.cleanup)
|
||||
self.addCleanup(assert_closed, self, self.tmp.name)
|
||||
self.db = PromptDatabase(os.path.join(self.tmp.name, "prompts.db"))
|
||||
self.addCleanup(self.db.close_all)
|
||||
self.tracker = make_tracker(self.db)
|
||||
|
||||
def save(self, text):
|
||||
return self.db.save_prompt(text=text, prompt_hash=generate_prompt_hash(text))
|
||||
|
||||
|
||||
class NodeCase(DatabaseCase):
|
||||
def node(self, cls):
|
||||
node = cls.__new__(cls)
|
||||
node.db = self.db
|
||||
node.logger = logging.getLogger("test.prompt_attribution.node")
|
||||
node.prompt_tracker = self.tracker
|
||||
node.comfyui_integration = mock.Mock()
|
||||
node._inject_lora_trigger_words = lambda text: text
|
||||
node.pending_first_use = PendingFirstUse()
|
||||
node.pending_pair_first_use = PendingFirstUse()
|
||||
return node
|
||||
|
||||
def clip(self):
|
||||
clip = mock.Mock()
|
||||
clip.tokenize.side_effect = lambda text: text
|
||||
clip.encode_from_tokens_scheduled.side_effect = lambda t: ("cond", t)
|
||||
return clip
|
||||
|
||||
def count_rows(self, table):
|
||||
conn = self.db.model.get_connection()
|
||||
return conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]
|
||||
|
||||
|
||||
class TestEmptyEditorSavesNothing(NodeCase):
|
||||
"""Prepend/append are never the stored prompt; an empty editor saves nothing."""
|
||||
|
||||
def test_pair_with_only_prepend_text_saves_nothing_and_logs_info(self):
|
||||
graph = issue_graph()
|
||||
with self.assertLogs("test.prompt_attribution.node", "INFO") as logs:
|
||||
result = self.node(PromptManagerPair).encode_prompt(
|
||||
self.clip(),
|
||||
"",
|
||||
negative_text="",
|
||||
prepend_text=LLM_TEXT,
|
||||
prompt=graph,
|
||||
unique_id="18:35",
|
||||
)
|
||||
self.assertEqual(result[2], LLM_TEXT, "the composed text is still encoded")
|
||||
self.assertEqual(self.count_rows("prompts"), 0)
|
||||
self.assertEqual(self.count_rows("prompt_pairs"), 0)
|
||||
self.assertFalse(
|
||||
any(record.levelno >= logging.WARNING for record in logs.records),
|
||||
[record.getMessage() for record in logs.records],
|
||||
)
|
||||
self.assertTrue(any("Prompt text is empty" in line for line in logs.output))
|
||||
|
||||
def test_prompt_manager_and_text_nodes_with_only_prepend_save_nothing(self):
|
||||
graph = issue_graph()
|
||||
graph["18:35"] = {
|
||||
"class_type": "PromptManager",
|
||||
"inputs": {"text": "", "prepend_text": ["18:13", 0], "clip": ["4", 1]},
|
||||
}
|
||||
self.node(PromptManager).encode_prompt(
|
||||
self.clip(), "", prepend_text=LLM_TEXT, prompt=graph, unique_id="18:35"
|
||||
)
|
||||
graph["18:35"]["class_type"] = "PromptManagerText"
|
||||
self.node(PromptManagerText).process_text(
|
||||
"", prepend_text=LLM_TEXT, prompt=graph, unique_id="18:35"
|
||||
)
|
||||
self.assertEqual(self.count_rows("prompts"), 0)
|
||||
|
||||
def test_empty_editor_marks_its_run_key_so_no_older_entry_is_used(self):
|
||||
graph = issue_graph()
|
||||
key = prompt_node_key(graph, "18:35")
|
||||
older = self.save("a woman holding a coffee cup")
|
||||
self.tracker.set_current_prompt(
|
||||
"a woman holding a coffee cup",
|
||||
{"prompt_id": older},
|
||||
push_to_queue=True,
|
||||
run_key=key,
|
||||
run_id="run-1",
|
||||
)
|
||||
self.node(PromptManagerPair).encode_prompt(
|
||||
self.clip(), "", prepend_text=LLM_TEXT, prompt=graph, unique_id="18:35"
|
||||
)
|
||||
latest, _run, _count = self.tracker.keyed_snapshot()[key]
|
||||
self.assertTrue(latest.get("placeholder"))
|
||||
self.assertIsNone(self.tracker.pop_for_key(key, "another run"))
|
||||
popped = self.tracker.pop_for_key(key)
|
||||
self.assertEqual(popped["id"], older, "an earlier queued item stays first")
|
||||
self.assertTrue(self.tracker.pop_for_key(key).get("placeholder"))
|
||||
|
||||
def test_empty_negative_node_is_not_marked(self):
|
||||
graph = issue_graph(text="a typed positive")
|
||||
graph["18:5"] = {
|
||||
"class_type": "PromptManager",
|
||||
"inputs": {"text": "", "clip": ["4", 1]},
|
||||
}
|
||||
self.node(PromptManager).encode_prompt(
|
||||
self.clip(), "", prompt=graph, unique_id="18:5"
|
||||
)
|
||||
self.assertEqual(self.tracker.keyed_snapshot(), {})
|
||||
|
||||
def test_queue_hook_counts_nothing_for_an_empty_editor(self):
|
||||
graph = issue_graph()
|
||||
self.assertEqual(queued_prompt_runs(graph), [])
|
||||
self.assertEqual(queued_pair_runs(graph), [])
|
||||
|
||||
def test_typed_text_with_prepend_still_stores_only_the_editor_text(self):
|
||||
graph = issue_graph(text="mountains")
|
||||
self.node(PromptManagerPair).encode_prompt(
|
||||
self.clip(),
|
||||
"mountains",
|
||||
prepend_text=LLM_TEXT,
|
||||
prompt=graph,
|
||||
unique_id="18:35",
|
||||
)
|
||||
self.assertIsNotNone(
|
||||
self.db.get_prompt_by_hash(generate_prompt_hash("mountains"))
|
||||
)
|
||||
self.assertIsNone(self.db.get_prompt_by_hash(generate_prompt_hash(LLM_TEXT)))
|
||||
context, _run, _count = self.tracker.keyed_snapshot()[
|
||||
prompt_node_key(graph, "18:35")
|
||||
]
|
||||
self.assertEqual(context["text"], f"{LLM_TEXT} mountains")
|
||||
|
||||
|
||||
class MonitorCase(DatabaseCase):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
fd, self.image = tempfile.mkstemp(suffix=".png", dir=self.tmp.name)
|
||||
os.close(fd)
|
||||
self.handler = ImageGenerationHandler(self.db, self.tracker)
|
||||
self.linked = []
|
||||
self.handler.link_image_to_prompt = lambda path, prompt, meta: (
|
||||
self.linked.append(prompt["id"])
|
||||
)
|
||||
|
||||
def process(self, metadata, snapshot=None):
|
||||
before = len(self.linked)
|
||||
with mock.patch.object(
|
||||
self.handler.metadata_extractor, "extract_metadata", return_value=metadata
|
||||
):
|
||||
self.handler.process_new_image(self.image, prompt_snapshot=snapshot)
|
||||
return self.linked[-1] if len(self.linked) > before else None
|
||||
|
||||
|
||||
class TestIssueReproduction(MonitorCase):
|
||||
def test_llm_prepend_image_is_left_unlinked_not_given_the_newest_prompt(self):
|
||||
"""Failing before #342: linked to the newest prompt in the database."""
|
||||
self.save("a woman holding a coffee cup")
|
||||
metadata = {"prompt": issue_graph(), EVIDENCE_KEY: pair_evidence()}
|
||||
with self.assertLogs("prompt_manager.image_monitor", "INFO") as logs:
|
||||
self.assertIsNone(self.process(metadata))
|
||||
self.assertTrue(any("left unlinked" in line for line in logs.output))
|
||||
|
||||
def test_a_stale_entry_of_another_workflow_is_never_used(self):
|
||||
"""Failing before #342: the live tracker's entry was taken."""
|
||||
stale = self.save("a woman holding a coffee cup")
|
||||
other = issue_graph(text="a woman holding a coffee cup")
|
||||
self.tracker.set_current_prompt(
|
||||
"a woman holding a coffee cup",
|
||||
{"prompt_id": stale},
|
||||
push_to_queue=True,
|
||||
run_key=prompt_node_key(other, "18:35"),
|
||||
run_id="run-0",
|
||||
)
|
||||
metadata = {"prompt": issue_graph(), EVIDENCE_KEY: pair_evidence()}
|
||||
self.assertIsNone(self.process(metadata, self.tracker.keyed_snapshot()))
|
||||
self.assertIsNone(self.process(metadata))
|
||||
self.assertEqual(len(self.tracker._prompt_queue), 1, "not consumed")
|
||||
|
||||
def test_an_entry_without_a_run_key_is_never_used(self):
|
||||
stale = self.save("previous run")
|
||||
self.tracker.set_current_prompt("previous run", {"prompt_id": stale})
|
||||
self.assertIsNone(self.process({"prompt": llm_text_graph()}))
|
||||
|
||||
def test_an_image_without_metadata_is_left_unlinked(self):
|
||||
"""Failing before #342: popped the queue, else took the newest prompt."""
|
||||
self.save("latest")
|
||||
self.assertIsNone(self.process(None))
|
||||
|
||||
def test_the_recency_fallback_is_gone(self):
|
||||
self.assertFalse(hasattr(self.handler, "_get_fallback_prompt"))
|
||||
|
||||
|
||||
class TestProvenRuns(MonitorCase):
|
||||
"""Entries the image's own graph proves still link (LLM into ``text``)."""
|
||||
|
||||
def track(self, graph, text, run_id="run-1", push=True):
|
||||
prompt_id = (
|
||||
self.save(text)
|
||||
if not self.db.get_prompt_by_hash(generate_prompt_hash(text))
|
||||
else self.db.get_prompt_by_hash(generate_prompt_hash(text))["id"]
|
||||
)
|
||||
self.tracker.set_current_prompt(
|
||||
text,
|
||||
{"prompt_id": prompt_id},
|
||||
push_to_queue=push,
|
||||
run_key=prompt_node_key(graph, "18:35"),
|
||||
run_id=run_id,
|
||||
)
|
||||
return prompt_id
|
||||
|
||||
def test_queued_entry_of_the_same_run_key_links(self):
|
||||
graph = llm_text_graph()
|
||||
prompt_id = self.track(graph, LLM_TEXT)
|
||||
self.assertEqual(self.process({"prompt": graph}), prompt_id)
|
||||
|
||||
def test_cached_node_rerun_links_through_the_snapshot(self):
|
||||
graph = llm_text_graph(seed=1)
|
||||
prompt_id = self.track(graph, LLM_TEXT)
|
||||
self.assertEqual(self.process({"prompt": graph}), prompt_id)
|
||||
# Only the sampler seed changed: the node was cached and did not run,
|
||||
# but its upstream (the run key) is unchanged.
|
||||
rerun = llm_text_graph(seed=2)
|
||||
self.assertEqual(
|
||||
prompt_node_key(rerun, "18:35"), prompt_node_key(graph, "18:35")
|
||||
)
|
||||
self.assertEqual(
|
||||
self.process({"prompt": rerun}, self.tracker.keyed_snapshot()), prompt_id
|
||||
)
|
||||
|
||||
def test_snapshot_needs_a_known_run(self):
|
||||
graph = llm_text_graph()
|
||||
self.track(graph, LLM_TEXT, run_id=None, push=False)
|
||||
self.assertIsNone(self.process({"prompt": graph}))
|
||||
|
||||
def test_batch_items_link_in_order_and_the_latest_alone_proves_nothing(self):
|
||||
graph = llm_text_graph()
|
||||
first = self.track(graph, "item one")
|
||||
second = self.track(graph, "item two")
|
||||
self.assertEqual(self.process({"prompt": graph}), first)
|
||||
self.assertEqual(self.process({"prompt": graph}), second)
|
||||
# Queue drained: two entries in one run, so the latest is not proof
|
||||
self.assertIsNone(self.process({"prompt": graph}))
|
||||
|
||||
def test_an_empty_batch_item_keeps_the_queue_aligned(self):
|
||||
graph = llm_text_graph()
|
||||
key = prompt_node_key(graph, "18:35")
|
||||
first = self.track(graph, "item one")
|
||||
self.tracker.mark_empty(key, "run-1", push_to_queue=True)
|
||||
third = self.track(graph, "item three")
|
||||
self.assertEqual(self.process({"prompt": graph}), first)
|
||||
self.assertIsNone(self.process({"prompt": graph}))
|
||||
self.assertEqual(self.process({"prompt": graph}), third)
|
||||
|
||||
def test_a_newer_run_drops_stale_entries_of_the_same_key(self):
|
||||
graph = llm_text_graph()
|
||||
# run-1 saved no image, so its entry was never taken
|
||||
self.track(graph, "earlier run text", run_id="run-1")
|
||||
newer = self.track(graph, "newer run text", run_id="run-2")
|
||||
self.assertEqual(self.process({"prompt": graph}), newer)
|
||||
self.assertEqual(self.tracker._prompt_queue, [])
|
||||
|
||||
def test_a_later_run_queued_before_processing_keeps_its_own_entry(self):
|
||||
# The image appears (snapshot), the next queued run executes the same
|
||||
# node again before the monitor's delay is over, then the image is
|
||||
# processed: it takes its own run's entry, the later run keeps its own.
|
||||
graph = llm_text_graph()
|
||||
own = self.track(graph, "first run text", run_id="run-1")
|
||||
snapshot = self.tracker.keyed_snapshot()
|
||||
later = self.track(graph, "second run text", run_id="run-2")
|
||||
self.assertEqual(self.process({"prompt": graph}, snapshot), own)
|
||||
self.assertEqual(self.process({"prompt": graph}), later)
|
||||
|
||||
def test_no_entry_when_the_image_appeared_means_no_queue_match(self):
|
||||
graph = llm_text_graph()
|
||||
empty_snapshot = self.tracker.keyed_snapshot()
|
||||
self.track(graph, "queued after the image appeared")
|
||||
self.assertIsNone(self.process({"prompt": graph}, empty_snapshot))
|
||||
self.assertEqual(len(self.tracker._prompt_queue), 1)
|
||||
|
||||
def test_hosts_without_run_ids_still_link_queued_entries_in_order(self):
|
||||
graph = llm_text_graph()
|
||||
first = self.track(graph, "item one", run_id=None)
|
||||
second = self.track(graph, "item two", run_id=None)
|
||||
self.assertEqual(self.process({"prompt": graph}), first)
|
||||
self.assertEqual(self.process({"prompt": graph}), second)
|
||||
self.assertIsNone(self.process({"prompt": graph}))
|
||||
|
||||
def test_other_keys_stay_queued_for_their_own_images(self):
|
||||
mine, other = llm_text_graph(), copy.deepcopy(llm_text_graph())
|
||||
other["18:17"]["inputs"]["value"] = "Describe a beach"
|
||||
theirs = self.track(other, "a beach")
|
||||
prompt_id = self.track(mine, LLM_TEXT)
|
||||
self.assertEqual(self.process({"prompt": mine}), prompt_id)
|
||||
self.assertEqual(self.process({"prompt": other}), theirs)
|
||||
|
||||
def test_two_positive_prompt_nodes_are_not_guessed(self):
|
||||
graph = llm_text_graph()
|
||||
self.track(graph, LLM_TEXT)
|
||||
graph["30"] = {
|
||||
"class_type": "PromptManager",
|
||||
"inputs": {"text": ["18:13", 0], "clip": ["4", 1]},
|
||||
}
|
||||
graph["31"] = dict(graph["3"])
|
||||
graph["31"]["inputs"] = {**graph["3"]["inputs"], "positive": ["30", 0]}
|
||||
self.assertEqual(len(image_run_keys({"prompt": graph})), 2)
|
||||
self.assertIsNone(self.process({"prompt": graph}))
|
||||
|
||||
def test_metadata_still_wins_and_leaves_the_queue_alone(self):
|
||||
typed = self.save("a typed prompt")
|
||||
graph = issue_graph(text="a typed prompt")
|
||||
self.tracker.set_current_prompt(
|
||||
"x",
|
||||
{"prompt_id": typed},
|
||||
run_key=prompt_node_key(graph, "18:35"),
|
||||
run_id="r",
|
||||
)
|
||||
self.assertEqual(self.process({"prompt": graph}), typed)
|
||||
self.assertEqual(len(self.tracker._prompt_queue), 1)
|
||||
|
||||
|
||||
class TestRunKeys(unittest.TestCase):
|
||||
def test_key_covers_the_upstream_and_ignores_downstream_and_titles(self):
|
||||
graph = issue_graph()
|
||||
key = prompt_node_key(graph, "18:35")
|
||||
changed = copy.deepcopy(graph)
|
||||
changed["3"]["inputs"]["seed"] = 99
|
||||
changed["18:35"]["_meta"] = {"title": "renamed"}
|
||||
self.assertEqual(prompt_node_key(changed, "18:35"), key)
|
||||
changed["18:11"]["inputs"]["seed"] = 8
|
||||
self.assertNotEqual(prompt_node_key(changed, "18:35"), key)
|
||||
|
||||
def test_unknowable_keys_are_none(self):
|
||||
graph = issue_graph()
|
||||
self.assertIsNone(prompt_node_key(None, "1"))
|
||||
self.assertIsNone(prompt_node_key(graph, None))
|
||||
self.assertIsNone(prompt_node_key(graph, "missing"))
|
||||
broken = copy.deepcopy(graph)
|
||||
broken["18:13"]["inputs"]["source"] = ["gone", 0]
|
||||
self.assertIsNone(prompt_node_key(broken, "18:35"))
|
||||
broken = copy.deepcopy(graph)
|
||||
broken["18:17"]["inputs"]["value"] = object()
|
||||
self.assertIsNone(prompt_node_key(broken, "18:35"))
|
||||
|
||||
def test_image_keys_read_string_graphs_and_reject_garbage(self):
|
||||
import json
|
||||
|
||||
graph = issue_graph()
|
||||
self.assertEqual(
|
||||
image_run_keys({"prompt": json.dumps(graph)}),
|
||||
{"18:35": prompt_node_key(graph, "18:35")},
|
||||
)
|
||||
self.assertEqual(image_run_keys({"prompt": "{not json"}), {})
|
||||
self.assertEqual(image_run_keys(None), {})
|
||||
self.assertEqual(image_run_keys({"prompt": []}), {})
|
||||
|
||||
def test_cyclic_and_huge_closures_have_no_key(self):
|
||||
from utils.prompt_graph import MAX_VISITS
|
||||
|
||||
cyclic = {
|
||||
"1": {"class_type": "A", "inputs": {"x": ["2", 0]}},
|
||||
"2": {"class_type": "B", "inputs": {"x": ["1", 0]}},
|
||||
}
|
||||
self.assertIsNotNone(prompt_node_key(cyclic, "1"))
|
||||
chain = {
|
||||
str(i): {"class_type": "N", "inputs": {"x": [str(i + 1), 0]}}
|
||||
for i in range(MAX_VISITS + 1)
|
||||
}
|
||||
chain[str(MAX_VISITS + 1)] = {"class_type": "N", "inputs": {}}
|
||||
self.assertIsNone(prompt_node_key(chain, "0"))
|
||||
|
||||
|
||||
class TestReviewFindings(MonitorCase, NodeCase):
|
||||
"""Final-review findings M1-M4 (#344)."""
|
||||
|
||||
def test_m1_a_positive_node_without_a_key_makes_the_image_unkeyable(self):
|
||||
graph = llm_text_graph()
|
||||
graph["30"] = {
|
||||
"class_type": "PromptManager",
|
||||
"inputs": {"text": ["missing", 0], "clip": ["4", 1]},
|
||||
}
|
||||
graph["31"] = {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {**graph["3"]["inputs"], "positive": ["30", 0]},
|
||||
}
|
||||
self.assertIsNone(prompt_node_key(graph, "30"))
|
||||
self.assertEqual(image_run_keys({"prompt": graph}), {})
|
||||
prompt_id = self.save(LLM_TEXT)
|
||||
self.tracker.set_current_prompt(
|
||||
LLM_TEXT,
|
||||
{"prompt_id": prompt_id},
|
||||
run_key=prompt_node_key(graph, "18:35"),
|
||||
run_id="run-1",
|
||||
)
|
||||
self.assertIsNone(self.process({"prompt": graph}))
|
||||
|
||||
def test_m2_links_are_exactly_comfyui_is_link(self):
|
||||
from utils.run_keys import _links
|
||||
|
||||
inputs = {
|
||||
"string_slot_int": ["7", 0],
|
||||
"string_slot_float": ["7", 0.0],
|
||||
"string_slot_bool": ["7", True], # bool is an int to ComfyUI too
|
||||
"int_source": [7, 0],
|
||||
"tuple": ("7", 0),
|
||||
"three": ["7", 0, 1],
|
||||
"slot_text": ["7", "0"],
|
||||
}
|
||||
self.assertEqual(sorted(_links(inputs)), ["7", "7", "7"])
|
||||
widget = issue_graph()
|
||||
widget["18:17"]["inputs"]["size"] = [512, 768] # a value, not a link
|
||||
self.assertIsNotNone(prompt_node_key(widget, "18:35"))
|
||||
floated = issue_graph()
|
||||
floated["18:35"]["inputs"]["prepend_text"] = ["18:13", 0.0]
|
||||
changed = copy.deepcopy(floated)
|
||||
changed["18:17"]["inputs"]["value"] = "another instruction"
|
||||
self.assertNotEqual(
|
||||
prompt_node_key(floated, "18:35"), prompt_node_key(changed, "18:35")
|
||||
)
|
||||
|
||||
def _failing_save(self, node):
|
||||
def explode(*args, **kwargs):
|
||||
raise RuntimeError("database is locked")
|
||||
|
||||
node.db = mock.Mock(wraps=self.db)
|
||||
for name in ("save_prompt", "save_prompt_pair", "get_prompt_by_hash"):
|
||||
setattr(node.db, name, mock.Mock(side_effect=explode))
|
||||
return node
|
||||
|
||||
def test_m3_a_failed_save_takes_over_its_run_key(self):
|
||||
graph = llm_text_graph()
|
||||
key = prompt_node_key(graph, "18:35")
|
||||
older = self.save("previous run text")
|
||||
self.tracker.set_current_prompt(
|
||||
"previous run text", {"prompt_id": older}, run_key=key, run_id="run-1"
|
||||
)
|
||||
node = self._failing_save(self.node(PromptManager))
|
||||
with mock.patch("prompt_manager_base.executing_run_id", return_value="run-2"):
|
||||
node.encode_prompt(
|
||||
self.clip(), "new run text", prompt=graph, unique_id="18:35"
|
||||
)
|
||||
latest, run_id, _count = self.tracker.keyed_snapshot()[key]
|
||||
self.assertTrue(latest.get("placeholder"))
|
||||
self.assertEqual(run_id, "run-2")
|
||||
self.assertIsNone(self.process({"prompt": graph}))
|
||||
|
||||
def test_m3_a_failed_pair_save_takes_over_its_run_key(self):
|
||||
graph = issue_graph(text="typed")
|
||||
key = prompt_node_key(graph, "18:35")
|
||||
node = self._failing_save(self.node(PromptManagerPair))
|
||||
with self.assertLogs("test.prompt_attribution.node", "WARNING"):
|
||||
node.encode_prompt(self.clip(), "typed", prompt=graph, unique_id="18:35")
|
||||
self.assertTrue(self.tracker.keyed_snapshot()[key][0].get("placeholder"))
|
||||
|
||||
def test_m3_a_failed_text_node_save_takes_over_its_run_key(self):
|
||||
graph = issue_graph()
|
||||
graph["18:35"] = {
|
||||
"class_type": "PromptManagerText",
|
||||
"inputs": {"text": "typed"},
|
||||
}
|
||||
graph["40"] = {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": {"text": ["18:35", 0], "clip": ["4", 1]},
|
||||
}
|
||||
graph["3"]["inputs"]["positive"] = ["40", 0]
|
||||
node = self._failing_save(self.node(PromptManagerText))
|
||||
with mock.patch.object(
|
||||
node, "_track_prompt_execution", side_effect=RuntimeError("boom")
|
||||
):
|
||||
node.process_text("typed", prompt=graph, unique_id="18:35")
|
||||
key = prompt_node_key(graph, "18:35")
|
||||
self.assertTrue(self.tracker.keyed_snapshot()[key][0].get("placeholder"))
|
||||
|
||||
def test_m4_a_placeholder_never_falls_through_to_history(self):
|
||||
graph = llm_text_graph()
|
||||
key = prompt_node_key(graph, "18:35")
|
||||
self.tracker.mark_empty(key, "run-1", push_to_queue=False)
|
||||
with mock.patch.object(
|
||||
self.handler, "_history_prompt", return_value=99
|
||||
) as history:
|
||||
self.assertIsNone(self.process({"prompt": graph})) # snapshot form
|
||||
self.tracker.mark_empty(key, "run-2", push_to_queue=True)
|
||||
self.assertIsNone(self.process({"prompt": graph})) # popped form
|
||||
history.assert_not_called()
|
||||
|
||||
|
||||
class TestRereviewFindings(MonitorCase, NodeCase):
|
||||
"""Re-review findings R4 and R5 (#344)."""
|
||||
|
||||
def test_r4_a_logging_failure_after_tracking_never_doubles_a_queue_slot(self):
|
||||
graph = llm_text_graph()
|
||||
key = prompt_node_key(graph, "18:35")
|
||||
node = self.node(PromptManager)
|
||||
logger = mock.Mock(wraps=self.tracker.logger)
|
||||
logger.debug.side_effect = RuntimeError("log handler broke")
|
||||
self.tracker.logger = logger
|
||||
node.encode_prompt(self.clip(), LLM_TEXT, prompt=graph, unique_id="18:35")
|
||||
queued = [ctx for ctx in self.tracker._prompt_queue if ctx["run_key"] == key]
|
||||
self.assertEqual(len(queued), 1)
|
||||
|
||||
def test_r5_a_keyless_image_outside_the_output_folder_does_not_wait(self):
|
||||
outside = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: __import__("shutil").rmtree(outside, True))
|
||||
stray = os.path.join(outside, "copied.png")
|
||||
with open(stray, "wb") as handle:
|
||||
handle.write(b"png")
|
||||
with mock.patch(
|
||||
"utils.image_monitor.host_output_directory", return_value=self.tmp.name
|
||||
):
|
||||
with mock.patch.object(
|
||||
self.handler.metadata_extractor, "extract_metadata", return_value=None
|
||||
):
|
||||
self.handler.process_new_image(stray)
|
||||
self.handler.process_new_image(self.image) # inside: waits
|
||||
self.assertEqual(
|
||||
self.handler.awaiting_history(),
|
||||
[os.path.normcase(os.path.realpath(self.image))],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,599 @@
|
||||
"""Reviewed restore and verified backup routes (#249 System S8).
|
||||
|
||||
- ``POST /prompt_manager/restore/preview`` stages an upload and writes
|
||||
nothing live; ``DELETE .../preview/{token}`` discards it;
|
||||
``POST /prompt_manager/restore/apply`` starts the ``exclusive`` restore
|
||||
job that publishes exactly the reviewed bytes.
|
||||
- A restore and the health check, maintenance, Move library, a Needs review
|
||||
decision and a maintenance preview refuse each other or wait (carry-overs
|
||||
1 and 2).
|
||||
- ``GET /prompt_manager/backup`` verifies its snapshot before sending a byte;
|
||||
``GET /prompt_manager/backup/status`` lists the safety backups on disk and
|
||||
this session's verified download.
|
||||
|
||||
Ordering is driven by events and job tasks, never by timers.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import glob
|
||||
import hashlib
|
||||
import os
|
||||
import sqlite3
|
||||
import threading
|
||||
from contextlib import closing
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import aiohttp
|
||||
|
||||
from database import restore_stage
|
||||
from database.models import PromptModel
|
||||
from database.operations import PromptDatabase
|
||||
from py.api import restore_plans
|
||||
from py.api.backup_restore import RESTORE_JOB_TITLE
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
try:
|
||||
from tests.test_api_admin import AdminAPITestCase # noqa: E402
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from test_api_admin import AdminAPITestCase # noqa: E402
|
||||
|
||||
WAIT = 60 # a ceiling only a hung test reaches
|
||||
PREVIEW = "/prompt_manager/restore/preview"
|
||||
APPLY = "/prompt_manager/restore/apply"
|
||||
SUMMARY_KEYS = {
|
||||
"prompts",
|
||||
"positives",
|
||||
"pairs",
|
||||
"archived_pairs",
|
||||
"image_links",
|
||||
"runs",
|
||||
"saved_views",
|
||||
}
|
||||
|
||||
|
||||
def _texts(path):
|
||||
with closing(sqlite3.connect(path)) as conn:
|
||||
return sorted(r[0] for r in conn.execute("SELECT text FROM prompts"))
|
||||
|
||||
|
||||
class RestoreTestCase(AdminAPITestCase):
|
||||
@property
|
||||
def db_path(self):
|
||||
return os.path.join(self.tmpdir, "prompts.db")
|
||||
|
||||
def save(self, text, db=None):
|
||||
return (db or self.api.db).save_prompt(
|
||||
text=text, prompt_hash=generate_prompt_hash(text)
|
||||
)
|
||||
|
||||
def upload(self, texts, name="upload.db"):
|
||||
folder = os.path.join(self.tmpdir, "uploads")
|
||||
os.makedirs(folder, exist_ok=True)
|
||||
path = os.path.join(folder, name)
|
||||
db = PromptDatabase(path)
|
||||
for text in texts:
|
||||
self.save(text, db)
|
||||
db.close_all()
|
||||
return Path(path).read_bytes()
|
||||
|
||||
def form(self, data):
|
||||
form = aiohttp.FormData()
|
||||
form.add_field(
|
||||
"database_file",
|
||||
data,
|
||||
filename="backup.db",
|
||||
content_type="application/octet-stream",
|
||||
)
|
||||
return form
|
||||
|
||||
async def preview(self, data, status=200):
|
||||
resp = await self.client.post(PREVIEW, data=self.form(data))
|
||||
body = await resp.json()
|
||||
self.assertEqual(resp.status, status, body)
|
||||
return body
|
||||
|
||||
async def apply(self, token, status=202):
|
||||
resp = await self.client.post(APPLY, json={"token": token})
|
||||
body = await resp.json()
|
||||
self.assertEqual(resp.status, status, body)
|
||||
return body
|
||||
|
||||
async def finish(self, job_id):
|
||||
job = self.api._jobs.get(job_id)
|
||||
await asyncio.wait_for(asyncio.shield(job.task), WAIT)
|
||||
return job
|
||||
|
||||
def live_digest(self):
|
||||
return {
|
||||
suffix: hashlib.sha256(Path(self.db_path + suffix).read_bytes()).hexdigest()
|
||||
for suffix in ("", "-wal")
|
||||
if os.path.exists(self.db_path + suffix)
|
||||
}
|
||||
|
||||
def backups(self):
|
||||
return sorted(glob.glob(self.db_path + ".backup_*"))
|
||||
|
||||
def stages(self):
|
||||
return sorted(
|
||||
name
|
||||
for name in os.listdir(self.tmpdir)
|
||||
if name.startswith(restore_stage.STAGE_PREFIX)
|
||||
)
|
||||
|
||||
def gated_job(self, kind, admission, title=None):
|
||||
gate = asyncio.Event()
|
||||
|
||||
async def runner(job):
|
||||
await gate.wait()
|
||||
|
||||
job = self.api._jobs.start(
|
||||
kind, title or kind.title(), admission=admission, scope={}, runner=runner
|
||||
)
|
||||
self.addAsyncCleanup(self._release, job, gate)
|
||||
return job, gate
|
||||
|
||||
async def download_backup(self):
|
||||
"""GET /backup, read every byte, then wait for its hold to end.
|
||||
|
||||
The client can hold the last byte before the handler has finished:
|
||||
it records the download, then closes and removes its snapshot. So
|
||||
the state is read only once the snapshot file is gone.
|
||||
"""
|
||||
resp = await self.client.get("/prompt_manager/backup")
|
||||
data = await resp.read()
|
||||
while self.api._jobs.active("backup") is not None or any(
|
||||
name.startswith(".pm-safety-") for name in os.listdir(self.tmpdir)
|
||||
):
|
||||
await asyncio.sleep(0)
|
||||
return resp, data
|
||||
|
||||
async def _release(self, job, gate):
|
||||
gate.set()
|
||||
await asyncio.gather(job.task, return_exceptions=True)
|
||||
|
||||
|
||||
class TestPreview(RestoreTestCase):
|
||||
async def test_preview_never_replaces_or_backs_up(self):
|
||||
self.save("live one")
|
||||
self.api.db.model.get_connection()
|
||||
before = self.live_digest()
|
||||
body = await self.preview(self.upload(["restored a", "restored b"]))
|
||||
self.assertEqual(self.live_digest(), before)
|
||||
self.assertEqual(self.backups(), [])
|
||||
self.assertEqual(
|
||||
[p["text"] for p in self.api.db.search_prompts()], ["live one"]
|
||||
)
|
||||
self.assertEqual(set(body["summary"]), SUMMARY_KEYS)
|
||||
self.assertEqual(set(body["live"]), SUMMARY_KEYS)
|
||||
self.assertEqual((body["summary"]["prompts"], body["live"]["prompts"]), (2, 1))
|
||||
self.assertEqual(body["schema"], {"migrated": False, "library_state": True})
|
||||
self.assertEqual(body["integrity"], "ok")
|
||||
self.assertEqual(body["documents"], {"repaired": 0, "cleared": 0, "images": 0})
|
||||
self.assertEqual(
|
||||
body["safety_backup"],
|
||||
{"destination_label": "prompts.db.backup_<date and time>"},
|
||||
)
|
||||
self.assertNotIn(self.tmpdir, str(body))
|
||||
# Staged in the library's folder; the upload itself is gone.
|
||||
self.assertEqual(len(self.stages()), 1)
|
||||
self.assertEqual(
|
||||
[n for n in os.listdir(self.tmpdir) if "upload" in n and "pm-" in n], []
|
||||
)
|
||||
self.assertEqual(self.api._jobs.list(), [])
|
||||
|
||||
async def test_invalid_upload_is_400_and_leaves_nothing(self):
|
||||
resp = await self.client.post(PREVIEW, data=self.form(b"not a database"))
|
||||
body = await resp.json()
|
||||
self.assertEqual((resp.status, body["code"]), (400, "invalid-backup"))
|
||||
self.assertEqual(self.stages(), [])
|
||||
|
||||
async def test_expired_or_discarded_token_is_410(self):
|
||||
body = await self.preview(self.upload(["restored"]))
|
||||
resp = await self.client.delete(f"{PREVIEW}/{body['token']}")
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertEqual(self.stages(), [])
|
||||
refused = await self.apply(body["token"], status=410)
|
||||
self.assertEqual(refused["code"], "plan-expired")
|
||||
resp = await self.client.delete(f"{PREVIEW}/{body['token']}")
|
||||
self.assertEqual(resp.status, 410)
|
||||
|
||||
clock = [1000.0]
|
||||
self.api._restore_plan_store = restore_plans.RestorePlanStore(
|
||||
clock=lambda: clock[0]
|
||||
)
|
||||
body = await self.preview(self.upload(["restored"], "second.db"))
|
||||
clock[0] += restore_plans.STAGE_TTL_SECONDS + 1
|
||||
refused = await self.apply(body["token"], status=410)
|
||||
self.assertEqual(refused["code"], "plan-expired")
|
||||
self.assertEqual(self.stages(), [])
|
||||
|
||||
async def test_stage_budget_429(self):
|
||||
data = self.upload(["restored"])
|
||||
first = await self.preview(data)
|
||||
await self.preview(data)
|
||||
refused = await self.preview(data, status=429)
|
||||
self.assertEqual(refused["code"], "too-large")
|
||||
self.assertEqual(len(self.stages()), 2)
|
||||
await self.client.delete(f"{PREVIEW}/{first['token']}")
|
||||
await self.preview(data)
|
||||
|
||||
async def test_preview_reports_repaired_documents(self):
|
||||
upload = os.path.join(self.tmpdir, "uploads", "damaged.db")
|
||||
os.makedirs(os.path.dirname(upload), exist_ok=True)
|
||||
src = PromptDatabase(upload)
|
||||
prompt_id = self.save("with a workflow", src)
|
||||
image = src.link_image_to_prompt(
|
||||
prompt_id, os.path.join(self.tmpdir, "a.png"), {"workflow": {"v": 1}}
|
||||
)
|
||||
conn = src.model.get_connection()
|
||||
with conn:
|
||||
conn.execute(
|
||||
"UPDATE workflow_documents SET data=x'00' WHERE id=(SELECT "
|
||||
"workflow_document_id FROM generated_images WHERE id=?)",
|
||||
(image,),
|
||||
)
|
||||
src.close_all()
|
||||
body = await self.preview(Path(upload).read_bytes())
|
||||
self.assertEqual(body["documents"], {"repaired": 0, "cleared": 1, "images": 1})
|
||||
|
||||
|
||||
class TestApply(RestoreTestCase):
|
||||
async def test_apply_restores_and_logs(self):
|
||||
self.save("old")
|
||||
old_db = self.api.db
|
||||
body = await self.preview(self.upload(["restored a", "restored b"]))
|
||||
started = await self.apply(body["token"])
|
||||
summary = started["job"]
|
||||
self.assertEqual(
|
||||
(summary["kind"], summary["title"], summary["cancellable"]),
|
||||
("restore", RESTORE_JOB_TITLE, False),
|
||||
)
|
||||
self.assertEqual(summary["admission"], "exclusive")
|
||||
job = await self.finish(summary["id"])
|
||||
self.assertEqual(job.state, "finished", job.log)
|
||||
self.assertIsNot(self.api.db, old_db)
|
||||
self.assertEqual(_texts(self.db_path), ["restored a", "restored b"])
|
||||
[backup] = self.backups()
|
||||
self.assertEqual(_texts(backup), ["old"])
|
||||
log = "\n".join(job.log)
|
||||
self.assertIn(f"Safety backup {os.path.basename(backup)}", log)
|
||||
self.assertIn("Restored 2 prompts", log)
|
||||
self.assertNotIn(self.tmpdir, log)
|
||||
self.assertEqual(self.stages(), [])
|
||||
wal = self.db_path + "-wal"
|
||||
self.assertEqual(os.path.getsize(wal) if os.path.exists(wal) else 0, 0)
|
||||
used = await self.apply(body["token"], status=409)
|
||||
self.assertEqual(used["code"], "plan-used")
|
||||
self.assertEqual(used["job"]["id"], summary["id"])
|
||||
|
||||
async def test_apply_binds_exact_staged_bytes(self):
|
||||
self.save("live")
|
||||
body = await self.preview(self.upload(["restored"]))
|
||||
held = self.api._restore_plans.peek(body["token"])
|
||||
with closing(sqlite3.connect(held.staged.path)) as conn:
|
||||
conn.execute("INSERT INTO prompts (text, hash) VALUES ('sneaked', 'x')")
|
||||
conn.commit()
|
||||
started = await self.apply(body["token"])
|
||||
job = await self.finish(started["job"]["id"])
|
||||
self.assertEqual(job.state, "failed")
|
||||
self.assertEqual(job.last_event["code"], "plan-changed")
|
||||
self.assertEqual(_texts(self.db_path), ["live"])
|
||||
self.assertEqual(self.backups(), [])
|
||||
self.assertEqual(self.stages(), [])
|
||||
|
||||
async def test_live_change_after_preview_is_409(self):
|
||||
self.save("live")
|
||||
body = await self.preview(self.upload(["restored"]))
|
||||
self.save("written after the review")
|
||||
refused = await self.apply(body["token"], status=409)
|
||||
self.assertEqual(refused["code"], "plan-changed")
|
||||
self.assertEqual(
|
||||
sorted(p["text"] for p in self.api.db.search_prompts()),
|
||||
["live", "written after the review"],
|
||||
)
|
||||
self.assertEqual(self.stages(), [])
|
||||
self.assertEqual(self.api._jobs.list(), [])
|
||||
|
||||
async def test_publish_failure_keeps_safety_backup_and_reports(self):
|
||||
self.save("live")
|
||||
body = await self.preview(self.upload(["restored"]))
|
||||
failure = sqlite3.OperationalError("injected page copy failure")
|
||||
with patch.object(PromptModel, "_replace_contents_from", side_effect=failure):
|
||||
started = await self.apply(body["token"])
|
||||
job = await self.finish(started["job"]["id"])
|
||||
self.assertEqual(job.state, "failed")
|
||||
self.assertEqual(job.last_event["code"], "restore-failed")
|
||||
self.assertIn("The safety backup was kept.", job.status_line)
|
||||
[backup] = self.backups()
|
||||
self.assertEqual(_texts(backup), ["live"])
|
||||
self.assertEqual([p["text"] for p in self.api.db.search_prompts()], ["live"])
|
||||
self.assertEqual(self.stages(), [])
|
||||
|
||||
async def test_a_restore_drops_previews_of_the_old_library(self):
|
||||
first = await self.preview(self.upload(["first"]))
|
||||
other = await self.preview(self.upload(["other"], "other.db"))
|
||||
job = await self.finish((await self.apply(first["token"]))["job"]["id"])
|
||||
self.assertEqual(job.state, "finished", job.log)
|
||||
self.assertEqual(self.stages(), [])
|
||||
refused = await self.apply(other["token"], status=410)
|
||||
self.assertEqual(refused["code"], "plan-expired")
|
||||
|
||||
async def test_jobs_survive_db_swap(self):
|
||||
earlier, gate = self.gated_job("health", "read", "Health check")
|
||||
gate.set()
|
||||
await earlier.task
|
||||
body = await self.preview(self.upload(["restored"]))
|
||||
job = await self.finish((await self.apply(body["token"]))["job"]["id"])
|
||||
self.assertEqual(job.state, "finished")
|
||||
listed = (await (await self.client.get("/prompt_manager/jobs")).json())["jobs"]
|
||||
self.assertEqual(
|
||||
[(j["kind"], j["state"]) for j in listed],
|
||||
[("restore", "finished"), ("health", "finished")],
|
||||
)
|
||||
|
||||
|
||||
class TestRestoreAdmission(RestoreTestCase):
|
||||
"""A restore never interleaves with other library work (carry-overs 1, 2)."""
|
||||
|
||||
async def test_restore_job_refused_while_maintenance_runs(self):
|
||||
self.gated_job("maintenance", "records", "Duplicate prompt text")
|
||||
body = await self.preview(self.upload(["restored"]))
|
||||
refused = await self.apply(body["token"], status=409)
|
||||
self.assertEqual(refused["code"], "busy")
|
||||
self.assertEqual(refused["running"]["title"], "Duplicate prompt text")
|
||||
self.api._restore_plans.peek(body["token"]) # the token stays usable
|
||||
|
||||
async def test_restore_and_health_check_refuse_each_other(self):
|
||||
health, gate = self.gated_job("health", "read", "Health check")
|
||||
body = await self.preview(self.upload(["restored"]))
|
||||
refused = await self.apply(body["token"], status=409)
|
||||
self.assertEqual(refused["running"]["title"], "Health check")
|
||||
legacy = await self.client.post(
|
||||
"/prompt_manager/restore", data=self.form(self.upload(["x"], "x.db"))
|
||||
)
|
||||
self.assertEqual(legacy.status, 409)
|
||||
gate.set()
|
||||
await health.task
|
||||
|
||||
restore, release = self.gated_job("restore", "exclusive", RESTORE_JOB_TITLE)
|
||||
resp = await self.client.post("/prompt_manager/system/health")
|
||||
refusal = await resp.json()
|
||||
self.assertEqual((resp.status, refusal["code"]), (409, "busy"))
|
||||
self.assertEqual(refusal["running"]["title"], RESTORE_JOB_TITLE)
|
||||
|
||||
async def test_restore_and_move_library_refuse_each_other(self):
|
||||
self.gated_job("library_move", "exclusive", "Move library")
|
||||
body = await self.preview(self.upload(["restored"]), status=409)
|
||||
self.assertEqual(body["running"]["title"], "Move library")
|
||||
|
||||
self.api._jobs.list()[0].finish()
|
||||
body = await self.preview(self.upload(["restored"]))
|
||||
self.gated_job("restore", "exclusive", RESTORE_JOB_TITLE)
|
||||
resp = await self.client.post("/prompt_manager/system/library/move")
|
||||
self.assertEqual(resp.status, 409)
|
||||
self.assertEqual((await resp.json())["running"]["title"], RESTORE_JOB_TITLE)
|
||||
|
||||
async def test_decision_and_preview_refused_while_restore_runs(self):
|
||||
self.gated_job("restore", "exclusive", RESTORE_JOB_TITLE)
|
||||
for url, payload in (
|
||||
("/prompt_manager/review/items/1/decision", {"choice": "unresolved"}),
|
||||
("/prompt_manager/maintenance/preview", {"operation": "optimize"}),
|
||||
):
|
||||
resp = await self.client.post(url, json=payload)
|
||||
body = await resp.json()
|
||||
self.assertEqual((resp.status, body["code"]), (409, "busy"), url)
|
||||
self.assertEqual(body["running"]["title"], RESTORE_JOB_TITLE)
|
||||
resp = await self.client.post(PREVIEW, data=self.form(self.upload(["r"])))
|
||||
self.assertEqual(resp.status, 409)
|
||||
|
||||
async def test_restore_waits_for_a_running_decision(self):
|
||||
self.save("live")
|
||||
body = await self.preview(self.upload(["restored"]))
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
def slow_item(item_id):
|
||||
entered.set()
|
||||
release.wait(WAIT)
|
||||
return None
|
||||
|
||||
with patch.object(self.api.db, "review_item", side_effect=slow_item):
|
||||
decision = asyncio.ensure_future(
|
||||
self.client.post(
|
||||
"/prompt_manager/review/items/1/decision",
|
||||
json={
|
||||
"expected_epoch": "00000000-0000-4000-8000-000000000000",
|
||||
"expected_revision": 0,
|
||||
"capture_token": None,
|
||||
"sha256": "0" * 64,
|
||||
"dependency_fingerprint": "0" * 64,
|
||||
"choice": "unresolved",
|
||||
},
|
||||
)
|
||||
)
|
||||
self.assertTrue(await loop.run_in_executor(None, entered.wait, WAIT))
|
||||
self.assertEqual(self.api._jobs.leases, {"Needs review decision": 1})
|
||||
# The apply's own check takes a lease too: it shares with the decision.
|
||||
started = await self.apply(body["token"])
|
||||
job = self.api._jobs.get(started["job"]["id"])
|
||||
while job.status_line != "Waiting for other requests to finish":
|
||||
await asyncio.sleep(0)
|
||||
self.assertEqual(_texts(self.db_path), ["live"])
|
||||
release.set()
|
||||
answered = await decision
|
||||
self.assertNotEqual(answered.status, 409)
|
||||
await self.finish(job.id)
|
||||
self.assertEqual(job.state, "finished", job.log)
|
||||
self.assertEqual(_texts(self.db_path), ["restored"])
|
||||
|
||||
|
||||
class TestWorkerHandles(RestoreTestCase):
|
||||
"""Carry-over 3: no worker holds the library open across a switch."""
|
||||
|
||||
async def test_preview_workers_pause_and_library_connections_are_released(self):
|
||||
restore, _gate = self.gated_job("restore", "exclusive", RESTORE_JOB_TITLE)
|
||||
# Automatic previews (thumbnail workers) write nothing while it runs;
|
||||
# they never open the library file at all.
|
||||
self.assertIs(self.api._auto_thumbnail_blocker(), restore)
|
||||
|
||||
async def test_a_restore_leaves_no_handle_on_the_stage_or_upload(self):
|
||||
from tests.open_handles import open_handles_under
|
||||
|
||||
self.save("live")
|
||||
body = await self.preview(self.upload(["restored"]))
|
||||
job = await self.finish((await self.apply(body["token"]))["job"]["id"])
|
||||
self.assertEqual(job.state, "finished", job.log)
|
||||
leftovers = [
|
||||
hit
|
||||
for hit in open_handles_under(self.tmpdir)
|
||||
if hit.startswith(".pm-") or hit.startswith("staged")
|
||||
]
|
||||
self.assertEqual(leftovers, [])
|
||||
self.assertEqual(
|
||||
[n for n in os.listdir(self.tmpdir) if n.startswith(".pm-")], []
|
||||
)
|
||||
|
||||
|
||||
class TestLegacyRestore(RestoreTestCase):
|
||||
async def test_legacy_restore_unchanged(self):
|
||||
self.save("old")
|
||||
resp = await self.client.post(
|
||||
"/prompt_manager/restore", data=self.form(self.upload(["a", "b"]))
|
||||
)
|
||||
body = await resp.json()
|
||||
self.assertEqual(resp.status, 200, body)
|
||||
self.assertEqual(
|
||||
set(body), {"success", "message", "prompt_count", "backup_created"}
|
||||
)
|
||||
self.assertEqual(
|
||||
body["message"], "Database restored successfully. Found 2 prompts."
|
||||
)
|
||||
self.assertEqual(self.stages(), [])
|
||||
[job] = self.api._jobs.list()
|
||||
self.assertEqual((job.kind, job.admission), ("restore", "exclusive"))
|
||||
|
||||
async def test_legacy_restore_reports_damaged_documents(self):
|
||||
upload = os.path.join(self.tmpdir, "uploads", "damaged.db")
|
||||
os.makedirs(os.path.dirname(upload), exist_ok=True)
|
||||
src = PromptDatabase(upload)
|
||||
prompt_id = self.save("with a workflow", src)
|
||||
src.link_image_to_prompt(
|
||||
prompt_id, os.path.join(self.tmpdir, "a.png"), {"workflow": {"v": 1}}
|
||||
)
|
||||
conn = src.model.get_connection()
|
||||
with conn:
|
||||
conn.execute("UPDATE workflow_documents SET data=x'00'")
|
||||
src.close_all()
|
||||
resp = await self.client.post(
|
||||
"/prompt_manager/restore", data=self.form(Path(upload).read_bytes())
|
||||
)
|
||||
body = await resp.json()
|
||||
self.assertEqual(resp.status, 200, body)
|
||||
self.assertEqual(
|
||||
body["workflow_documents"], {"repaired": 0, "cleared": 2, "images": 1}
|
||||
) # the image's workflow and its prompt graph
|
||||
self.assertTrue(
|
||||
body["message"].endswith(
|
||||
"Could not repair 2 workflow documents: 1 image shows no workflow."
|
||||
),
|
||||
body["message"],
|
||||
)
|
||||
ok, reason = PromptModel.verify_database_file(self.db_path)
|
||||
self.assertTrue(ok, reason)
|
||||
|
||||
|
||||
class TestBackupDownload(RestoreTestCase):
|
||||
async def test_backup_verifies_snapshot_before_streaming(self):
|
||||
# A corrupt snapshot is refused (a semantic-only failure still
|
||||
# downloads: tests/test_restore_review_fixes.py, #335 review M5).
|
||||
self.save("keep")
|
||||
with patch(
|
||||
"py.api.backup_restore.snapshot_corruption",
|
||||
return_value="Integrity check failed: page 3",
|
||||
):
|
||||
resp = await self.client.get("/prompt_manager/backup")
|
||||
self.assertEqual(resp.status, 500)
|
||||
self.assertEqual(
|
||||
await resp.json(),
|
||||
{"success": False, "error": "Failed to create database backup"},
|
||||
)
|
||||
self.assertEqual(
|
||||
[n for n in os.listdir(self.tmpdir) if n.startswith(".pm-")], []
|
||||
)
|
||||
status = await (await self.client.get("/prompt_manager/backup/status")).json()
|
||||
self.assertIsNone(status["last_download"])
|
||||
|
||||
async def test_temp_snapshot_removed_and_closed(self):
|
||||
self.save("keep")
|
||||
created = []
|
||||
from database import safety_backup
|
||||
|
||||
original = safety_backup.safety_temp
|
||||
|
||||
def spy(db_path):
|
||||
path = original(db_path)
|
||||
created.append(path)
|
||||
return path
|
||||
|
||||
with patch("py.api.backup_restore.safety_temp", spy):
|
||||
resp, data = await self.download_backup()
|
||||
self.assertEqual(resp.status, 200)
|
||||
[temp] = created
|
||||
self.assertEqual(os.path.dirname(temp), self.tmpdir)
|
||||
self.assertFalse(os.path.exists(temp))
|
||||
downloaded = os.path.join(self.tmpdir, "downloaded.db")
|
||||
Path(downloaded).write_bytes(data)
|
||||
self.assertEqual(_texts(downloaded), ["keep"])
|
||||
with closing(sqlite3.connect(downloaded)) as conn:
|
||||
self.assertEqual(
|
||||
conn.execute("PRAGMA journal_mode").fetchone()[0], "delete"
|
||||
)
|
||||
|
||||
async def test_status_null_download_on_fresh_server(self):
|
||||
resp = await self.client.get("/prompt_manager/backup/status")
|
||||
body = await resp.json()
|
||||
self.assertEqual(
|
||||
body, {"success": True, "safety_backups": [], "last_download": None}
|
||||
)
|
||||
await self.download_backup()
|
||||
body = await (await self.client.get("/prompt_manager/backup/status")).json()
|
||||
download = body["last_download"]
|
||||
self.assertEqual(set(download), {"at", "bytes", "verified"})
|
||||
self.assertTrue(download["verified"])
|
||||
self.assertGreater(download["bytes"], 0)
|
||||
|
||||
async def test_status_lists_safety_backups_from_disk(self):
|
||||
for index in range(12):
|
||||
path = Path(f"{self.db_path}.backup_2026010{index % 10}_00000{index}")
|
||||
path.write_bytes(b"x" * (index + 1))
|
||||
os.utime(path, (1_000_000 + index, 1_000_000 + index))
|
||||
Path(self.tmpdir, "other.db.backup_1").write_bytes(b"x")
|
||||
body = await (await self.client.get("/prompt_manager/backup/status")).json()
|
||||
listed = body["safety_backups"]
|
||||
self.assertEqual(len(listed), 10)
|
||||
self.assertEqual(listed[0]["bytes"], 12)
|
||||
self.assertEqual(listed[-1]["bytes"], 3)
|
||||
for row in listed:
|
||||
self.assertEqual(set(row), {"name", "bytes", "modified_at"})
|
||||
self.assertTrue(row["name"].startswith("prompts.db.backup_"))
|
||||
self.assertNotIn(os.sep, row["name"])
|
||||
self.assertNotIn(self.tmpdir, str(body))
|
||||
|
||||
async def test_backup_refused_while_restore_runs(self):
|
||||
self.gated_job("restore", "exclusive", RESTORE_JOB_TITLE)
|
||||
resp = await self.client.get("/prompt_manager/backup")
|
||||
self.assertEqual(resp.status, 409)
|
||||
self.assertEqual(
|
||||
await resp.json(),
|
||||
{"success": False, "error": f"{RESTORE_JOB_TITLE} already running"},
|
||||
)
|
||||
|
||||
async def test_backup_is_listed_while_it_streams(self):
|
||||
self.save("keep")
|
||||
await self.download_backup()
|
||||
[job] = self.api._jobs.list()
|
||||
self.assertEqual(
|
||||
(job.kind, job.title, job.admission, job.state),
|
||||
("backup", "Database backup", "guarded", "finished"),
|
||||
)
|
||||
@@ -0,0 +1,612 @@
|
||||
"""PR #335 final review fixes (#249 System S8): I1, M1 to M7, N1 and N3.
|
||||
|
||||
- I1: once the stage replaced the library, the restore is done: later steps
|
||||
(pruning, the switch) only log, and the job and legacy route say so.
|
||||
- M1: a backup download holds the registry only while its snapshot is made.
|
||||
- M2: a restore preview leases only its live read; a busy drain keeps the
|
||||
reviewed stage.
|
||||
- M3: the reviewed-preview store changes only on the event loop.
|
||||
- M4: a document whose body is intact but whose key or length is damaged is
|
||||
re-keyed, not dropped.
|
||||
- M5: a backup download is refused only for a corrupt snapshot; a semantic
|
||||
check failure downloads, logged and reported.
|
||||
- M6: a cancelled preview removes what it staged.
|
||||
- M7 / N1 / N3: repair reads one row at a time; images counted once; the
|
||||
fingerprint never commits a caller's transaction.
|
||||
|
||||
Ordering is driven by events and job state, never by timers.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import os
|
||||
import sqlite3
|
||||
import threading
|
||||
import zlib
|
||||
from contextlib import closing
|
||||
from unittest.mock import patch
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from database import restore_stage
|
||||
from database import workflow_documents as wd
|
||||
from database.managed_connection import ManagedCursor
|
||||
from database.models import PromptModel
|
||||
from database.operations import PromptDatabase
|
||||
from py.api import admin as admin_mod
|
||||
from py.api import backup_restore, jobs
|
||||
|
||||
try:
|
||||
from tests.test_restore_plans import RestoreTestCase, _texts
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from test_restore_plans import RestoreTestCase, _texts
|
||||
|
||||
WAIT = 60
|
||||
|
||||
|
||||
class TestPostPublishSteps(RestoreTestCase):
|
||||
"""I1: a failure after the copy never says the library was not changed."""
|
||||
|
||||
async def test_job_finishes_when_pruning_fails_after_the_copy(self):
|
||||
self.save("old")
|
||||
old_db = self.api.db
|
||||
generation = self.api._db_generation
|
||||
body = await self.preview(self.upload(["restored"]))
|
||||
with (
|
||||
patch.object(admin_mod, "_prune_safety_backups", side_effect=OSError("x")),
|
||||
self.assertLogs(self.api.logger, "WARNING") as logs,
|
||||
):
|
||||
started = await self.apply(body["token"])
|
||||
job = await self.finish(started["job"]["id"])
|
||||
self.assertEqual(job.state, "finished", job.log)
|
||||
self.assertIn("Restored 1 prompt", job.status_line)
|
||||
self.assertNotIn("not changed", "\n".join(job.log))
|
||||
self.assertIsNot(self.api.db, old_db)
|
||||
self.assertEqual(self.api._db_generation, generation + 1)
|
||||
self.assertEqual(_texts(self.db_path), ["restored"])
|
||||
self.assertIn("prune", "\n".join(logs.output).lower())
|
||||
|
||||
async def test_job_adopts_the_library_when_closing_the_old_one_fails(self):
|
||||
self.save("old")
|
||||
old_db = self.api.db
|
||||
body = await self.preview(self.upload(["restored"]))
|
||||
try:
|
||||
with patch.object(old_db, "close_all", side_effect=RuntimeError("close")):
|
||||
started = await self.apply(body["token"])
|
||||
job = await self.finish(started["job"]["id"])
|
||||
self.assertEqual(job.state, "finished", job.log)
|
||||
self.assertIsNot(self.api.db, old_db)
|
||||
self.assertEqual(
|
||||
[p["text"] for p in self.api.db.search_prompts()], ["restored"]
|
||||
)
|
||||
finally:
|
||||
# The patched close left the old object's executor-thread
|
||||
# connections open; close them before teardown removes the files
|
||||
# (once the object is collected, close_all_instances misses it).
|
||||
await asyncio.get_running_loop().run_in_executor(None, old_db.close_all)
|
||||
|
||||
async def test_job_finishes_when_opening_the_new_object_fails(self):
|
||||
self.save("old")
|
||||
generation = self.api._db_generation
|
||||
body = await self.preview(self.upload(["restored"]))
|
||||
with patch.object(
|
||||
backup_restore, "PromptDatabase", side_effect=RuntimeError("open")
|
||||
):
|
||||
started = await self.apply(body["token"])
|
||||
job = await self.finish(started["job"]["id"])
|
||||
self.assertEqual(job.state, "finished", job.log)
|
||||
# The old object still points at the same, restored file.
|
||||
self.assertEqual(
|
||||
[p["text"] for p in self.api.db.search_prompts()], ["restored"]
|
||||
)
|
||||
self.assertEqual(self.api._db_generation, generation + 1)
|
||||
|
||||
async def test_legacy_restore_succeeds_when_pruning_fails_after_the_copy(self):
|
||||
self.save("old")
|
||||
old_db = self.api.db
|
||||
with patch.object(admin_mod, "_prune_safety_backups", side_effect=OSError("x")):
|
||||
resp = await self.client.post(
|
||||
"/prompt_manager/restore", data=self.form(self.upload(["restored"]))
|
||||
)
|
||||
body = await resp.json()
|
||||
self.assertEqual(resp.status, 200, body)
|
||||
self.assertEqual(
|
||||
body["message"], "Database restored successfully. Found 1 prompts."
|
||||
)
|
||||
self.assertIsNot(self.api.db, old_db)
|
||||
|
||||
|
||||
class TestBackupHoldScope(RestoreTestCase):
|
||||
"""M1 and M5: the hold covers the snapshot only; semantic issues warn."""
|
||||
|
||||
async def test_hold_ends_before_the_first_byte_is_sent(self):
|
||||
self.save("keep")
|
||||
seen = []
|
||||
real = web.StreamResponse.prepare
|
||||
|
||||
async def prepare(response, request):
|
||||
seen.append(self.api._jobs.active("backup"))
|
||||
return await real(response, request)
|
||||
|
||||
with patch.object(web.StreamResponse, "prepare", prepare):
|
||||
resp, data = await self.download_backup()
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertTrue(seen)
|
||||
self.assertEqual(set(seen), {None}) # no hold while bytes are sent
|
||||
[job] = self.api._jobs.list()
|
||||
self.assertEqual((job.kind, job.state), ("backup", "finished"))
|
||||
self.assertTrue(data.startswith(b"SQLite format 3\x00"))
|
||||
|
||||
async def test_a_corrupt_snapshot_is_refused(self):
|
||||
self.save("keep")
|
||||
with patch.object(
|
||||
backup_restore, "snapshot_corruption", return_value="Integrity check failed"
|
||||
):
|
||||
resp = await self.client.get("/prompt_manager/backup")
|
||||
self.assertEqual(resp.status, 500)
|
||||
self.assertEqual(
|
||||
await resp.json(),
|
||||
{"success": False, "error": "Failed to create database backup"},
|
||||
)
|
||||
self.assertEqual(
|
||||
[n for n in os.listdir(self.tmpdir) if n.startswith(".pm-")], []
|
||||
)
|
||||
[job] = self.api._jobs.list()
|
||||
self.assertEqual(job.state, "failed")
|
||||
|
||||
async def test_a_semantic_check_failure_still_downloads_and_says_so(self):
|
||||
self.save("keep")
|
||||
reason = "Image pair association has a different positive prompt owner"
|
||||
with (
|
||||
patch.object(
|
||||
PromptModel, "verify_database_file", return_value=(False, reason)
|
||||
),
|
||||
self.assertLogs(self.api.logger, "WARNING") as logs,
|
||||
):
|
||||
resp, data = await self.download_backup()
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertEqual(resp.headers["X-PromptManager-Verify"], reason)
|
||||
self.assertTrue(data.startswith(b"SQLite format 3\x00"))
|
||||
self.assertIn(reason, "\n".join(logs.output))
|
||||
status = await (await self.client.get("/prompt_manager/backup/status")).json()
|
||||
self.assertFalse(status["last_download"]["verified"])
|
||||
|
||||
async def test_a_verified_download_says_ok(self):
|
||||
self.save("keep")
|
||||
resp, _data = await self.download_backup()
|
||||
self.assertEqual(resp.headers["X-PromptManager-Verify"], "ok")
|
||||
|
||||
|
||||
class TestPreviewLease(RestoreTestCase):
|
||||
"""M2: upload and staging take no lease; a busy drain keeps the stage."""
|
||||
|
||||
async def test_staging_holds_no_lease(self):
|
||||
leases = []
|
||||
real = PromptModel.stage_restore
|
||||
|
||||
def stage(model, path):
|
||||
leases.append(dict(self.api._jobs.leases))
|
||||
return real(model, path)
|
||||
|
||||
with patch.object(PromptModel, "stage_restore", stage):
|
||||
await self.preview(self.upload(["restored"]))
|
||||
self.assertEqual(leases, [{}])
|
||||
|
||||
async def test_a_busy_drain_keeps_the_reviewed_stage(self):
|
||||
self.save("live")
|
||||
body = await self.preview(self.upload(["restored"]))
|
||||
lease = self.api._jobs.lease("Stuck preview")
|
||||
await lease.__aenter__()
|
||||
try:
|
||||
with patch.object(jobs, "LEASE_DRAIN_SECONDS", 0):
|
||||
started = await self.apply(body["token"])
|
||||
job = await self.finish(started["job"]["id"])
|
||||
finally:
|
||||
await lease.__aexit__(None, None, None)
|
||||
self.assertEqual(job.state, "failed")
|
||||
self.assertEqual(job.last_event["code"], "busy")
|
||||
# S9 I1: the page that started it reopens the kept review (Restore again
|
||||
# or Discard); the flag tells it so, and the line says where the review
|
||||
# is, never "start the restore again" from a place that cannot.
|
||||
self.assertIs(job.scope.get("review_kept"), True)
|
||||
self.assertEqual(job.status_line, backup_restore.REVIEW_KEPT)
|
||||
self.assertIn("reopened in the page that started", job.status_line)
|
||||
self.assertEqual(len(self.stages()), 1)
|
||||
# The same review applies once the requests are done.
|
||||
job = await self.finish((await self.apply(body["token"]))["job"]["id"])
|
||||
self.assertEqual(job.state, "finished", job.log)
|
||||
self.assertEqual(_texts(self.db_path), ["restored"])
|
||||
|
||||
async def test_a_review_that_cannot_be_kept_is_not_flagged_kept(self):
|
||||
self.save("live")
|
||||
body = await self.preview(self.upload(["restored"]))
|
||||
lease = self.api._jobs.lease("Stuck preview")
|
||||
await lease.__aenter__()
|
||||
try:
|
||||
with (
|
||||
patch.object(jobs, "LEASE_DRAIN_SECONDS", 0),
|
||||
patch.object(
|
||||
backup_restore.RestorePlanStore, "give_back", lambda *_: False
|
||||
),
|
||||
):
|
||||
started = await self.apply(body["token"])
|
||||
job = await self.finish(started["job"]["id"])
|
||||
finally:
|
||||
await lease.__aexit__(None, None, None)
|
||||
self.assertEqual(job.state, "failed")
|
||||
self.assertNotIn("review_kept", job.scope)
|
||||
self.assertEqual(job.status_line, backup_restore.REQUESTS_BUSY)
|
||||
self.assertEqual(len(self.stages()), 0)
|
||||
|
||||
|
||||
class TestStoreOnTheLoop(RestoreTestCase):
|
||||
"""M3: entries leave the store on the loop; only file removal is threaded."""
|
||||
|
||||
async def test_expired_entries_leave_the_store_before_their_files_go(self):
|
||||
clock = [1000.0]
|
||||
store = self.api._restore_plan_store = backup_restore.RestorePlanStore(
|
||||
clock=lambda: clock[0]
|
||||
)
|
||||
body = await self.preview(self.upload(["restored"]))
|
||||
held = store.peek(body["token"])
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
real = held.staged.discard
|
||||
|
||||
def slow_discard():
|
||||
entered.set()
|
||||
release.wait(WAIT)
|
||||
real()
|
||||
|
||||
held.staged.discard = slow_discard
|
||||
clock[0] += 10_000
|
||||
cleanup = asyncio.ensure_future(self.api._drop_expired_restores())
|
||||
loop = asyncio.get_running_loop()
|
||||
self.assertTrue(await loop.run_in_executor(None, entered.wait, WAIT))
|
||||
# The removal runs on a worker; the store already forgot the entry.
|
||||
self.assertEqual(len(store), 0)
|
||||
second = await self.preview(self.upload(["other"], "other.db"))
|
||||
self.assertEqual(len(store), 1)
|
||||
release.set()
|
||||
await cleanup
|
||||
self.assertEqual(len(self.stages()), 1)
|
||||
store.peek(second["token"])
|
||||
|
||||
async def test_an_expired_preview_is_removed_without_another_request(self):
|
||||
clock = [1000.0]
|
||||
self.api._restore_plan_store = backup_restore.RestorePlanStore(
|
||||
clock=lambda: clock[0], ttl=0
|
||||
)
|
||||
await self.preview(self.upload(["restored"]))
|
||||
# The expiry timer fires without another request: wait on its state.
|
||||
while self.stages() or len(self.api._restore_plans):
|
||||
await asyncio.sleep(0)
|
||||
self.assertEqual(self.stages(), [])
|
||||
|
||||
|
||||
class TestCancelledPreview(RestoreTestCase):
|
||||
"""M6: a cancelled preview leaves no stage behind."""
|
||||
|
||||
async def test_cancel_while_reading_the_live_library(self):
|
||||
path = os.path.join(self.tmpdir, "uploads", "up.db")
|
||||
self.upload(["restored"], "up.db")
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
real = self.api._live_fingerprint
|
||||
done = threading.Event()
|
||||
|
||||
def slow(db):
|
||||
entered.set()
|
||||
release.wait(WAIT)
|
||||
try:
|
||||
return real(db)
|
||||
finally:
|
||||
done.set()
|
||||
|
||||
with patch.object(self.api, "_live_fingerprint", side_effect=slow):
|
||||
task = asyncio.ensure_future(
|
||||
self.api._review_upload(
|
||||
self.api.db, self.api._database_identity(self.api.db), path
|
||||
)
|
||||
)
|
||||
loop = asyncio.get_running_loop()
|
||||
self.assertTrue(await loop.run_in_executor(None, entered.wait, WAIT))
|
||||
self.assertEqual(len(self.stages()), 1)
|
||||
task.cancel()
|
||||
release.set()
|
||||
with self.assertRaises(asyncio.CancelledError):
|
||||
await task
|
||||
# The cancel cannot stop the worker thread: it still opens the live
|
||||
# library. Let it finish, or teardown can find the file open.
|
||||
self.assertTrue(await loop.run_in_executor(None, done.wait, WAIT))
|
||||
self.assertEqual(self.stages(), [])
|
||||
self.assertEqual(len(self.api._restore_plans), 0)
|
||||
|
||||
async def test_cancel_while_staging(self):
|
||||
path = os.path.join(self.tmpdir, "uploads", "up.db")
|
||||
self.upload(["restored"], "up.db")
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
real = PromptModel.stage_restore
|
||||
done = threading.Event()
|
||||
|
||||
def slow(model, src):
|
||||
entered.set()
|
||||
release.wait(WAIT)
|
||||
try:
|
||||
return real(model, src)
|
||||
finally:
|
||||
done.set()
|
||||
|
||||
with patch.object(PromptModel, "stage_restore", slow):
|
||||
task = asyncio.ensure_future(
|
||||
self.api._review_upload(
|
||||
self.api.db, self.api._database_identity(self.api.db), path
|
||||
)
|
||||
)
|
||||
loop = asyncio.get_running_loop()
|
||||
self.assertTrue(await loop.run_in_executor(None, entered.wait, WAIT))
|
||||
task.cancel()
|
||||
with self.assertRaises(asyncio.CancelledError):
|
||||
await task
|
||||
release.set()
|
||||
self.assertTrue(await loop.run_in_executor(None, done.wait, WAIT))
|
||||
# The worker's finished stage is removed without another request.
|
||||
while self.api._orphan_stages or self.stages():
|
||||
await asyncio.sleep(0)
|
||||
self.assertEqual(self.stages(), [])
|
||||
|
||||
|
||||
class TestRekeyDocuments(RestoreTestCase):
|
||||
"""M4: an intact body with a damaged key or length is re-keyed."""
|
||||
|
||||
def library(self, damage):
|
||||
path = os.path.join(self.tmpdir, "uploads", "d.db")
|
||||
self.upload(["x"], "d.db")
|
||||
raw = b'{"nodes": [1, 2, 3]}'
|
||||
digest = hashlib.sha256(raw).digest()
|
||||
with closing(sqlite3.connect(path)) as conn:
|
||||
sha, length = damage(digest, len(raw))
|
||||
doc = conn.execute(
|
||||
"INSERT INTO workflow_documents (sha256, byte_length, data) "
|
||||
"VALUES (?,?,?)",
|
||||
(sha, length, zlib.compress(raw)),
|
||||
).lastrowid
|
||||
prompt_id = conn.execute("SELECT id FROM prompts").fetchone()[0]
|
||||
conn.execute(
|
||||
"INSERT INTO generated_images (prompt_id, image_path, filename, "
|
||||
"workflow_document_id) VALUES (?, '/x/a.png', 'a.png', ?)",
|
||||
(prompt_id, doc),
|
||||
)
|
||||
conn.commit()
|
||||
return path, raw.decode()
|
||||
|
||||
def staged_texts(self, staged):
|
||||
with closing(sqlite3.connect(staged.path)) as conn:
|
||||
return [
|
||||
wd.read_stored_text(conn, None, row[0])
|
||||
for row in conn.execute(
|
||||
"SELECT workflow_document_id FROM generated_images"
|
||||
)
|
||||
]
|
||||
|
||||
def test_a_damaged_key_is_recomputed(self):
|
||||
def flip(digest, length):
|
||||
bad = bytearray(digest)
|
||||
bad[0] ^= 1
|
||||
return bytes(bad), length
|
||||
|
||||
path, text = self.library(flip)
|
||||
staged = self.api.db.model.stage_restore(path)
|
||||
try:
|
||||
self.assertEqual(
|
||||
staged.documents, {"repaired": 1, "cleared": 0, "images": 0}
|
||||
)
|
||||
self.assertEqual(self.staged_texts(staged), [text])
|
||||
finally:
|
||||
staged.discard()
|
||||
|
||||
def test_a_damaged_length_is_recomputed(self):
|
||||
path, text = self.library(lambda digest, length: (digest, length + 7))
|
||||
staged = self.api.db.model.stage_restore(path)
|
||||
try:
|
||||
self.assertEqual(staged.documents["repaired"], 1)
|
||||
self.assertEqual(self.staged_texts(staged), [text])
|
||||
finally:
|
||||
staged.discard()
|
||||
|
||||
def test_a_key_that_another_document_holds_is_merged_into_it(self):
|
||||
path, text = self.library(lambda digest, length: (b"\0" * 32, length))
|
||||
with closing(sqlite3.connect(path)) as conn:
|
||||
raw = text.encode()
|
||||
conn.execute(
|
||||
"INSERT INTO workflow_documents (sha256, byte_length, data) "
|
||||
"VALUES (?,?,?)",
|
||||
(hashlib.sha256(raw).digest(), len(raw), zlib.compress(raw)),
|
||||
)
|
||||
conn.commit()
|
||||
staged = self.api.db.model.stage_restore(path)
|
||||
try:
|
||||
self.assertEqual(staged.documents["repaired"], 1)
|
||||
self.assertEqual(self.staged_texts(staged), [text])
|
||||
with closing(sqlite3.connect(staged.path)) as conn:
|
||||
self.assertEqual(
|
||||
conn.execute("SELECT COUNT(*) FROM workflow_documents").fetchone(),
|
||||
(1,),
|
||||
)
|
||||
finally:
|
||||
staged.discard()
|
||||
|
||||
|
||||
class TestRepairReadsOneRowAtATime(RestoreTestCase):
|
||||
"""M7: repair never iterates a managed cursor (128-row prefetch)."""
|
||||
|
||||
def test_repair_uses_fetchone(self):
|
||||
upload = os.path.join(self.tmpdir, "uploads", "dmg.db")
|
||||
os.makedirs(os.path.dirname(upload), exist_ok=True)
|
||||
src = PromptDatabase(upload)
|
||||
prompt_id = self.save("with a workflow", src)
|
||||
image = src.link_image_to_prompt(
|
||||
prompt_id, os.path.join(self.tmpdir, "a.png"), {"workflow": {"v": 1}}
|
||||
)
|
||||
conn = src.model.get_connection()
|
||||
with conn:
|
||||
text = wd.read_stored_text(
|
||||
conn,
|
||||
None,
|
||||
conn.execute(
|
||||
"SELECT workflow_document_id FROM generated_images WHERE id=?",
|
||||
(image,),
|
||||
).fetchone()[0],
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO generated_images (prompt_id, image_path, filename, "
|
||||
"workflow_data) VALUES (?, '/x/b.png', 'b.png', ?)",
|
||||
(prompt_id, text),
|
||||
)
|
||||
conn.execute("UPDATE workflow_documents SET data=x'00'")
|
||||
try:
|
||||
with patch.object(
|
||||
ManagedCursor, "fetchmany", side_effect=AssertionError("prefetch")
|
||||
):
|
||||
result = wd.repair_damaged_documents(conn)
|
||||
self.assertEqual(result["repaired"], 1)
|
||||
finally:
|
||||
src.close_all()
|
||||
|
||||
|
||||
class TestFingerprintTransaction(RestoreTestCase):
|
||||
"""N3: the fingerprint never commits a transaction it did not open."""
|
||||
|
||||
def test_a_callers_transaction_is_left_open(self):
|
||||
conn = self.api.db.model.get_connection()
|
||||
conn.execute("BEGIN")
|
||||
conn.execute("INSERT INTO prompts (text, hash) VALUES ('pending', 'p')")
|
||||
restore_stage.live_fingerprint(conn)
|
||||
self.assertTrue(conn.in_transaction)
|
||||
conn.rollback()
|
||||
self.assertEqual([p["text"] for p in self.api.db.search_prompts()], [])
|
||||
|
||||
|
||||
class TestRereviewFixes(RestoreTestCase):
|
||||
"""PR #335 re-review: R1 to R4."""
|
||||
|
||||
def test_r1_a_merge_into_a_damaged_holder_keeps_both_images(self):
|
||||
rekey = TestRekeyDocuments.library
|
||||
path, text = rekey(self, lambda digest, length: (b"\0" * 32, length))
|
||||
raw = text.encode()
|
||||
with closing(sqlite3.connect(path)) as conn:
|
||||
holder = conn.execute(
|
||||
"INSERT INTO workflow_documents (sha256, byte_length, data) "
|
||||
"VALUES (?,?,?)",
|
||||
(hashlib.sha256(raw).digest(), len(raw), b"garbage-not-zlib"),
|
||||
).lastrowid
|
||||
prompt_id = conn.execute("SELECT id FROM prompts").fetchone()[0]
|
||||
conn.execute(
|
||||
"INSERT INTO generated_images (prompt_id, image_path, filename, "
|
||||
"workflow_document_id) VALUES (?, '/x/b.png', 'b.png', ?)",
|
||||
(prompt_id, holder),
|
||||
)
|
||||
conn.commit()
|
||||
staged = self.api.db.model.stage_restore(path)
|
||||
try:
|
||||
self.assertEqual(
|
||||
staged.documents, {"repaired": 2, "cleared": 0, "images": 0}
|
||||
)
|
||||
self.assertEqual(
|
||||
TestRekeyDocuments.staged_texts(self, staged), [text, text]
|
||||
)
|
||||
with closing(sqlite3.connect(staged.path)) as conn:
|
||||
self.assertEqual(
|
||||
conn.execute("SELECT COUNT(*) FROM workflow_documents").fetchone(),
|
||||
(1,),
|
||||
)
|
||||
finally:
|
||||
staged.discard()
|
||||
|
||||
async def test_r2_a_losing_duplicate_apply_leaves_the_taken_stage(self):
|
||||
self.save("live")
|
||||
body = await self.preview(self.upload(["restored"]))
|
||||
store = self.api._restore_plans
|
||||
held = store.peek(body["token"])
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
real = self.api._live_fingerprint
|
||||
|
||||
def gated(db):
|
||||
entered.set()
|
||||
release.wait(WAIT)
|
||||
return real(db)
|
||||
|
||||
with patch.object(self.api, "_live_fingerprint", side_effect=gated):
|
||||
second = asyncio.ensure_future(
|
||||
self.client.post(
|
||||
"/prompt_manager/restore/apply", json={"token": body["token"]}
|
||||
)
|
||||
)
|
||||
loop = asyncio.get_running_loop()
|
||||
self.assertTrue(await loop.run_in_executor(None, entered.wait, WAIT))
|
||||
# Meanwhile the first apply's job took the stage, and a write landed.
|
||||
store.take(body["token"], "first-job")
|
||||
self.save("a run count")
|
||||
release.set()
|
||||
resp = await second
|
||||
self.assertEqual(resp.status, 409)
|
||||
self.assertTrue(os.path.isdir(held.staged.directory))
|
||||
await self._discard_quietly(held)
|
||||
|
||||
async def _discard_quietly(self, held):
|
||||
await asyncio.get_running_loop().run_in_executor(None, held.staged.discard)
|
||||
|
||||
def test_r3_give_back_refuses_a_full_store_or_an_expired_review(self):
|
||||
clock = [1000.0]
|
||||
store = backup_restore.RestorePlanStore(clock=lambda: clock[0], max_stages=1)
|
||||
first = store.issue(_Stage(), "f", "id", {})
|
||||
held = store.take(first, "job")
|
||||
other = store.issue(_Stage(), "f", "id", {})
|
||||
self.assertFalse(store.give_back(first, held)) # the slot was reused
|
||||
self.assertEqual(len(store), 1)
|
||||
store.pop(other)
|
||||
clock[0] = held.expires_at
|
||||
self.assertFalse(store.give_back(first, held)) # expired meanwhile
|
||||
clock[0] -= 1
|
||||
self.assertTrue(store.give_back(first, held))
|
||||
self.assertIs(store.peek(first), held)
|
||||
|
||||
async def test_r4_a_cancelled_backup_removes_its_snapshot(self):
|
||||
from aiohttp.test_utils import make_mocked_request
|
||||
|
||||
self.save("keep")
|
||||
entered, release, done = (threading.Event() for _ in range(3))
|
||||
real = backup_restore.write_backup_snapshot
|
||||
|
||||
def slow(model):
|
||||
entered.set()
|
||||
release.wait(WAIT)
|
||||
try:
|
||||
return real(model)
|
||||
finally:
|
||||
done.set()
|
||||
|
||||
with patch.object(backup_restore, "write_backup_snapshot", slow):
|
||||
task = asyncio.ensure_future(
|
||||
self.api._stream_backup(
|
||||
make_mocked_request("GET", "/prompt_manager/backup")
|
||||
)
|
||||
)
|
||||
loop = asyncio.get_running_loop()
|
||||
self.assertTrue(await loop.run_in_executor(None, entered.wait, WAIT))
|
||||
task.cancel()
|
||||
with self.assertRaises(asyncio.CancelledError):
|
||||
await task
|
||||
release.set()
|
||||
self.assertTrue(await loop.run_in_executor(None, done.wait, WAIT))
|
||||
|
||||
async def snapshot_gone():
|
||||
while any(n.startswith(".pm-safety-") for n in os.listdir(self.tmpdir)):
|
||||
await asyncio.sleep(0)
|
||||
|
||||
# A ceiling only a regression reaches (the file never goes).
|
||||
await asyncio.wait_for(snapshot_gone(), WAIT)
|
||||
self.assertIsNone(self.api._jobs.active("backup"))
|
||||
|
||||
|
||||
class _Stage:
|
||||
def discard(self):
|
||||
pass
|
||||
@@ -0,0 +1,440 @@
|
||||
"""A restore is staged beside the library, then published (#249 System S8).
|
||||
|
||||
``PromptModel.stage_restore`` copies an upload into a stage folder in the
|
||||
library's own folder, migrates, repairs and verifies it there, and never
|
||||
touches the live database or the source. ``publish_staged`` re-checks the
|
||||
staged bytes, keeps a uniquely named safety backup and copies the stage over
|
||||
the live pages, then truncates the WAL. ``restore_from_file`` is the two in a
|
||||
row, with the behavior it always had.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import hashlib
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
import zlib
|
||||
from contextlib import closing
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from database import restore_stage, safety_backup # noqa: E402
|
||||
from database import workflow_documents as wd # noqa: E402
|
||||
from database.models import PromptModel # noqa: E402
|
||||
from database.operations import PromptDatabase # noqa: E402
|
||||
from utils.hashing import generate_prompt_hash # noqa: E402
|
||||
|
||||
try:
|
||||
from tests.open_handles import assert_closed # noqa: E402
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from open_handles import assert_closed # noqa: E402
|
||||
|
||||
WAIT = 60 # a bound only a hung test reaches
|
||||
|
||||
|
||||
def _texts(path):
|
||||
with closing(sqlite3.connect(path)) as conn:
|
||||
return sorted(r[0] for r in conn.execute("SELECT text FROM prompts"))
|
||||
|
||||
|
||||
def _digests(path):
|
||||
"""sha256 of the file and each sidecar that exists."""
|
||||
return {
|
||||
suffix: hashlib.sha256(Path(path + suffix).read_bytes()).hexdigest()
|
||||
for suffix in ("", "-wal", "-shm", "-journal")
|
||||
if os.path.exists(path + suffix)
|
||||
}
|
||||
|
||||
|
||||
class StageTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmpdir = tempfile.mkdtemp()
|
||||
self.library = os.path.join(self.tmpdir, "library")
|
||||
os.mkdir(self.library)
|
||||
self.uploads = os.path.join(self.tmpdir, "uploads")
|
||||
os.mkdir(self.uploads)
|
||||
self.path = os.path.join(self.library, "prompts.db")
|
||||
self.db = PromptDatabase(self.path)
|
||||
|
||||
def tearDown(self):
|
||||
PromptModel.close_all_instances()
|
||||
assert_closed(self, self.tmpdir)
|
||||
for root, dirs, files in os.walk(self.tmpdir, topdown=False):
|
||||
for name in files:
|
||||
os.unlink(os.path.join(root, name))
|
||||
for name in dirs:
|
||||
os.rmdir(os.path.join(root, name))
|
||||
os.rmdir(self.tmpdir)
|
||||
|
||||
def save(self, db, text):
|
||||
return db.save_prompt(text=text, prompt_hash=generate_prompt_hash(text))
|
||||
|
||||
def source(self, texts, name="source.db", journal="wal"):
|
||||
path = os.path.join(self.uploads, name)
|
||||
src = PromptDatabase(path)
|
||||
for text in texts:
|
||||
self.save(src, text)
|
||||
src.close_all()
|
||||
if journal != "wal":
|
||||
with closing(sqlite3.connect(path)) as conn:
|
||||
conn.execute(f"PRAGMA journal_mode={journal}").fetchone()
|
||||
return path
|
||||
|
||||
def stage_dirs(self):
|
||||
return sorted(
|
||||
name
|
||||
for name in os.listdir(self.library)
|
||||
if name.startswith(restore_stage.STAGE_PREFIX)
|
||||
)
|
||||
|
||||
def backups(self):
|
||||
return sorted(
|
||||
name
|
||||
for name in os.listdir(self.library)
|
||||
if name.startswith("prompts.db.backup_")
|
||||
)
|
||||
|
||||
|
||||
class TestStageAndPublish(StageTestCase):
|
||||
def test_restore_from_file_unchanged_via_stage_publish(self):
|
||||
self.save(self.db, "old prompt")
|
||||
src = self.source(["restored one", "restored two"])
|
||||
model = self.db.model
|
||||
calls = []
|
||||
real_stage, real_publish = model.stage_restore, model.publish_staged
|
||||
|
||||
def stage(path):
|
||||
calls.append("stage")
|
||||
return real_stage(path)
|
||||
|
||||
def publish(staged):
|
||||
calls.append("publish")
|
||||
return real_publish(staged)
|
||||
|
||||
with (
|
||||
patch.object(model, "stage_restore", stage),
|
||||
patch.object(model, "publish_staged", publish),
|
||||
):
|
||||
backup_path = model.restore_from_file(src)
|
||||
|
||||
self.assertEqual(calls, ["stage", "publish"])
|
||||
self.assertTrue(backup_path.startswith(self.path + ".backup_"))
|
||||
self.assertEqual(_texts(backup_path), ["old prompt"])
|
||||
self.assertEqual(_texts(self.path), ["restored one", "restored two"])
|
||||
self.assertEqual(self.stage_dirs(), [])
|
||||
|
||||
def test_stage_never_touches_live_or_source(self):
|
||||
self.save(self.db, "live")
|
||||
self.db.model.get_connection() # the live file stays open, WAL and all
|
||||
src = self.source(["restored"])
|
||||
live_before = _digests(self.path)
|
||||
source_before = _digests(src)[""]
|
||||
|
||||
staged = self.db.model.stage_restore(src)
|
||||
try:
|
||||
self.assertEqual(_digests(self.path), live_before)
|
||||
# Read-only opens of a WAL-flagged file add empty sidecars, as
|
||||
# they always did; the source's bytes never change.
|
||||
self.assertEqual(_digests(src)[""], source_before)
|
||||
self.assertEqual(self.backups(), [])
|
||||
self.assertEqual(_texts(self.path), ["live"])
|
||||
finally:
|
||||
staged.discard()
|
||||
|
||||
def test_stage_lives_in_db_directory_and_is_removed(self):
|
||||
src = self.source(["restored"])
|
||||
staged = self.db.model.stage_restore(src)
|
||||
stage_dir = os.path.dirname(staged.path)
|
||||
self.assertEqual(os.path.dirname(stage_dir), self.library)
|
||||
self.assertEqual(self.stage_dirs(), [os.path.basename(stage_dir)])
|
||||
# One self-contained file: no sidecar of an open or WAL database.
|
||||
self.assertEqual(os.listdir(stage_dir), [os.path.basename(staged.path)])
|
||||
staged.discard()
|
||||
self.assertEqual(self.stage_dirs(), [])
|
||||
staged.discard() # a second discard is harmless
|
||||
|
||||
# A failed stage removes its folder too.
|
||||
bad = os.path.join(self.uploads, "notes.db")
|
||||
Path(bad).write_bytes(b"not a database")
|
||||
with self.assertRaises(ValueError):
|
||||
self.db.model.stage_restore(bad)
|
||||
self.assertEqual(self.stage_dirs(), [])
|
||||
|
||||
def test_staged_handles_closed_before_cleanup(self):
|
||||
src = self.source(["restored"])
|
||||
staged = self.db.model.stage_restore(src)
|
||||
assert_closed(self, os.path.dirname(staged.path))
|
||||
self.db.model.publish_staged(staged)
|
||||
assert_closed(self, os.path.dirname(staged.path))
|
||||
staged.discard()
|
||||
self.assertFalse(os.path.exists(os.path.dirname(staged.path)))
|
||||
|
||||
def test_safety_backup_name_unique_within_one_second(self):
|
||||
self.save(self.db, "first")
|
||||
frozen = datetime.datetime(2026, 10, 25, 9, 30, 0, 123456)
|
||||
names = []
|
||||
with patch.object(safety_backup, "_now", return_value=frozen):
|
||||
for text in ("second", "third"):
|
||||
staged = self.db.model.stage_restore(self.source([text], f"{text}.db"))
|
||||
try:
|
||||
names.append(os.path.basename(self.db.model.publish_staged(staged)))
|
||||
finally:
|
||||
staged.discard()
|
||||
self.assertEqual(
|
||||
names,
|
||||
[
|
||||
"prompts.db.backup_20261025_093000_123456",
|
||||
"prompts.db.backup_20261025_093000_123456_1",
|
||||
],
|
||||
)
|
||||
self.assertEqual(_texts(os.path.join(self.library, names[1])), ["second"])
|
||||
for name in names: # standalone: no WAL flag, no sidecars when read
|
||||
with closing(sqlite3.connect(os.path.join(self.library, name))) as conn:
|
||||
self.assertEqual(
|
||||
conn.execute("PRAGMA journal_mode").fetchone()[0], "delete"
|
||||
)
|
||||
if os.name != "nt":
|
||||
mode = os.stat(os.path.join(self.library, name)).st_mode & 0o777
|
||||
self.assertEqual(mode, 0o600)
|
||||
|
||||
def test_publish_binds_the_staged_bytes(self):
|
||||
self.save(self.db, "live")
|
||||
staged = self.db.model.stage_restore(self.source(["restored"]))
|
||||
try:
|
||||
with closing(sqlite3.connect(staged.path)) as conn:
|
||||
conn.execute("INSERT INTO prompts (text, hash) VALUES ('sneaked', 'x')")
|
||||
conn.commit()
|
||||
with self.assertRaises(restore_stage.StageChanged):
|
||||
self.db.model.publish_staged(staged)
|
||||
finally:
|
||||
staged.discard()
|
||||
self.assertEqual(_texts(self.path), ["live"])
|
||||
self.assertEqual(self.backups(), [])
|
||||
|
||||
def test_summary_schema_and_digest(self):
|
||||
prompt_id = self.save(self.db, "unused")
|
||||
src = self.source(["a", "b"])
|
||||
staged = self.db.model.stage_restore(src)
|
||||
try:
|
||||
self.assertEqual(staged.summary["prompts"], 2)
|
||||
self.assertEqual(staged.summary["positives"], 2)
|
||||
for key in ("pairs", "archived_pairs", "image_links", "runs"):
|
||||
self.assertEqual(staged.summary[key], 0, key)
|
||||
self.assertEqual(staged.summary["saved_views"], 0)
|
||||
self.assertEqual(staged.schema, {"migrated": False, "library_state": True})
|
||||
self.assertEqual(
|
||||
staged.digest,
|
||||
"sha256:" + hashlib.sha256(Path(staged.path).read_bytes()).hexdigest(),
|
||||
)
|
||||
self.assertEqual(staged.size, os.path.getsize(staged.path))
|
||||
live = restore_stage.library_summary(self.db.model.get_connection())
|
||||
self.assertEqual(live["prompts"], 1)
|
||||
self.assertTrue(prompt_id)
|
||||
finally:
|
||||
staged.discard()
|
||||
|
||||
|
||||
class TestOlderBackups(StageTestCase):
|
||||
"""Rollback-journal and older-schema backups restore exactly as before."""
|
||||
|
||||
def test_rollback_journal_backup_restores(self):
|
||||
self.save(self.db, "live")
|
||||
src = self.source(["from a delete-mode file"], journal="delete")
|
||||
with closing(sqlite3.connect(src)) as conn:
|
||||
self.assertEqual(
|
||||
conn.execute("PRAGMA journal_mode").fetchone()[0], "delete"
|
||||
)
|
||||
before = _digests(src)
|
||||
self.db.model.restore_from_file(src)
|
||||
self.assertEqual(_texts(self.path), ["from a delete-mode file"])
|
||||
self.assertEqual(_digests(src), before)
|
||||
|
||||
def test_older_schema_backup_is_migrated(self):
|
||||
src = os.path.join(self.uploads, "legacy.db")
|
||||
with closing(sqlite3.connect(src)) as conn:
|
||||
conn.execute(
|
||||
"CREATE TABLE prompts (id INTEGER PRIMARY KEY AUTOINCREMENT,"
|
||||
" text TEXT NOT NULL, created_at TIMESTAMP, updated_at TIMESTAMP,"
|
||||
" category TEXT, tags TEXT, rating INTEGER, notes TEXT,"
|
||||
" hash TEXT UNIQUE, workflow_name TEXT)"
|
||||
)
|
||||
conn.execute("INSERT INTO prompts (text) VALUES ('legacy')")
|
||||
conn.commit()
|
||||
staged = self.db.model.stage_restore(src)
|
||||
try:
|
||||
self.assertEqual(staged.schema, {"migrated": True, "library_state": False})
|
||||
self.assertEqual(staged.summary["prompts"], 1)
|
||||
self.db.model.publish_staged(staged)
|
||||
finally:
|
||||
staged.discard()
|
||||
self.assertEqual(_texts(self.path), ["legacy"])
|
||||
ok, reason = PromptModel.verify_database_file(
|
||||
self.path, require_library_state=True
|
||||
)
|
||||
self.assertTrue(ok, reason)
|
||||
|
||||
|
||||
class TestWalAfterRestore(StageTestCase):
|
||||
"""Carry-over 4: the live WAL is checkpointed and truncated after a restore."""
|
||||
|
||||
def test_wal_is_truncated_after_restore(self):
|
||||
src = self.source(["restored"])
|
||||
with closing(sqlite3.connect(src)) as conn:
|
||||
conn.execute("CREATE TABLE padding(b BLOB)")
|
||||
conn.execute("INSERT INTO padding VALUES (zeroblob(?))", (4 * 2**20,))
|
||||
conn.commit()
|
||||
self.save(self.db, "live")
|
||||
# Another thread's idle connection (the image monitor, a node).
|
||||
bystander = PromptDatabase(self.path)
|
||||
opened = threading.Event()
|
||||
worker = threading.Thread(
|
||||
target=lambda: (bystander.search_prompts(), opened.set())
|
||||
)
|
||||
worker.start()
|
||||
worker.join(WAIT)
|
||||
self.assertTrue(opened.is_set())
|
||||
|
||||
try:
|
||||
self.db.model.restore_from_file(src)
|
||||
|
||||
self.assertEqual(_texts(self.path), ["restored"])
|
||||
wal = self.path + "-wal"
|
||||
self.assertEqual(os.path.getsize(wal) if os.path.exists(wal) else 0, 0)
|
||||
self.assertEqual(
|
||||
[p["text"] for p in bystander.search_prompts()], ["restored"]
|
||||
)
|
||||
finally:
|
||||
bystander.close_all()
|
||||
|
||||
def test_a_reader_left_open_is_waited_for_then_logged(self):
|
||||
src = self.source(["restored"])
|
||||
self.save(self.db, "live")
|
||||
stuck = PromptDatabase(self.path)
|
||||
conn = stuck.model.get_connection()
|
||||
conn.execute("BEGIN")
|
||||
conn.execute("SELECT COUNT(*) FROM prompts").fetchone() # holds a snapshot
|
||||
try:
|
||||
with (
|
||||
patch.object(restore_stage, "CHECKPOINT_BUSY_MS", 50),
|
||||
self.assertLogs(
|
||||
"prompt_manager.database.restore_stage", "WARNING"
|
||||
) as logs,
|
||||
):
|
||||
self.db.model.restore_from_file(src)
|
||||
finally:
|
||||
conn.rollback()
|
||||
stuck.close_all()
|
||||
self.assertIn("could not be truncated", "\n".join(logs.output))
|
||||
self.assertEqual(_texts(self.path), ["restored"])
|
||||
|
||||
|
||||
class TestDamagedDocuments(StageTestCase):
|
||||
"""Carry-over 5: repair what can be repaired, report what cannot."""
|
||||
|
||||
def library_with_documents(self, name, inline_copy):
|
||||
"""A backup whose two documents are damaged; one has an inline copy."""
|
||||
path = os.path.join(self.uploads, name)
|
||||
src = PromptDatabase(path)
|
||||
prompt_id = self.save(src, "with workflows")
|
||||
first = src.link_image_to_prompt(
|
||||
prompt_id, os.path.join(self.uploads, "a.png"), {"workflow": {"v": 1}}
|
||||
)
|
||||
second = src.link_image_to_prompt(
|
||||
prompt_id, os.path.join(self.uploads, "b.png"), {"workflow": {"v": 2}}
|
||||
)
|
||||
conn = src.model.get_connection()
|
||||
ids = [
|
||||
conn.execute(
|
||||
"SELECT workflow_document_id FROM generated_images WHERE id=?", (i,)
|
||||
).fetchone()[0]
|
||||
for i in (first, second)
|
||||
]
|
||||
texts = [wd.read_stored_text(conn, None, document_id) for document_id in ids]
|
||||
with conn:
|
||||
if inline_copy:
|
||||
# An older row that still holds the first text inline.
|
||||
conn.execute(
|
||||
"INSERT INTO generated_images (prompt_id, image_path, filename,"
|
||||
" workflow_data) VALUES (?, ?, 'c.png', ?)",
|
||||
(prompt_id, os.path.join(self.uploads, "c.png"), texts[0]),
|
||||
)
|
||||
for document_id in ids:
|
||||
conn.execute(
|
||||
"UPDATE workflow_documents SET data=? WHERE id=?",
|
||||
(zlib.compress(b"damaged"), document_id),
|
||||
)
|
||||
src.close_all()
|
||||
return path, ids, texts, (first, second)
|
||||
|
||||
def test_damaged_documents_are_repaired_or_cleared_and_reported(self):
|
||||
src, ids, texts, images = self.library_with_documents("damaged.db", True)
|
||||
ok, reason = PromptModel.verify_database_file(src)
|
||||
self.assertFalse(ok)
|
||||
self.assertTrue(reason.startswith("Workflow document "), reason)
|
||||
before = _digests(src)
|
||||
|
||||
staged = self.db.model.stage_restore(src)
|
||||
try:
|
||||
self.assertEqual(
|
||||
staged.documents, {"repaired": 1, "cleared": 1, "images": 1}
|
||||
)
|
||||
self.db.model.publish_staged(staged)
|
||||
finally:
|
||||
staged.discard()
|
||||
|
||||
self.assertEqual(_digests(src), before) # the upload is never changed
|
||||
ok, reason = PromptModel.verify_database_file(self.path)
|
||||
self.assertTrue(ok, reason)
|
||||
conn = self.db.model.get_connection()
|
||||
rows = {
|
||||
row[0]: (row[1], row[2])
|
||||
for row in conn.execute(
|
||||
"SELECT id, workflow_data, workflow_document_id FROM generated_images"
|
||||
)
|
||||
}
|
||||
repaired = rows[images[0]]
|
||||
self.assertEqual(repaired[1], ids[0])
|
||||
self.assertEqual(wd.read_stored_text(conn, None, repaired[1]), texts[0])
|
||||
self.assertEqual(rows[images[1]], (None, None)) # cleared, nothing readable
|
||||
|
||||
def test_restore_from_file_accepts_a_backup_with_damaged_documents(self):
|
||||
src, _ids, _texts_, _images = self.library_with_documents("damaged.db", False)
|
||||
self.db.model.restore_from_file(src)
|
||||
ok, reason = PromptModel.verify_database_file(self.path)
|
||||
self.assertTrue(ok, reason)
|
||||
self.assertEqual(_texts(self.path), ["with workflows"])
|
||||
|
||||
def test_an_intact_backup_reports_nothing_repaired(self):
|
||||
staged = self.db.model.stage_restore(self.source(["clean"]))
|
||||
try:
|
||||
self.assertEqual(
|
||||
staged.documents, {"repaired": 0, "cleared": 0, "images": 0}
|
||||
)
|
||||
finally:
|
||||
staged.discard()
|
||||
|
||||
|
||||
class TestSweep(StageTestCase):
|
||||
def test_interrupted_stages_and_uploads_are_swept(self):
|
||||
stage = os.path.join(self.library, restore_stage.STAGE_PREFIX + "abc")
|
||||
os.mkdir(stage)
|
||||
Path(stage, "staged.db").write_bytes(b"x")
|
||||
upload = os.path.join(self.library, restore_stage.UPLOAD_PREFIX + "x.tmp")
|
||||
Path(upload).write_bytes(b"x")
|
||||
keep = os.path.join(self.library, "notes.txt")
|
||||
Path(keep).write_bytes(b"x")
|
||||
swept = restore_stage.sweep_restore_temps(self.path)
|
||||
self.assertEqual(
|
||||
sorted(swept), sorted([os.path.basename(stage), os.path.basename(upload)])
|
||||
)
|
||||
self.assertEqual(self.stage_dirs(), [])
|
||||
self.assertTrue(os.path.exists(keep))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,255 @@
|
||||
"""tools/review_image_links.py: a dry run by default, changes only named images."""
|
||||
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import unittest
|
||||
import unittest.mock
|
||||
|
||||
from PIL import Image
|
||||
from PIL.PngImagePlugin import PngInfo
|
||||
|
||||
try:
|
||||
from tests.open_handles import assert_closed
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from open_handles import assert_closed
|
||||
|
||||
from database.operations import PromptDatabase
|
||||
from tests.test_api_images import ImageAPITestCase
|
||||
from tests.test_prompt_attribution import issue_graph, pair_evidence
|
||||
from tools import review_image_links as tool
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
|
||||
def png(path, prompt=None, evidence=None):
|
||||
info = PngInfo()
|
||||
if prompt is not None:
|
||||
info.add_text("prompt", json.dumps(prompt))
|
||||
if evidence is not None:
|
||||
info.add_text("comfyassets_prompt_pairs", json.dumps(evidence))
|
||||
Image.new("RGB", (4, 4)).save(path, pnginfo=info)
|
||||
|
||||
|
||||
class ToolFixture(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, self.tmp, True)
|
||||
self.addCleanup(assert_closed, self, self.tmp)
|
||||
self.output = os.path.join(self.tmp, "output")
|
||||
os.makedirs(self.output)
|
||||
self.db_path = os.path.join(self.tmp, "prompts.db")
|
||||
db = PromptDatabase(self.db_path)
|
||||
try:
|
||||
save = lambda text: db.save_prompt( # noqa: E731
|
||||
text=text, prompt_hash=generate_prompt_hash(text)
|
||||
)
|
||||
self.coffee = save("a woman holding a coffee cup")
|
||||
self.typed = save("snowy peaks")
|
||||
# The six #342 images: only prepend text, linked by the old fallback
|
||||
self.guessed = []
|
||||
for n in range(9, 12):
|
||||
path = os.path.join(self.output, f"moss_{n:05}_.png")
|
||||
png(path, issue_graph(), pair_evidence())
|
||||
self.guessed.append(
|
||||
db.link_image_to_prompt(self.coffee, path, metadata={})
|
||||
)
|
||||
# A typed image linked to the wrong prompt, and a correct one
|
||||
wrong = os.path.join(self.output, "typed_wrong.png")
|
||||
png(wrong, issue_graph(text="snowy peaks"))
|
||||
self.wrong = db.link_image_to_prompt(self.coffee, wrong, metadata={})
|
||||
right = os.path.join(self.output, "typed_right.png")
|
||||
png(right, issue_graph(text="snowy peaks"))
|
||||
self.right = db.link_image_to_prompt(self.typed, right, metadata={})
|
||||
gone = os.path.join(self.output, "gone.png")
|
||||
self.gone = db.link_image_to_prompt(self.coffee, gone, metadata={})
|
||||
finally:
|
||||
db.close_all()
|
||||
|
||||
def run_tool(self, *args):
|
||||
out = io.StringIO()
|
||||
code = tool.main([self.db_path, "--output-dir", self.output, *args], out=out)
|
||||
return code, out.getvalue()
|
||||
|
||||
def links(self):
|
||||
db = PromptDatabase(self.db_path)
|
||||
try:
|
||||
rows = db.model.get_connection().execute(
|
||||
"SELECT id, prompt_id FROM generated_images"
|
||||
)
|
||||
return {row[0]: row[1] for row in rows}
|
||||
finally:
|
||||
db.close_all()
|
||||
|
||||
|
||||
class ReviewToolTests(ToolFixture):
|
||||
def test_dry_run_judges_and_changes_nothing(self):
|
||||
before = self.links()
|
||||
ids = f"{self.guessed[0]}-{self.gone}"
|
||||
code, out = self.run_tool("--ids", ids, "--json")
|
||||
self.assertEqual(code, 0)
|
||||
verdicts = {v["image_id"]: v for v in json.loads(out.split("--apply, for")[0])}
|
||||
for image_id in self.guessed:
|
||||
self.assertEqual(verdicts[image_id]["verdict"], "unlink")
|
||||
self.assertEqual(verdicts[image_id]["sources"], {})
|
||||
self.assertEqual(verdicts[self.wrong]["verdict"], "relink")
|
||||
self.assertEqual(verdicts[self.wrong]["target"], self.typed)
|
||||
self.assertEqual(verdicts[self.right]["verdict"], "keep")
|
||||
self.assertEqual(verdicts[self.gone]["verdict"], "missing")
|
||||
self.assertIn("Dry run: nothing was changed.", out)
|
||||
self.assertEqual(self.links(), before)
|
||||
self.assertEqual(
|
||||
sorted(os.listdir(self.tmp)), ["output", "prompts.db"], "no side files"
|
||||
)
|
||||
|
||||
def test_prompt_selection_and_text_report(self):
|
||||
code, out = self.run_tool("--prompt", str(self.coffee))
|
||||
self.assertEqual(code, 0)
|
||||
self.assertIn("unlink: 3", out)
|
||||
self.assertIn("relink: 1", out)
|
||||
self.assertIn("missing: 1", out)
|
||||
self.assertIn(f"-> {self.typed}", out)
|
||||
|
||||
def test_apply_backs_up_then_changes_only_named_images(self):
|
||||
named = f"{self.guessed[0]}-{self.guessed[1]},{self.wrong}"
|
||||
code, out = self.run_tool(
|
||||
"--ids", named, "--prompt", str(self.coffee), "--apply"
|
||||
)
|
||||
self.assertEqual(code, 0)
|
||||
self.assertIn("Applied: 1 relinked, 2 unlinked, 0 skipped", out)
|
||||
links = self.links()
|
||||
self.assertNotIn(self.guessed[0], links)
|
||||
self.assertNotIn(self.guessed[1], links)
|
||||
self.assertEqual(links[self.guessed[2]], self.coffee, "not named: kept")
|
||||
self.assertEqual(links[self.wrong], self.typed)
|
||||
self.assertEqual(links[self.gone], self.coffee, "missing files are kept")
|
||||
backups = [n for n in os.listdir(self.tmp) if ".pre-relink_" in n]
|
||||
self.assertEqual(len(backups), 1)
|
||||
restored = PromptDatabase(os.path.join(self.tmp, backups[0]))
|
||||
try:
|
||||
count = (
|
||||
restored.model.get_connection()
|
||||
.execute("SELECT COUNT(*) FROM generated_images")
|
||||
.fetchone()[0]
|
||||
)
|
||||
finally:
|
||||
restored.close_all()
|
||||
self.assertEqual(count, 6)
|
||||
|
||||
def test_arguments_are_validated(self):
|
||||
for args in (
|
||||
[],
|
||||
["--ids", "5-2"],
|
||||
["--ids", "x"],
|
||||
["--prompt", "1", "--apply"],
|
||||
):
|
||||
with self.subTest(args=args), self.assertRaises(SystemExit):
|
||||
with unittest.mock.patch("sys.stderr", io.StringIO()):
|
||||
self.run_tool(*args)
|
||||
with (
|
||||
self.assertRaises(SystemExit),
|
||||
unittest.mock.patch("sys.stderr", io.StringIO()),
|
||||
):
|
||||
tool.main(
|
||||
[os.path.join(self.tmp, "none.db"), "--output-dir", "x", "--ids", "1"]
|
||||
)
|
||||
self.assertEqual(tool.parse_ids(" 3, 1-2 ,"), [1, 2, 3])
|
||||
|
||||
def test_conflicting_sources_are_left_to_the_maintainer(self):
|
||||
db = PromptDatabase(self.db_path)
|
||||
try:
|
||||
handler = tool.ImageGenerationHandler(db, None)
|
||||
row = {
|
||||
"id": 1,
|
||||
"prompt_id": self.coffee,
|
||||
"image_path": os.path.join(self.output, "typed_right.png"),
|
||||
}
|
||||
with unittest.mock.patch.object(
|
||||
tool, "certain_prompts", return_value={"scan": 1, "history": 2}
|
||||
):
|
||||
verdict = tool.judge(db, handler, row, self.output)
|
||||
finally:
|
||||
db.close_all()
|
||||
self.assertEqual(verdict["verdict"], "conflict")
|
||||
|
||||
|
||||
class ApplyGuardTests(ToolFixture):
|
||||
"""M6: unlink clears only the link, and only the link it was judged on."""
|
||||
|
||||
def test_unlink_skips_a_row_linked_elsewhere_since_the_review(self):
|
||||
db = PromptDatabase(self.db_path)
|
||||
try:
|
||||
verdicts = tool.review(db, [self.guessed[0]], None, self.output)
|
||||
self.assertEqual(verdicts[0]["verdict"], "unlink")
|
||||
db.model.get_connection().execute(
|
||||
"UPDATE generated_images SET prompt_id=? WHERE id=?",
|
||||
(self.typed, self.guessed[0]),
|
||||
)
|
||||
db.model.get_connection().commit()
|
||||
result = tool.apply(db, verdicts, [self.guessed[0]])
|
||||
finally:
|
||||
db.close_all()
|
||||
self.assertEqual(result["unlinked"], 0)
|
||||
self.assertEqual(self.links()[self.guessed[0]], self.typed)
|
||||
|
||||
def test_unlink_keeps_the_file_and_its_favorite(self):
|
||||
path = os.path.join(self.output, "moss_00009_.png")
|
||||
db = PromptDatabase(self.db_path)
|
||||
try:
|
||||
db.set_image_favorite("a" * 64, "moss_00009_.png")
|
||||
finally:
|
||||
db.close_all()
|
||||
self.run_tool("--ids", str(self.guessed[0]), "--apply")
|
||||
self.assertTrue(os.path.isfile(path))
|
||||
db = PromptDatabase(self.db_path)
|
||||
try:
|
||||
self.assertEqual(db.get_root_favorites("a" * 64), {"moss_00009_.png"})
|
||||
summary = db.get_image_link_summaries([path]).get(path) or {}
|
||||
finally:
|
||||
db.close_all()
|
||||
self.assertNotEqual(summary.get("status"), "linked")
|
||||
|
||||
def test_dry_run_states_what_apply_does(self):
|
||||
_code, out = self.run_tool("--ids", str(self.guessed[0]))
|
||||
self.assertIn("removes only the image's link record", out)
|
||||
self.assertIn("file stays on disk and in the gallery as unlinked", out)
|
||||
|
||||
|
||||
class GalleryAfterUnlinkTests(ImageAPITestCase):
|
||||
"""M6: after --apply the file is still listed, as an unlinked image."""
|
||||
|
||||
async def test_unlinked_file_stays_in_the_workspace_gallery(self):
|
||||
db = self.api.db
|
||||
prompt = db.save_prompt(
|
||||
text="a woman holding a coffee cup",
|
||||
prompt_hash=generate_prompt_hash("a woman holding a coffee cup"),
|
||||
)
|
||||
path = self.output_dir / "moss_00009_.png"
|
||||
png(str(path), issue_graph(), pair_evidence())
|
||||
image_id = db.link_image_to_prompt(prompt, str(path), metadata={})
|
||||
out = io.StringIO()
|
||||
tool.main(
|
||||
[
|
||||
db.model.db_path,
|
||||
"--output-dir",
|
||||
str(self.output_dir),
|
||||
"--ids",
|
||||
str(image_id),
|
||||
"--apply",
|
||||
],
|
||||
out=out,
|
||||
)
|
||||
self.assertIn("Applied: 0 relinked, 1 unlinked", out.getvalue())
|
||||
response = await self.client.get(
|
||||
"/prompt_manager/images/output", params={"mode": "workspace"}
|
||||
)
|
||||
self.assertEqual(response.status, 200)
|
||||
images = {i["filename"]: i for i in (await response.json())["images"]}
|
||||
self.assertIn("moss_00009_.png", images)
|
||||
link = images["moss_00009_.png"].get("linked") or {}
|
||||
self.assertNotEqual(link.get("status"), "linked")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,674 @@
|
||||
"""#342: recorded history attributes a file only to one certain, stored prompt."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
from database.run_attribution import stored_run_prompt
|
||||
from tests import test_execution_history_database as history_tests
|
||||
from tests.test_api_admin import AdminAPITestCase
|
||||
from tests.test_execution_context_database import snapshot
|
||||
from tests.test_prompt_attribution import issue_graph, llm_text_graph, make_tracker
|
||||
from utils.execution_contexts import ContextBundle, RecipeContext
|
||||
from utils.execution_outputs import (
|
||||
OutputBundle,
|
||||
OutputReference,
|
||||
snapshot_output_roots,
|
||||
)
|
||||
from utils.hashing import generate_prompt_hash
|
||||
from utils.image_monitor import ImageGenerationHandler
|
||||
|
||||
STYLE = "windows" if os.name == "nt" else "posix"
|
||||
LLM_TEXT = "Photorealistic aerial landscape of misty mountains at dawn"
|
||||
|
||||
|
||||
def now_ms():
|
||||
return time.time_ns() // 1_000_000
|
||||
|
||||
|
||||
class RecordedRuns:
|
||||
"""Write runs into execution history the way the capture pipeline does."""
|
||||
|
||||
start = history_tests.HistoryDatabaseTests.start
|
||||
finish = history_tests.HistoryDatabaseTests.finish
|
||||
|
||||
def record_run(self, db, output_root, files, positives, item=0, window=None):
|
||||
started, finished = window or (now_ms() - 5000, now_ms() + 5000)
|
||||
start = self.start(db, item, started)
|
||||
db.write_execution_start(start)
|
||||
finish = self.finish(start, time=finished)
|
||||
db.write_execution_finish(finish)
|
||||
db.write_execution_contexts(
|
||||
start.run_id,
|
||||
start.storage_epoch,
|
||||
ContextBundle(
|
||||
tuple(
|
||||
RecipeContext(str(4 + i), snapshot(text), snapshot(""), "inferred")
|
||||
for i, text in enumerate(positives)
|
||||
),
|
||||
"complete",
|
||||
"captured",
|
||||
),
|
||||
)
|
||||
root = snapshot_output_roots(STYLE, output_root, output_root).output_key
|
||||
references = tuple(
|
||||
OutputReference("9", i, "output", sub, name, STYLE, root)
|
||||
for i, (sub, name) in enumerate(files)
|
||||
)
|
||||
db.write_execution_outputs_for_finish(
|
||||
finish, OutputBundle(references, "complete", "reported")
|
||||
)
|
||||
return start.run_id
|
||||
|
||||
|
||||
class TestStoredRunPrompt(RecordedRuns, unittest.TestCase):
|
||||
setUp = history_tests.HistoryDatabaseTests.setUp
|
||||
close_databases = history_tests.HistoryDatabaseTests.close_databases
|
||||
database = history_tests.HistoryDatabaseTests.database
|
||||
|
||||
def setUp(self): # noqa: F811 - the shared temp dir and database list first
|
||||
history_tests.HistoryDatabaseTests.setUp(self)
|
||||
self.db = self.database()
|
||||
self.output = str(Path(self.temp.name) / "output")
|
||||
os.makedirs(os.path.join(self.output, "sub"))
|
||||
self.image = os.path.join(self.output, "sub", "img_00001_.png")
|
||||
Path(self.image).write_bytes(b"png")
|
||||
|
||||
def save(self, text):
|
||||
return self.db.save_prompt(text=text, prompt_hash=generate_prompt_hash(text))
|
||||
|
||||
def attribute(self, path=None):
|
||||
return stored_run_prompt(self.db, path or self.image, self.output)
|
||||
|
||||
def test_one_run_naming_one_stored_prompt_attributes_the_file(self):
|
||||
prompt_id = self.save("an llm prompt")
|
||||
self.record_run(
|
||||
self.db, self.output, [("sub", "img_00001_.png")], ["an llm prompt"]
|
||||
)
|
||||
self.assertEqual(self.attribute(), prompt_id)
|
||||
|
||||
def test_text_that_is_not_stored_attributes_nothing(self):
|
||||
"""The #342 run: raw editor text was empty, nothing was stored."""
|
||||
self.save("a woman holding a coffee cup")
|
||||
self.record_run(self.db, self.output, [("sub", "img_00001_.png")], [""])
|
||||
self.assertIsNone(self.attribute())
|
||||
|
||||
def test_two_runs_reporting_the_same_file_attribute_nothing(self):
|
||||
self.save("first")
|
||||
self.save("second")
|
||||
self.record_run(self.db, self.output, [("sub", "img_00001_.png")], ["first"])
|
||||
self.record_run(
|
||||
self.db, self.output, [("sub", "img_00001_.png")], ["second"], item=1
|
||||
)
|
||||
self.assertIsNone(self.attribute())
|
||||
|
||||
def test_a_file_written_outside_the_run_attributes_nothing(self):
|
||||
self.save("an llm prompt")
|
||||
self.record_run(
|
||||
self.db,
|
||||
self.output,
|
||||
[("sub", "img_00001_.png")],
|
||||
["an llm prompt"],
|
||||
window=(1_000_000, 2_000_000),
|
||||
)
|
||||
self.assertIsNone(self.attribute())
|
||||
|
||||
def test_contexts_naming_different_prompts_attribute_nothing(self):
|
||||
self.save("left")
|
||||
self.save("right")
|
||||
self.record_run(
|
||||
self.db, self.output, [("sub", "img_00001_.png")], ["left", "right"]
|
||||
)
|
||||
self.assertIsNone(self.attribute())
|
||||
|
||||
def test_files_outside_the_output_root_or_unrecorded_attribute_nothing(self):
|
||||
self.save("an llm prompt")
|
||||
self.record_run(
|
||||
self.db, self.output, [("sub", "img_00001_.png")], ["an llm prompt"]
|
||||
)
|
||||
outside = os.path.join(self.temp.name, "img_00001_.png")
|
||||
Path(outside).write_bytes(b"png")
|
||||
self.assertIsNone(self.attribute(outside))
|
||||
other = os.path.join(self.output, "sub", "other.png")
|
||||
Path(other).write_bytes(b"png")
|
||||
self.assertIsNone(self.attribute(other))
|
||||
self.assertIsNone(stored_run_prompt(self.db, self.image, None))
|
||||
self.assertIsNone(stored_run_prompt(self.db, self.image, "relative/root"))
|
||||
|
||||
def test_a_library_without_history_tables_attributes_nothing(self):
|
||||
broken = mock.Mock()
|
||||
broken.model.get_connection.return_value.execute.side_effect = (
|
||||
sqlite3.OperationalError("no such table")
|
||||
)
|
||||
self.assertIsNone(stored_run_prompt(broken, self.image, self.output))
|
||||
|
||||
def test_the_monitor_links_from_history_when_nothing_else_is_certain(self):
|
||||
prompt_id = self.save(LLM_TEXT)
|
||||
self.record_run(self.db, self.output, [("sub", "img_00001_.png")], [LLM_TEXT])
|
||||
handler = ImageGenerationHandler(self.db, make_tracker(self.db))
|
||||
linked = []
|
||||
handler.link_image_to_prompt = lambda path, prompt, meta: linked.append(
|
||||
prompt["id"]
|
||||
)
|
||||
with (
|
||||
mock.patch(
|
||||
"utils.image_monitor.host_output_directory", return_value=self.output
|
||||
),
|
||||
mock.patch.object(
|
||||
handler.metadata_extractor,
|
||||
"extract_metadata",
|
||||
return_value={"prompt": llm_text_graph()},
|
||||
),
|
||||
):
|
||||
handler.process_new_image(self.image)
|
||||
self.assertEqual(linked, [prompt_id])
|
||||
|
||||
|
||||
class TestScanUsesHistory(RecordedRuns, AdminAPITestCase):
|
||||
"""The output scan applies the same rules as the monitor."""
|
||||
|
||||
def linked_prompt(self, path):
|
||||
row = (
|
||||
self.api.db.model.get_connection()
|
||||
.execute(
|
||||
"SELECT prompt_id FROM generated_images WHERE image_path=?",
|
||||
(str(path),),
|
||||
)
|
||||
.fetchone()
|
||||
)
|
||||
return row[0] if row else None
|
||||
|
||||
def write(self, name):
|
||||
path = self.output_dir / name
|
||||
path.write_bytes(b"png")
|
||||
return path
|
||||
|
||||
def test_an_llm_text_file_links_to_the_prompt_its_run_stored(self):
|
||||
db = self.api.db
|
||||
prompt_id = db.save_prompt(
|
||||
text=LLM_TEXT, prompt_hash=generate_prompt_hash(LLM_TEXT)
|
||||
)
|
||||
path = self.write("llm_00001_.png")
|
||||
self.record_run(db, str(self.output_dir), [("", path.name)], [LLM_TEXT])
|
||||
meta = {"prompt": json.dumps(llm_text_graph())}
|
||||
self.assertEqual(self.api._ingest_scanned_file(path, meta), "linked")
|
||||
self.assertEqual(self.linked_prompt(path), prompt_id)
|
||||
|
||||
def test_a_metadata_less_file_links_through_its_recorded_run(self):
|
||||
"""I1: --disable-metadata outputs carry no prompt; history still proves one."""
|
||||
db = self.api.db
|
||||
prompt_id = db.save_prompt(
|
||||
text=LLM_TEXT, prompt_hash=generate_prompt_hash(LLM_TEXT)
|
||||
)
|
||||
path = self.write("bare_00001_.png")
|
||||
self.record_run(db, str(self.output_dir), [("", path.name)], [LLM_TEXT])
|
||||
self.assertEqual(self.api._ingest_scanned_file(path, {}), "linked")
|
||||
self.assertEqual(self.linked_prompt(path), prompt_id)
|
||||
self.assertEqual(self.api._ingest_scanned_file(path, None), "unchanged")
|
||||
detailed = self.api._ingest_scanned_file(path, {}, detailed=True)
|
||||
self.assertEqual(detailed["status"], "unchanged")
|
||||
self.assertTrue(detailed["title"])
|
||||
|
||||
def test_a_metadata_less_file_without_a_recorded_run_is_still_skipped(self):
|
||||
path = self.write("bare_00002_.png")
|
||||
self.assertIsNone(self.api._ingest_scanned_file(path, {}))
|
||||
self.assertIsNone(self.api._ingest_scanned_file(path, {"workflow": {}}))
|
||||
self.assertIsNone(self.linked_prompt(path))
|
||||
|
||||
def test_the_issue_file_stays_unlinked_even_with_its_pair_record(self):
|
||||
"""Resolved with an empty stored positive: before #342 the scan raised."""
|
||||
db = self.api.db
|
||||
db.save_prompt(
|
||||
text="a woman holding a coffee cup",
|
||||
prompt_hash=generate_prompt_hash("a woman holding a coffee cup"),
|
||||
)
|
||||
path = self.write("moss_00009_.png")
|
||||
self.record_run(db, str(self.output_dir), [("", path.name)], [""])
|
||||
evidence = {
|
||||
"version": 1,
|
||||
"nodes": {
|
||||
"18:35": {
|
||||
"records": [
|
||||
{
|
||||
"text": "",
|
||||
"negative_text": "",
|
||||
"positive_text": LLM_TEXT,
|
||||
"negative_effective_text": "",
|
||||
"prepend_text": LLM_TEXT,
|
||||
"append_text": "",
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
}
|
||||
meta = {
|
||||
"prompt": json.dumps(issue_graph()),
|
||||
"comfyassets_prompt_pairs": json.dumps(evidence),
|
||||
}
|
||||
self.assertEqual(self.api._ingest_scanned_file(path, meta), "unresolved")
|
||||
self.assertIsNone(self.linked_prompt(path))
|
||||
self.assertEqual(db.list_prompt_pairs()["pairs"], [])
|
||||
|
||||
|
||||
class TestHostHelpers(unittest.TestCase):
|
||||
"""The host lookups degrade to "no proof", never to an error."""
|
||||
|
||||
def test_host_output_directory(self):
|
||||
import sys
|
||||
import types
|
||||
|
||||
from database import run_attribution
|
||||
|
||||
cases = (
|
||||
(types.SimpleNamespace(get_output_directory=lambda: "/out"), "/out"),
|
||||
(types.SimpleNamespace(get_output_directory=lambda: ""), None),
|
||||
(types.SimpleNamespace(get_output_directory=lambda: 1 / 0), None),
|
||||
)
|
||||
original = sys.modules.get("folder_paths")
|
||||
try:
|
||||
for module, expected in cases:
|
||||
sys.modules["folder_paths"] = module
|
||||
self.assertEqual(run_attribution.host_output_directory(), expected)
|
||||
sys.modules["folder_paths"] = None # import fails: outside ComfyUI
|
||||
self.assertIsNone(run_attribution.host_output_directory())
|
||||
finally:
|
||||
if original is None:
|
||||
sys.modules.pop("folder_paths", None)
|
||||
else:
|
||||
sys.modules["folder_paths"] = original
|
||||
|
||||
def test_run_window_needs_integer_times_and_a_readable_file(self):
|
||||
from database.run_attribution import _written_during
|
||||
|
||||
self.assertFalse(_written_during(__file__, None, 5))
|
||||
self.assertFalse(_written_during(os.path.join("no", "such.png"), 0, 5))
|
||||
|
||||
def test_executing_run_id(self):
|
||||
import sys
|
||||
import types
|
||||
|
||||
from utils.run_keys import executing_run_id
|
||||
|
||||
def host(context):
|
||||
def get_executing_context():
|
||||
if isinstance(context, Exception):
|
||||
raise context
|
||||
return context
|
||||
|
||||
return types.SimpleNamespace(get_executing_context=get_executing_context)
|
||||
|
||||
original = sys.modules.get("comfy_execution.utils")
|
||||
try:
|
||||
for context, expected in (
|
||||
(types.SimpleNamespace(prompt_id="run-1"), "run-1"),
|
||||
(types.SimpleNamespace(prompt_id=""), None),
|
||||
(None, None),
|
||||
(RuntimeError("no context"), None),
|
||||
):
|
||||
sys.modules["comfy_execution.utils"] = host(context)
|
||||
self.assertEqual(executing_run_id(), expected)
|
||||
sys.modules["comfy_execution.utils"] = None
|
||||
self.assertIsNone(executing_run_id())
|
||||
finally:
|
||||
if original is None:
|
||||
sys.modules.pop("comfy_execution.utils", None)
|
||||
else:
|
||||
sys.modules["comfy_execution.utils"] = original
|
||||
|
||||
|
||||
class TestMonitorHistoryRecheck(RecordedRuns, unittest.TestCase):
|
||||
"""I1: a keyless image gets one history-only re-check when its run is sealed."""
|
||||
|
||||
def setUp(self):
|
||||
history_tests.HistoryDatabaseTests.setUp(self)
|
||||
self.db = history_tests.HistoryDatabaseTests.database(self)
|
||||
self.output = str(Path(self.temp.name) / "output")
|
||||
os.makedirs(self.output)
|
||||
self.handler = ImageGenerationHandler(self.db, make_tracker(self.db))
|
||||
self.handler.processing_delay = 0
|
||||
self.handler.settle_interval = 0.01
|
||||
self.addCleanup(self.handler.shutdown)
|
||||
self.linked = []
|
||||
self.handler.link_image_to_prompt = lambda path, prompt, meta: (
|
||||
self.linked.append((os.path.basename(path), prompt["id"]))
|
||||
)
|
||||
patches = (
|
||||
mock.patch(
|
||||
"utils.image_monitor.host_output_directory", return_value=self.output
|
||||
),
|
||||
mock.patch.object(
|
||||
self.handler.metadata_extractor, "extract_metadata", return_value=None
|
||||
),
|
||||
)
|
||||
for patch in patches:
|
||||
patch.start()
|
||||
self.addCleanup(patch.stop)
|
||||
|
||||
close_databases = history_tests.HistoryDatabaseTests.close_databases
|
||||
|
||||
def image(self, name):
|
||||
path = os.path.join(self.output, name)
|
||||
Path(path).write_bytes(b"png")
|
||||
return path
|
||||
|
||||
def save(self, text):
|
||||
return self.db.save_prompt(text=text, prompt_hash=generate_prompt_hash(text))
|
||||
|
||||
def wait_for(self, predicate, timeout=5.0):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if predicate():
|
||||
return True
|
||||
time.sleep(0.01)
|
||||
return predicate()
|
||||
|
||||
def test_the_run_ending_after_processing_links_the_image_once(self):
|
||||
prompt_id = self.save(LLM_TEXT)
|
||||
path = self.image("late_00001_.png")
|
||||
self.handler.process_new_image(path) # the run is still executing
|
||||
self.assertEqual(self.linked, [])
|
||||
self.assertIn(
|
||||
os.path.normcase(os.path.realpath(path)), self.handler.awaiting_history()
|
||||
)
|
||||
self.record_run(self.db, self.output, [("", "late_00001_.png")], [LLM_TEXT])
|
||||
self.assertTrue(self.wait_for(lambda: self.linked))
|
||||
self.assertTrue(self.wait_for(lambda: self.handler.pending_count() == 0))
|
||||
self.assertEqual(self.linked, [("late_00001_.png", prompt_id)])
|
||||
self.assertEqual(self.handler.awaiting_history(), [])
|
||||
# Another run reporting the same name later is not a second re-check
|
||||
self.record_run(
|
||||
self.db, self.output, [("", "late_00001_.png")], [LLM_TEXT], item=1
|
||||
)
|
||||
self.assertTrue(self.wait_for(lambda: self.handler.pending_count() == 0))
|
||||
self.assertEqual(len(self.linked), 1)
|
||||
|
||||
def test_a_seal_during_the_first_pass_is_not_lost(self):
|
||||
"""R1: the run seals while the image's own pass is still pending."""
|
||||
prompt_id = self.save(LLM_TEXT)
|
||||
path = self.image("race_00001_.png")
|
||||
real = self.handler._history_prompt
|
||||
sealed = []
|
||||
|
||||
def history(image_path):
|
||||
result = real(image_path) # before the run's outputs commit: None
|
||||
if not sealed:
|
||||
sealed.append(True)
|
||||
# The writer commits and notifies now; the path is still pending
|
||||
self.record_run(
|
||||
self.db, self.output, [("", "race_00001_.png")], [LLM_TEXT]
|
||||
)
|
||||
return result
|
||||
|
||||
self.handler._history_prompt = history
|
||||
self.assertTrue(self.handler.schedule(path))
|
||||
self.assertTrue(self.wait_for(lambda: self.linked))
|
||||
self.assertTrue(self.wait_for(lambda: self.handler.pending_count() == 0))
|
||||
self.assertEqual(self.linked, [("race_00001_.png", prompt_id)])
|
||||
|
||||
def test_a_seal_while_the_pending_set_is_full_is_rechecked_later(self):
|
||||
from utils import image_monitor
|
||||
|
||||
prompt_id = self.save(LLM_TEXT)
|
||||
path = self.image("full_00001_.png")
|
||||
self.handler.process_new_image(path) # waiting, unlinked
|
||||
blocker = self.image("blocker.png")
|
||||
release = threading.Event()
|
||||
original = self.handler.process_new_image
|
||||
|
||||
def slow(image_path, prompt_snapshot=None):
|
||||
if image_path == blocker:
|
||||
release.wait(5)
|
||||
return original(image_path, prompt_snapshot=prompt_snapshot)
|
||||
|
||||
self.handler.process_new_image = slow
|
||||
with mock.patch.object(image_monitor, "MAX_PENDING_IMAGES", 1):
|
||||
self.assertTrue(self.handler.schedule(blocker))
|
||||
self.assertTrue(self.wait_for(lambda: self.handler.pending_count() == 1))
|
||||
self.record_run(self.db, self.output, [("", "full_00001_.png")], [LLM_TEXT])
|
||||
self.assertEqual(self.linked, [], "not scheduled while the set is full")
|
||||
release.set()
|
||||
self.assertTrue(self.wait_for(lambda: self.linked))
|
||||
self.assertTrue(self.wait_for(lambda: self.handler.pending_count() == 0))
|
||||
self.assertEqual(self.linked, [("full_00001_.png", prompt_id)])
|
||||
|
||||
def test_a_run_naming_nothing_stored_leaves_the_image_unlinked(self):
|
||||
path = self.image("empty_00001_.png")
|
||||
self.handler.process_new_image(path)
|
||||
self.record_run(self.db, self.output, [("", "empty_00001_.png")], [""])
|
||||
self.assertTrue(self.wait_for(lambda: not self.handler.awaiting_history()))
|
||||
self.assertTrue(self.wait_for(lambda: self.handler.pending_count() == 0))
|
||||
self.assertEqual(self.linked, [])
|
||||
|
||||
def test_history_found_at_once_leaves_nothing_awaiting(self):
|
||||
prompt_id = self.save(LLM_TEXT)
|
||||
self.record_run(self.db, self.output, [("", "done_00001_.png")], [LLM_TEXT])
|
||||
self.handler.process_new_image(self.image("done_00001_.png"))
|
||||
self.assertEqual(self.linked, [("done_00001_.png", prompt_id)])
|
||||
self.assertEqual(self.handler.awaiting_history(), [])
|
||||
|
||||
def test_keyed_images_do_not_wait_for_history(self):
|
||||
with mock.patch.object(
|
||||
self.handler.metadata_extractor,
|
||||
"extract_metadata",
|
||||
return_value={"prompt": llm_text_graph()},
|
||||
):
|
||||
self.handler.process_new_image(self.image("keyed_00001_.png"))
|
||||
self.assertEqual(self.handler.awaiting_history(), [])
|
||||
|
||||
def test_the_waiting_set_is_bounded_and_expires(self):
|
||||
from utils import image_monitor
|
||||
|
||||
with mock.patch.object(image_monitor, "MAX_AWAITING_HISTORY", 2):
|
||||
for n in range(3):
|
||||
self.handler.process_new_image(self.image(f"b_{n}.png"))
|
||||
self.assertEqual(
|
||||
[os.path.basename(p) for p in self.handler.awaiting_history()],
|
||||
["b_1.png", "b_2.png"],
|
||||
)
|
||||
later = time.monotonic() + image_monitor.AWAIT_HISTORY_SECONDS + 1
|
||||
with mock.patch.object(image_monitor.time, "monotonic", return_value=later):
|
||||
self.assertEqual(self.handler.awaiting_history(), [])
|
||||
|
||||
def test_shutdown_forgets_waiting_images_and_a_dead_handler_is_dropped(self):
|
||||
import gc
|
||||
|
||||
from database import run_attribution
|
||||
|
||||
self.handler.process_new_image(self.image("gone_00001_.png"))
|
||||
self.handler.shutdown()
|
||||
self.assertEqual(self.handler.awaiting_history(), [])
|
||||
other = ImageGenerationHandler(self.db, make_tracker(self.db))
|
||||
with mock.patch.object(
|
||||
other.metadata_extractor, "extract_metadata", return_value=None
|
||||
):
|
||||
other.process_new_image(self.image("gone_00002_.png"))
|
||||
del other
|
||||
gc.collect()
|
||||
run_attribution.notify_outputs_sealed("run", []) # no dead callback raises
|
||||
|
||||
|
||||
class TestBoundedLookup(RecordedRuns, unittest.TestCase):
|
||||
"""M5: history lookups are batched, and the history schema is unchanged."""
|
||||
|
||||
setUp = history_tests.HistoryDatabaseTests.setUp
|
||||
close_databases = history_tests.HistoryDatabaseTests.close_databases
|
||||
database = history_tests.HistoryDatabaseTests.database
|
||||
|
||||
def output_lookups(self, db, action):
|
||||
statements = []
|
||||
conn = db.model.get_connection()
|
||||
conn.set_trace_callback(statements.append)
|
||||
try:
|
||||
result = action()
|
||||
finally:
|
||||
conn.set_trace_callback(None)
|
||||
return result, [s for s in statements if "FROM execution_run_outputs" in s]
|
||||
|
||||
def test_one_query_per_chunk_not_per_file_and_same_answers(self):
|
||||
from database import run_attribution
|
||||
|
||||
db = self.database()
|
||||
output = str(Path(self.temp.name) / "output")
|
||||
os.makedirs(output)
|
||||
prompt_id = db.save_prompt(
|
||||
text=LLM_TEXT, prompt_hash=generate_prompt_hash(LLM_TEXT)
|
||||
)
|
||||
names = [f"img_{n:05}_.png" for n in range(25)]
|
||||
paths = [os.path.join(output, name) for name in names]
|
||||
for path in paths:
|
||||
Path(path).write_bytes(b"png")
|
||||
self.record_run(db, output, [("", n) for n in names[:10]], [LLM_TEXT])
|
||||
self.record_run(db, output, [("", n) for n in names[5:10]], [""], item=1)
|
||||
with mock.patch.object(run_attribution, "LOOKUP_CHUNK", 10):
|
||||
result, lookups = self.output_lookups(
|
||||
db, lambda: run_attribution.stored_run_prompts(db, paths, output)
|
||||
)
|
||||
self.assertEqual(len(lookups), 3) # 25 names in chunks of 10
|
||||
expected = {path: None for path in paths}
|
||||
expected.update({path: prompt_id for path in paths[:5]})
|
||||
self.assertEqual(result, expected)
|
||||
for path in paths[:12]:
|
||||
self.assertEqual(
|
||||
run_attribution.stored_run_prompt(db, path, output), expected[path]
|
||||
)
|
||||
|
||||
def test_the_history_schema_gains_no_index(self):
|
||||
from database.execution_history_schema import history_schema_error
|
||||
|
||||
db = self.database()
|
||||
conn = db.model.get_connection()
|
||||
indexes = [
|
||||
row[1]
|
||||
for row in conn.execute("PRAGMA index_list(execution_run_outputs)")
|
||||
if row[3] != "pk"
|
||||
]
|
||||
self.assertEqual(indexes, [])
|
||||
self.assertIsNone(history_schema_error(conn, allow_missing=False))
|
||||
|
||||
|
||||
class TestScanBatchLookup(RecordedRuns, AdminAPITestCase):
|
||||
"""M5: a scan batch asks history once per chunk of names, not per file."""
|
||||
|
||||
def test_a_batch_its_metadata_links_never_asks_history(self):
|
||||
"""R3: the batch lookup is made only when a file needs it."""
|
||||
files = []
|
||||
for n in range(3):
|
||||
path = self.output_dir / f"typed_{n:05}_.png"
|
||||
path.write_bytes(b"png")
|
||||
files.append(path)
|
||||
meta = {"prompt": json.dumps(issue_graph(text="typed mountains"))}
|
||||
statements = []
|
||||
conn = self.api.db.model.get_connection()
|
||||
conn.set_trace_callback(statements.append)
|
||||
try:
|
||||
counts = self.api._ingest_scan_batch_sync([(f, meta) for f in files])
|
||||
finally:
|
||||
conn.set_trace_callback(None)
|
||||
self.assertEqual(counts["added"] + counts["linked"], 3, counts)
|
||||
lookups = [s for s in statements if "FROM execution_run_outputs" in s]
|
||||
self.assertEqual(lookups, [])
|
||||
|
||||
def test_metadata_less_batch_links_with_one_history_lookup(self):
|
||||
db = self.api.db
|
||||
prompt_id = db.save_prompt(
|
||||
text=LLM_TEXT, prompt_hash=generate_prompt_hash(LLM_TEXT)
|
||||
)
|
||||
files = []
|
||||
for n in range(6):
|
||||
path = self.output_dir / f"bare_{n:05}_.png"
|
||||
path.write_bytes(b"png")
|
||||
files.append(path)
|
||||
self.record_run(
|
||||
db, str(self.output_dir), [("", f.name) for f in files[:4]], [LLM_TEXT]
|
||||
)
|
||||
statements = []
|
||||
conn = db.model.get_connection()
|
||||
conn.set_trace_callback(statements.append)
|
||||
try:
|
||||
counts = self.api._ingest_scan_batch_sync([(f, {}) for f in files])
|
||||
finally:
|
||||
conn.set_trace_callback(None)
|
||||
lookups = [s for s in statements if "FROM execution_run_outputs" in s]
|
||||
self.assertEqual(len(lookups), 1)
|
||||
self.assertEqual(counts["linked"], 4)
|
||||
linked = {
|
||||
row[0]
|
||||
for row in conn.execute(
|
||||
"SELECT filename FROM generated_images WHERE prompt_id=?",
|
||||
(prompt_id,),
|
||||
)
|
||||
}
|
||||
self.assertEqual(linked, {f.name for f in files[:4]})
|
||||
|
||||
|
||||
class TestLinkedOutputRoot(RecordedRuns, unittest.TestCase):
|
||||
"""R2: ComfyUI's output folder reached through a symlink or junction."""
|
||||
|
||||
setUp = history_tests.HistoryDatabaseTests.setUp
|
||||
close_databases = history_tests.HistoryDatabaseTests.close_databases
|
||||
database = history_tests.HistoryDatabaseTests.database
|
||||
|
||||
def linked_root(self):
|
||||
self.real = str(Path(self.temp.name) / "big_drive" / "output")
|
||||
os.makedirs(os.path.join(self.real, "sub"))
|
||||
link = str(Path(self.temp.name) / "output_link")
|
||||
try:
|
||||
os.symlink(self.real, link, target_is_directory=True)
|
||||
except (OSError, NotImplementedError) as error:
|
||||
self.skipTest(f"symlinks unavailable: {error}")
|
||||
return link
|
||||
|
||||
def test_either_spelling_of_the_file_is_attributed(self):
|
||||
db = self.database()
|
||||
link = self.linked_root()
|
||||
prompt_id = db.save_prompt(
|
||||
text=LLM_TEXT, prompt_hash=generate_prompt_hash(LLM_TEXT)
|
||||
)
|
||||
real_file = os.path.join(self.real, "sub", "img_00001_.png")
|
||||
Path(real_file).write_bytes(b"png")
|
||||
# History keeps the host's spelling of its output folder (the link)
|
||||
self.record_run(db, link, [("sub", "img_00001_.png")], [LLM_TEXT])
|
||||
link_file = os.path.join(link, "sub", "img_00001_.png")
|
||||
self.assertEqual(stored_run_prompt(db, real_file, link), prompt_id)
|
||||
self.assertEqual(stored_run_prompt(db, link_file, link), prompt_id)
|
||||
# The real folder named as the root is another root key: not attributed
|
||||
self.assertIsNone(stored_run_prompt(db, real_file, self.real))
|
||||
|
||||
def test_the_monitor_rechecks_a_file_seen_through_the_real_path(self):
|
||||
db = self.database()
|
||||
link = self.linked_root()
|
||||
prompt_id = db.save_prompt(
|
||||
text=LLM_TEXT, prompt_hash=generate_prompt_hash(LLM_TEXT)
|
||||
)
|
||||
real_file = os.path.join(self.real, "late_00001_.png")
|
||||
Path(real_file).write_bytes(b"png")
|
||||
handler = ImageGenerationHandler(db, make_tracker(db))
|
||||
handler.processing_delay = 0
|
||||
handler.settle_interval = 0.01
|
||||
self.addCleanup(handler.shutdown)
|
||||
linked = []
|
||||
handler.link_image_to_prompt = lambda path, prompt, meta: linked.append(
|
||||
prompt["id"]
|
||||
)
|
||||
with (
|
||||
mock.patch("utils.image_monitor.host_output_directory", return_value=link),
|
||||
mock.patch.object(
|
||||
handler.metadata_extractor, "extract_metadata", return_value=None
|
||||
),
|
||||
):
|
||||
handler.process_new_image(real_file)
|
||||
self.assertEqual(linked, [])
|
||||
self.record_run(db, link, [("", "late_00001_.png")], [LLM_TEXT])
|
||||
deadline = time.monotonic() + 5
|
||||
while not linked and time.monotonic() < deadline:
|
||||
time.sleep(0.01)
|
||||
self.assertEqual(linked, [prompt_id])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,261 @@
|
||||
"""Tests for tests/run_suite.py, the CI test runner and its hang watchdog (#350).
|
||||
|
||||
Each test writes a temporary folder of tiny test modules and runs
|
||||
``python -m tests.run_suite -s <folder> -v`` in a subprocess from the repo
|
||||
root. Every subprocess call has a timeout, so a broken watchdog fails this
|
||||
file instead of hanging the suite. Probes that should hang wait a bounded
|
||||
time, so a watchdog that never fires shows up as a plain assertion failure.
|
||||
"""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import textwrap
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
SUBPROCESS_TIMEOUT = 60
|
||||
WATCHDOG_ENV = ("PM_TEST_HANG_LIMIT", "PM_TEST_SHUTDOWN_LIMIT")
|
||||
|
||||
|
||||
def run_suite(folder, extra_env=None, *args, verbosity=("-v",)):
|
||||
"""Run the runner on ``folder`` and return the completed process."""
|
||||
env = {k: v for k, v in os.environ.items() if k not in WATCHDOG_ENV}
|
||||
# Windows writes a piped stderr in the ANSI codepage; ask for the UTF-8
|
||||
# this helper decodes, so non-ASCII values and paths survive.
|
||||
env["PYTHONIOENCODING"] = "utf-8"
|
||||
env.update(extra_env or {})
|
||||
command = [sys.executable, "-m", "tests.run_suite", "-s", str(folder)]
|
||||
return subprocess.run(
|
||||
[*command, *verbosity, *args],
|
||||
cwd=REPO_ROOT,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
timeout=SUBPROCESS_TIMEOUT,
|
||||
)
|
||||
|
||||
|
||||
class RunSuiteTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self._tmp = tempfile.TemporaryDirectory(prefix="pm-run-suite-")
|
||||
self.addCleanup(self._tmp.cleanup)
|
||||
self.folder = Path(self._tmp.name)
|
||||
|
||||
def write(self, name, source):
|
||||
(self.folder / name).write_text(textwrap.dedent(source), encoding="utf-8")
|
||||
|
||||
def assertDumped(self, result, test_id):
|
||||
detail = result.stderr[-4000:]
|
||||
self.assertNotEqual(result.returncode, 0, detail)
|
||||
self.assertIn(f"start: {test_id}", result.stderr, detail)
|
||||
self.assertIn("Timeout (", result.stderr, detail)
|
||||
self.assertIn("Thread 0x", result.stderr, detail)
|
||||
|
||||
def test_passing_test_exits_zero_and_names_each_start(self):
|
||||
self.write(
|
||||
"test_probe_pass.py",
|
||||
"""
|
||||
import unittest
|
||||
|
||||
class Probe(unittest.TestCase):
|
||||
def test_ok(self):
|
||||
pass
|
||||
""",
|
||||
)
|
||||
result = run_suite(self.folder)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertIn("start: test_probe_pass.Probe.test_ok", result.stderr)
|
||||
|
||||
def test_start_line_is_written_at_quiet_verbosity_too(self):
|
||||
self.write(
|
||||
"test_probe_quiet.py",
|
||||
"""
|
||||
import unittest
|
||||
|
||||
class Probe(unittest.TestCase):
|
||||
def test_ok(self):
|
||||
pass
|
||||
""",
|
||||
)
|
||||
result = run_suite(self.folder, None, "-q")
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertIn("start: test_probe_quiet.Probe.test_ok", result.stderr)
|
||||
|
||||
def test_start_line_begins_its_own_line_at_default_verbosity(self):
|
||||
self.write(
|
||||
"test_probe_dots.py",
|
||||
"""
|
||||
import unittest
|
||||
|
||||
class Probe(unittest.TestCase):
|
||||
def test_a(self):
|
||||
pass
|
||||
|
||||
def test_b(self):
|
||||
pass
|
||||
""",
|
||||
)
|
||||
result = run_suite(self.folder, verbosity=())
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
lines = result.stderr.splitlines()
|
||||
for name in ("test_a", "test_b"):
|
||||
self.assertIn(f"start: test_probe_dots.Probe.{name}", lines)
|
||||
|
||||
def test_warnings_are_shown_as_with_unittest_discover(self):
|
||||
self.write(
|
||||
"test_probe_warn.py",
|
||||
"""
|
||||
import unittest
|
||||
import warnings
|
||||
|
||||
class Probe(unittest.TestCase):
|
||||
def test_warns(self):
|
||||
warnings.warn("probe-deprecated", DeprecationWarning)
|
||||
""",
|
||||
)
|
||||
env = {"PYTHONWARNINGS": ""}
|
||||
result = run_suite(self.folder, env)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertIn("probe-deprecated", result.stderr)
|
||||
|
||||
def test_failing_test_exits_one(self):
|
||||
self.write(
|
||||
"test_probe_fail.py",
|
||||
"""
|
||||
import unittest
|
||||
|
||||
class Probe(unittest.TestCase):
|
||||
def test_fails(self):
|
||||
self.fail("expected")
|
||||
""",
|
||||
)
|
||||
result = run_suite(self.folder)
|
||||
self.assertEqual(result.returncode, 1, result.stderr)
|
||||
|
||||
def test_pattern_option_selects_modules(self):
|
||||
self.write(
|
||||
"check_probe.py",
|
||||
"""
|
||||
import unittest
|
||||
|
||||
class Probe(unittest.TestCase):
|
||||
def test_fails(self):
|
||||
self.fail("selected by -p")
|
||||
""",
|
||||
)
|
||||
self.assertEqual(run_suite(self.folder).returncode, 5)
|
||||
result = run_suite(self.folder, None, "-p", "check_*.py")
|
||||
self.assertEqual(result.returncode, 1, result.stderr)
|
||||
|
||||
def test_empty_folder_exits_five(self):
|
||||
result = run_suite(self.folder)
|
||||
self.assertEqual(result.returncode, 5, result.stderr)
|
||||
|
||||
def test_repo_root_modules_import_as_with_unittest_discover(self):
|
||||
self.write(
|
||||
"test_probe_import.py",
|
||||
"""
|
||||
import unittest
|
||||
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
class Probe(unittest.TestCase):
|
||||
def test_import(self):
|
||||
same = generate_prompt_hash(" A ") == generate_prompt_hash("a")
|
||||
self.assertTrue(same)
|
||||
""",
|
||||
)
|
||||
result = run_suite(self.folder)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
|
||||
def test_hung_test_dumps_every_thread_and_exits(self):
|
||||
self.write(
|
||||
"test_probe_hang.py",
|
||||
"""
|
||||
import time
|
||||
import unittest
|
||||
|
||||
class Probe(unittest.TestCase):
|
||||
def test_hangs(self):
|
||||
time.sleep(30)
|
||||
""",
|
||||
)
|
||||
result = run_suite(self.folder, {"PM_TEST_HANG_LIMIT": "2"})
|
||||
self.assertDumped(result, "test_probe_hang.Probe.test_hangs")
|
||||
|
||||
def test_hang_in_async_teardown_dumps_and_exits(self):
|
||||
self.write(
|
||||
"test_probe_teardown.py",
|
||||
"""
|
||||
import threading
|
||||
import unittest
|
||||
|
||||
class Probe(unittest.IsolatedAsyncioTestCase):
|
||||
async def asyncTearDown(self):
|
||||
threading.Event().wait(30)
|
||||
|
||||
async def test_body_passes(self):
|
||||
pass
|
||||
""",
|
||||
)
|
||||
result = run_suite(self.folder, {"PM_TEST_HANG_LIMIT": "2"})
|
||||
self.assertDumped(result, "test_probe_teardown.Probe.test_body_passes")
|
||||
|
||||
def test_hang_in_event_loop_shutdown_after_ok_dumps_and_exits(self):
|
||||
# The #349 shape: the test passes ("ok" is printed), then the loop's
|
||||
# shutdown waits for a default-executor thread that never finishes.
|
||||
self.write(
|
||||
"test_probe_loop.py",
|
||||
"""
|
||||
import asyncio
|
||||
import threading
|
||||
import unittest
|
||||
|
||||
class Probe(unittest.IsolatedAsyncioTestCase):
|
||||
async def test_leaves_executor_busy(self):
|
||||
loop = asyncio.get_running_loop()
|
||||
loop.run_in_executor(None, threading.Event().wait, 30)
|
||||
""",
|
||||
)
|
||||
result = run_suite(self.folder, {"PM_TEST_HANG_LIMIT": "2"})
|
||||
self.assertDumped(result, "test_probe_loop.Probe.test_leaves_executor_busy")
|
||||
start = result.stderr.index("start: test_probe_loop")
|
||||
self.assertIn("ok", result.stderr[start:], result.stderr[-4000:])
|
||||
|
||||
def test_hang_at_interpreter_exit_dumps_and_exits(self):
|
||||
self.write(
|
||||
"test_probe_exit.py",
|
||||
"""
|
||||
import threading
|
||||
import unittest
|
||||
|
||||
class Probe(unittest.TestCase):
|
||||
def test_leaves_thread(self):
|
||||
threading.Thread(target=threading.Event().wait, args=(30,)).start()
|
||||
""",
|
||||
)
|
||||
result = run_suite(self.folder, {"PM_TEST_SHUTDOWN_LIMIT": "2"})
|
||||
detail = result.stderr[-4000:]
|
||||
self.assertNotEqual(result.returncode, 0, detail)
|
||||
self.assertIn("ok", result.stderr, detail)
|
||||
self.assertIn("Timeout (", result.stderr, detail)
|
||||
|
||||
def test_invalid_limit_is_a_usage_error(self):
|
||||
for name in WATCHDOG_ENV:
|
||||
for value in ("abc", "0", "-3", "1.5", "\u00b2"):
|
||||
with self.subTest(name=name, value=value):
|
||||
result = run_suite(self.folder, {name: value})
|
||||
self.assertEqual(result.returncode, 2, result.stderr)
|
||||
lines = result.stderr.strip().splitlines()
|
||||
self.assertEqual(len(lines), 1, result.stderr)
|
||||
self.assertIn(name, lines[0])
|
||||
self.assertIn(repr(value), lines[0])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+22
-10
@@ -60,8 +60,8 @@ class SystemAPITestCase(AdminAPITestCase):
|
||||
self.assertEqual(response.status, status, await response.text())
|
||||
return await response.json()
|
||||
|
||||
async def restore_snapshot(self):
|
||||
"""Restore the live database from a copy of itself through the legacy route."""
|
||||
def snapshot_form(self):
|
||||
"""A multipart upload of a copy of the live database."""
|
||||
from contextlib import closing
|
||||
|
||||
from aiohttp import FormData
|
||||
@@ -74,8 +74,12 @@ class SystemAPITestCase(AdminAPITestCase):
|
||||
upload = handle.read()
|
||||
form = FormData()
|
||||
form.add_field("database_file", upload, filename="upload.db")
|
||||
return form
|
||||
|
||||
async def restore_snapshot(self):
|
||||
"""Restore the live database from a copy of itself through the legacy route."""
|
||||
response = await self.client.request(
|
||||
"POST", "/prompt_manager/restore", data=form
|
||||
"POST", "/prompt_manager/restore", data=self.snapshot_form()
|
||||
)
|
||||
self.assertEqual(response.status, 200, await response.text())
|
||||
|
||||
@@ -193,8 +197,10 @@ class TestHealthRoute(SystemAPITestCase):
|
||||
overview = await self.json("GET", "/prompt_manager/system/overview")
|
||||
self.assertIsNone(overview["health"])
|
||||
|
||||
async def test_check_finishing_after_a_restore_is_not_kept(self):
|
||||
# A check that started on the old database describes it, not the new one.
|
||||
async def test_restore_refused_while_a_check_runs(self):
|
||||
# #249 S8 carry-over: a restore and the health check never interleave
|
||||
# (the restore is exclusive), so no check outlives the database it
|
||||
# read; a restore after the check forgets its report.
|
||||
self.save("Any")
|
||||
release = threading.Event()
|
||||
entered = threading.Event()
|
||||
@@ -209,18 +215,24 @@ class TestHealthRoute(SystemAPITestCase):
|
||||
body = await self.json("POST", "/prompt_manager/system/health", 202)
|
||||
loop = asyncio.get_running_loop()
|
||||
await loop.run_in_executor(None, entered.wait, WAIT)
|
||||
await self.restore_snapshot()
|
||||
refused = await self.client.post(
|
||||
"/prompt_manager/restore", data=self.snapshot_form()
|
||||
)
|
||||
self.assertEqual(refused.status, 409)
|
||||
self.assertEqual(
|
||||
await refused.json(),
|
||||
{"success": False, "error": "Health check already running"},
|
||||
)
|
||||
release.set()
|
||||
job = self.api._jobs.get(body["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(job.task), WAIT)
|
||||
self.assertEqual(job.state, "finished")
|
||||
body = await self.json("GET", "/prompt_manager/system/health")
|
||||
self.assertEqual(body, {"success": True, "report": None})
|
||||
# A check started on the restored database is kept again.
|
||||
await self.run_health()
|
||||
self.assertIsNotNone(
|
||||
(await self.json("GET", "/prompt_manager/system/health"))["report"]
|
||||
)
|
||||
await self.restore_snapshot()
|
||||
body = await self.json("GET", "/prompt_manager/system/health")
|
||||
self.assertEqual(body, {"success": True, "report": None})
|
||||
|
||||
async def test_health_sources_resolve_off_the_event_loop(self):
|
||||
loop_thread = threading.get_ident()
|
||||
|
||||
@@ -477,11 +477,20 @@ class TestPendingFirstUse(unittest.TestCase):
|
||||
class FakeTracker:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
self.runs = []
|
||||
|
||||
def set_current_prompt(self, prompt_text, additional_data=None, push_to_queue=True):
|
||||
def set_current_prompt(
|
||||
self, prompt_text, additional_data=None, push_to_queue=True, **run
|
||||
):
|
||||
self.calls.append({"text": prompt_text, "push_to_queue": push_to_queue})
|
||||
self.runs.append(run)
|
||||
return "exec-1"
|
||||
|
||||
def mark_empty(self, run_key=None, run_id=None, push_to_queue=False):
|
||||
self.calls.append(
|
||||
{"empty": True, "run_key": run_key, "push_to_queue": push_to_queue}
|
||||
)
|
||||
|
||||
|
||||
class TestNodeRoles(DbTestCase):
|
||||
def _node(self):
|
||||
|
||||
Vendored
+28
-2
@@ -1,5 +1,6 @@
|
||||
// Structural checks on .github/workflows/viewerjs-check.yml. The tooling has no YAML parser, so these
|
||||
// read the workflow's fixed two-space layout: jobs at two spaces, their keys at four, steps at six.
|
||||
// Structural checks on .github/workflows/viewerjs-check.yml, plus the time limits of every CI workflow.
|
||||
// The tooling has no YAML parser, so these read the workflow's fixed two-space layout: jobs at two
|
||||
// spaces, their keys at four, steps at six.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
@@ -221,3 +222,28 @@ test('browser job timeouts keep a real margin over measured runs', () => {
|
||||
assert.ok(minutes >= 20, `${id} allows at least 20 minutes, since runner speed varies`);
|
||||
}
|
||||
});
|
||||
|
||||
// Every job is bounded (#350): a hang ends at a measured limit instead of GitHub's 6-hour default.
|
||||
const BOUNDED_WORKFLOWS = ['test.yml', 'code-quality.yml', 'viewerjs-check.yml'];
|
||||
const readWorkflow = name => readFileSync(new URL(`../../.github/workflows/${name}`, import.meta.url), 'utf8');
|
||||
|
||||
test('every job in the CI workflows has a numeric timeout-minutes', () => {
|
||||
for (const name of BOUNDED_WORKFLOWS) {
|
||||
const found = jobs(readWorkflow(name));
|
||||
assert.ok(found.size > 0, `${name} has jobs`);
|
||||
for (const [id, body] of found) {
|
||||
const limit = /^ {4}timeout-minutes: (\d+)\s*$/m.exec(body);
|
||||
assert.ok(limit, `${name} job ${id} sets a numeric timeout-minutes`);
|
||||
assert.ok(Number(limit[1]) >= 5, `${name} job ${id} allows at least 5 minutes`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('the Python matrix and coverage run the suite through the watchdog runner', () => {
|
||||
const found = jobs(readWorkflow('test.yml'));
|
||||
assert.match(found.get('test'), /^ {8}run: python -m tests\.run_suite -v$/m);
|
||||
assert.match(found.get('coverage'), /^ {10}python -m coverage run -m tests\.run_suite$/m);
|
||||
for (const id of ['test', 'coverage']) {
|
||||
assert.doesNotMatch(found.get(id), /unittest discover/, `${id} no longer runs the bare unittest discover`);
|
||||
}
|
||||
});
|
||||
|
||||
Vendored
+10
-3
@@ -15,26 +15,33 @@ test('real-ComfyUI suite registers Classic and Nodes 2.0 startup projects', () =
|
||||
assert.match(output, /\[nodes2-start\]/);
|
||||
});
|
||||
|
||||
test('execution script discovers only the new matrix and preserves all 78 original stock cases plus the native editor host case', () => {
|
||||
test('execution script discovers only the new matrix and preserves all 78 original stock cases plus the native editor and renderer host cases', () => {
|
||||
const manifest = JSON.parse(readFileSync(new URL('../../tools/vendor/package.json', import.meta.url), 'utf8'));
|
||||
assert.equal(manifest.scripts['test:executions'], 'playwright test --config playwright.execution.config.mjs');
|
||||
const list = config => execFileSync(process.execPath, ['tools/vendor/node_modules/@playwright/test/cli.js',
|
||||
'test', '--config', config, '--list'], { cwd: root, encoding: 'utf8',
|
||||
env: { ...process.env, PM_COMFY_ROOT: '/unused-pinned-checkout' } });
|
||||
const stock = list('tools/vendor/comfyui.config.mjs');
|
||||
assert.match(stock, /Total: 82 tests in 8 files/); // + local Inspect host case (inspect-sender.spec.mjs, 2 projects)
|
||||
// + local Inspect host case (inspect-sender.spec.mjs, 2 projects), + the S10 renderer case (prompt-bridge.spec.mjs, 2 projects)
|
||||
// + #343 linked text: five node types, PromptSearchList, the pre-#343 workflows, legacy Primitive, moved link,
|
||||
// paste/clone/undo, wildcard Primitive, subgraph Primitive (text-link.spec.mjs, 2 projects)
|
||||
assert.match(stock, /Total: 110 tests in 9 files/);
|
||||
assert.doesNotMatch(stock, /execution-history/);
|
||||
assert.doesNotMatch(stock, /execution-outputs/);
|
||||
assert.doesNotMatch(stock, /library-location/); // it starts its own fresh host (#249 S6b)
|
||||
assert.doesNotMatch(stock, /image-attribution/); // needs the owned fixture pack (#342)
|
||||
const executions = list('tools/vendor/playwright.execution.config.mjs');
|
||||
assert.match(executions, /\[classic-start\]/);
|
||||
assert.match(executions, /\[nodes2-start\]/);
|
||||
assert.match(executions, /Total: 48 tests in 3 files/); // + the fresh-host library location case
|
||||
// + the fresh-host library location case, + #342 image attribution, + #343 linked text
|
||||
assert.match(executions, /Total: 90 tests in 5 files/);
|
||||
for (const project of ['classic-start', 'nodes2-start']) {
|
||||
const cases = executions.split('\n').filter(line => line.includes(`[${project}]`));
|
||||
assert.equal(cases.filter(line => line.includes('execution-history.spec.mjs')).length, 15);
|
||||
assert.equal(cases.filter(line => line.includes('execution-outputs.spec.mjs')).length, 8);
|
||||
assert.equal(cases.filter(line => line.includes('library-location.spec.mjs')).length, 1);
|
||||
assert.equal(cases.filter(line => line.includes('image-attribution.spec.mjs')).length, 2);
|
||||
assert.equal(cases.filter(line => line.includes('execution-text-link.spec.mjs')).length, 19);
|
||||
}
|
||||
assert.doesNotMatch(executions, /prompt-manager.spec/);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,330 @@
|
||||
"""Review, and on request repair, image links that nothing certain supports (#342).
|
||||
|
||||
Before #342 the image monitor linked an image it could not identify to the
|
||||
newest prompt in the database or to a stale tracker entry. This tool judges the
|
||||
selected images again by today's rules, using only what is certain:
|
||||
|
||||
- the image's own metadata, as the monitor reads it;
|
||||
- the output scan's resolver, and its legacy browser-parser floor (#170);
|
||||
- recorded execution history (``database.run_attribution``).
|
||||
|
||||
Each image gets one verdict:
|
||||
|
||||
``keep`` a certain source names the prompt it is linked to
|
||||
``relink`` every certain source names one other stored prompt
|
||||
``unlink`` nothing certain names a prompt (the link was a guess)
|
||||
``conflict`` certain sources disagree: decide by hand
|
||||
``missing`` the file is gone: leave it to Maintenance's missing links
|
||||
|
||||
Dry run by default: the database is copied (SQLite backup from a read-only
|
||||
connection) into a temporary folder and judged there, so the library is never
|
||||
written. ``--apply`` writes the ``relink`` and ``unlink`` verdicts of the
|
||||
selected images, after saving a backup beside the database
|
||||
(``<db>.pre-relink_<stamp>``). Only ids named on the command line are ever
|
||||
changed, and only while each is still linked to the prompt it was judged on.
|
||||
|
||||
What ``--apply`` does (APPLY_EFFECTS below, printed by every run):
|
||||
|
||||
- ``relink`` points the image's link record at the other prompt and drops its
|
||||
pair links (they named the old positive).
|
||||
- ``unlink`` removes only the image's link record (its ``generated_images`` row
|
||||
with the metadata snapshot stored in it, and its pair links). That is how the
|
||||
app represents an unlinked image, exactly like a file the scan could not
|
||||
attribute: the file stays on disk and in the gallery as unlinked, and its
|
||||
favorite star and tags (kept by file, not by link) are untouched.
|
||||
|
||||
A batch item linked through the monitor's queue (PromptSearchList) can't be
|
||||
proven after the fact and reads ``unlink``: review before applying.
|
||||
|
||||
python3 tools/review_image_links.py <db> --output-dir <ComfyUI output> \\
|
||||
--ids 58005-58010
|
||||
python3 tools/review_image_links.py <db> --output-dir <dir> --prompt 5278
|
||||
python3 tools/review_image_links.py <db> --output-dir <dir> \\
|
||||
--ids 58005-58010 --apply
|
||||
|
||||
Stop ComfyUI before ``--apply``. Run from this repository with its venv; the
|
||||
package's logger may write to its usual log file.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
from contextlib import closing
|
||||
from pathlib import Path
|
||||
|
||||
_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_ROOT))
|
||||
|
||||
from database.maintenance import _delete_ids # noqa: E402
|
||||
from database.operations import PromptDatabase # noqa: E402
|
||||
from database.pair_ingestion import blank_positive # noqa: E402
|
||||
from database.run_attribution import stored_run_prompt # noqa: E402
|
||||
from utils.generation_params import ( # noqa: E402
|
||||
PLACEHOLDERS,
|
||||
GenerationParamsError,
|
||||
extract_generation_params,
|
||||
)
|
||||
from utils.hashing import generate_prompt_hash # noqa: E402
|
||||
from utils.image_monitor import ImageGenerationHandler # noqa: E402
|
||||
from utils.prompt_pair_resolver import resolve_prompt_pair # noqa: E402
|
||||
|
||||
VERDICTS = ("keep", "relink", "unlink", "conflict", "missing")
|
||||
APPLY_EFFECTS = (
|
||||
"--apply, for ids named with --ids only, while each is still linked to the "
|
||||
"prompt it was judged on:\n"
|
||||
" relink: points the image's link record at the other prompt and drops "
|
||||
"its pair links;\n"
|
||||
" unlink: removes only the image's link record (its stored metadata "
|
||||
"snapshot and pair links go with it); the file stays on disk and in the "
|
||||
"gallery as unlinked, with its favorite and tags.\n"
|
||||
)
|
||||
# Ids per query (well under SQLite's bound-parameter limit)
|
||||
ID_BATCH = 500
|
||||
|
||||
|
||||
def parse_ids(text):
|
||||
"""``"1,4-6"`` -> ``[1, 4, 5, 6]``; raises ValueError on anything else."""
|
||||
ids = set()
|
||||
for part in (text or "").split(","):
|
||||
part = part.strip()
|
||||
if not part:
|
||||
continue
|
||||
low, _, high = part.partition("-")
|
||||
first, last = int(low), int(high or low)
|
||||
if first < 1 or last < first or last - first > 100_000:
|
||||
raise ValueError(f"Bad id range: {part}")
|
||||
ids.update(range(first, last + 1))
|
||||
return sorted(ids)
|
||||
|
||||
|
||||
def _stored_id(db, text):
|
||||
if not isinstance(text, str) or not text.strip():
|
||||
return None
|
||||
row = db.get_prompt_by_hash(generate_prompt_hash(text.strip()))
|
||||
return row["id"] if row else None
|
||||
|
||||
|
||||
def _parser_text(metadata):
|
||||
"""The legacy browser parser's positive prompt (the scan's floor), or None."""
|
||||
records = {key: metadata.get(key) for key in ("prompt", "workflow")}
|
||||
try:
|
||||
positive = extract_generation_params(records)["positivePrompt"]
|
||||
except (GenerationParamsError, ValueError, TypeError, KeyError):
|
||||
return None
|
||||
if not isinstance(positive, str) or positive == PLACEHOLDERS["positivePrompt"]:
|
||||
return None
|
||||
return positive.strip() or None
|
||||
|
||||
|
||||
def certain_prompts(db, handler, path, metadata, output_dir):
|
||||
"""``{source: prompt id}`` of every certain source that names a stored prompt."""
|
||||
found = {}
|
||||
if metadata:
|
||||
state, prompt = handler._find_prompt_from_metadata(metadata)
|
||||
if state == "linked" and prompt:
|
||||
found["metadata"] = prompt["id"]
|
||||
resolution = resolve_prompt_pair(metadata)
|
||||
if resolution["state"] == "resolved" and not blank_positive(resolution):
|
||||
positive = resolution["positive"]
|
||||
scanned = _stored_id(db, positive.get("storage_text", positive["text"]))
|
||||
else:
|
||||
scanned = _stored_id(db, _parser_text(metadata))
|
||||
if scanned is not None:
|
||||
found["scan"] = scanned
|
||||
recorded = stored_run_prompt(db, path, output_dir)
|
||||
if recorded is not None:
|
||||
found["history"] = recorded
|
||||
return found
|
||||
|
||||
|
||||
def judge(db, handler, row, output_dir):
|
||||
"""One image's verdict dict."""
|
||||
path = row["image_path"]
|
||||
verdict = {
|
||||
"image_id": row["id"],
|
||||
"file": path,
|
||||
"linked_prompt": row["prompt_id"],
|
||||
"sources": {},
|
||||
"target": None,
|
||||
}
|
||||
if not os.path.isfile(path):
|
||||
return {**verdict, "verdict": "missing"}
|
||||
try:
|
||||
metadata = handler.metadata_extractor.extract_metadata(path)
|
||||
except Exception:
|
||||
metadata = None
|
||||
sources = certain_prompts(db, handler, path, metadata, output_dir)
|
||||
named = set(sources.values())
|
||||
verdict["sources"] = sources
|
||||
if not named:
|
||||
return {**verdict, "verdict": "unlink"}
|
||||
if len(named) > 1:
|
||||
return {**verdict, "verdict": "conflict"}
|
||||
(target,) = named
|
||||
if target == row["prompt_id"]:
|
||||
return {**verdict, "verdict": "keep", "target": target}
|
||||
return {**verdict, "verdict": "relink", "target": target}
|
||||
|
||||
|
||||
def selected_rows(db, ids, prompt_id):
|
||||
"""The named images and every image of ``prompt_id``, by id."""
|
||||
conn = db.model.get_connection()
|
||||
columns = "SELECT id, prompt_id, image_path FROM generated_images "
|
||||
rows = {}
|
||||
for start in range(0, len(ids), ID_BATCH):
|
||||
batch = ids[start : start + ID_BATCH]
|
||||
marks = ",".join("?" for _ in batch)
|
||||
query = columns + f"WHERE id IN ({marks})" # only ? marks
|
||||
rows.update((row["id"], row) for row in conn.execute(query, batch))
|
||||
if prompt_id is not None:
|
||||
found = conn.execute(columns + "WHERE prompt_id = ?", (prompt_id,))
|
||||
rows.update((row["id"], row) for row in found)
|
||||
return [rows[key] for key in sorted(rows)]
|
||||
|
||||
|
||||
def review(db, ids, prompt_id, output_dir):
|
||||
handler = ImageGenerationHandler(db, None)
|
||||
return [
|
||||
judge(db, handler, row, output_dir) for row in selected_rows(db, ids, prompt_id)
|
||||
]
|
||||
|
||||
|
||||
def apply(db, verdicts, ids):
|
||||
"""Write the relink/unlink verdicts of explicitly named ids; returns counts.
|
||||
|
||||
An image whose link changed since it was judged (now another prompt, or
|
||||
gone) is skipped.
|
||||
"""
|
||||
allowed = set(ids)
|
||||
chosen = [
|
||||
v
|
||||
for v in verdicts
|
||||
if v["verdict"] in ("relink", "unlink") and v["image_id"] in allowed
|
||||
]
|
||||
with db._pair_transaction() as conn:
|
||||
current = [
|
||||
item
|
||||
for item in chosen
|
||||
if (
|
||||
row := conn.execute(
|
||||
"SELECT prompt_id FROM generated_images WHERE id=?",
|
||||
(item["image_id"],),
|
||||
).fetchone()
|
||||
)
|
||||
and row[0] == item["linked_prompt"]
|
||||
]
|
||||
relink = [item for item in current if item["verdict"] == "relink"]
|
||||
for item in relink:
|
||||
# A pair link records the old positive: it goes with the old link
|
||||
conn.execute(
|
||||
"DELETE FROM image_prompt_pairs WHERE image_id=?", (item["image_id"],)
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE generated_images SET prompt_id=? WHERE id=? AND prompt_id=?",
|
||||
(item["target"], item["image_id"], item["linked_prompt"]),
|
||||
)
|
||||
removed = _delete_ids(
|
||||
conn,
|
||||
"generated_images",
|
||||
[item["image_id"] for item in current if item["verdict"] == "unlink"],
|
||||
)
|
||||
return {
|
||||
"relinked": len(relink),
|
||||
"unlinked": removed,
|
||||
"skipped": len(chosen) - len(current),
|
||||
}
|
||||
|
||||
|
||||
def backup_database(source, target):
|
||||
"""SQLite online backup of ``source`` (opened read-only) into ``target``."""
|
||||
uri = Path(os.path.abspath(source)).as_uri() + "?mode=ro"
|
||||
with (
|
||||
closing(sqlite3.connect(uri, uri=True)) as src,
|
||||
closing(sqlite3.connect(target)) as dst,
|
||||
):
|
||||
src.backup(dst)
|
||||
|
||||
|
||||
def _print(verdicts, out):
|
||||
for item in verdicts:
|
||||
sources = ", ".join(f"{k}={v}" for k, v in sorted(item["sources"].items()))
|
||||
target = f" -> {item['target']}" if item["verdict"] == "relink" else ""
|
||||
linked = item["linked_prompt"]
|
||||
out.write(
|
||||
f"{item['image_id']:>8} {item['verdict']:<8} prompt {linked}{target}"
|
||||
f" [{sources or 'nothing certain'}] {item['file']}\n"
|
||||
)
|
||||
counts = {name: sum(v["verdict"] == name for v in verdicts) for name in VERDICTS}
|
||||
out.write(" ".join(f"{name}: {count}" for name, count in counts.items()) + "\n")
|
||||
|
||||
|
||||
def main(argv=None, out=sys.stdout):
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0])
|
||||
parser.add_argument("database", help="the library database (prompts.db)")
|
||||
parser.add_argument(
|
||||
"--output-dir", required=True, help="ComfyUI's output directory (history)"
|
||||
)
|
||||
parser.add_argument("--ids", default="", help="image ids, e.g. 58005-58010,58020")
|
||||
parser.add_argument("--prompt", type=int, help="every image linked to this prompt")
|
||||
parser.add_argument(
|
||||
"--apply", action="store_true", help="write relink/unlink for --ids images"
|
||||
)
|
||||
parser.add_argument("--json", action="store_true", help="print JSON verdicts")
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
ids = parse_ids(args.ids)
|
||||
except ValueError as error:
|
||||
parser.error(str(error))
|
||||
if not ids and args.prompt is None:
|
||||
parser.error("select images with --ids and/or --prompt")
|
||||
if args.apply and not ids:
|
||||
parser.error("--apply changes only images named with --ids")
|
||||
if not os.path.isfile(args.database):
|
||||
parser.error(f"no database at {args.database}")
|
||||
|
||||
if not args.apply:
|
||||
with tempfile.TemporaryDirectory() as scratch:
|
||||
copy = os.path.join(scratch, "review.db")
|
||||
backup_database(args.database, copy)
|
||||
db = PromptDatabase(copy)
|
||||
try:
|
||||
verdicts = review(db, ids, args.prompt, args.output_dir)
|
||||
finally:
|
||||
db.close_all()
|
||||
_report(verdicts, args.json, out)
|
||||
out.write(APPLY_EFFECTS)
|
||||
out.write("Dry run: nothing was changed.\n")
|
||||
return 0
|
||||
|
||||
stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
backup = f"{args.database}.pre-relink_{stamp}"
|
||||
backup_database(args.database, backup)
|
||||
out.write(f"Backup: {backup}\n")
|
||||
db = PromptDatabase(args.database)
|
||||
try:
|
||||
verdicts = review(db, ids, args.prompt, args.output_dir)
|
||||
_report(verdicts, args.json, out)
|
||||
result = apply(db, verdicts, ids)
|
||||
finally:
|
||||
db.close_all()
|
||||
out.write(
|
||||
f"Applied: {result['relinked']} relinked, {result['unlinked']} unlinked, "
|
||||
f"{result['skipped']} skipped (link changed since review)\n"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def _report(verdicts, as_json, out):
|
||||
if as_json:
|
||||
out.write(json.dumps(verdicts, indent=2, sort_keys=True) + "\n")
|
||||
else:
|
||||
_print(verdicts, out)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Vendored
+2
-1
@@ -19,7 +19,8 @@ const servers = [{
|
||||
|
||||
export default defineConfig({
|
||||
testDir: '../../tests/comfyui',
|
||||
testIgnore: ['execution-history.spec.mjs', 'execution-outputs.spec.mjs', 'library-location.spec.mjs'],
|
||||
testIgnore: ['execution-history.spec.mjs', 'execution-outputs.spec.mjs', 'execution-text-link.spec.mjs', 'library-location.spec.mjs',
|
||||
'image-attribution.spec.mjs'],
|
||||
outputDir: './test-results/comfyui',
|
||||
timeout: 60000,
|
||||
workers: 1,
|
||||
|
||||
+40
-4
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"admin.js": {
|
||||
"lines": 32,
|
||||
"branches": 58,
|
||||
"branches": 61,
|
||||
"functions": 27,
|
||||
"statements": 32
|
||||
},
|
||||
@@ -11,6 +11,12 @@
|
||||
"functions": 98,
|
||||
"statements": 98
|
||||
},
|
||||
"cache-ownership.js": {
|
||||
"lines": 94,
|
||||
"branches": 89,
|
||||
"functions": 98,
|
||||
"statements": 94
|
||||
},
|
||||
"classic-node.js": {
|
||||
"lines": 98,
|
||||
"branches": 79,
|
||||
@@ -43,7 +49,7 @@
|
||||
},
|
||||
"gallery.js": {
|
||||
"lines": 45,
|
||||
"branches": 64,
|
||||
"branches": 66,
|
||||
"functions": 48,
|
||||
"statements": 45
|
||||
},
|
||||
@@ -143,6 +149,12 @@
|
||||
"functions": 56,
|
||||
"statements": 57
|
||||
},
|
||||
"text-links.js": {
|
||||
"lines": 98,
|
||||
"branches": 98,
|
||||
"functions": 98,
|
||||
"statements": 98
|
||||
},
|
||||
"theme-bridge.js": {
|
||||
"lines": 90,
|
||||
"branches": 53,
|
||||
@@ -157,7 +169,7 @@
|
||||
},
|
||||
"workspace/activity-job-view.mjs": {
|
||||
"lines": 97,
|
||||
"branches": 86,
|
||||
"branches": 87,
|
||||
"functions": 98,
|
||||
"statements": 97
|
||||
},
|
||||
@@ -169,7 +181,7 @@
|
||||
},
|
||||
"workspace/activity-scan.mjs": {
|
||||
"lines": 96,
|
||||
"branches": 78,
|
||||
"branches": 87,
|
||||
"functions": 98,
|
||||
"statements": 96
|
||||
},
|
||||
@@ -407,6 +419,12 @@
|
||||
"functions": 98,
|
||||
"statements": 98
|
||||
},
|
||||
"workspace/maintenance-review.mjs": {
|
||||
"lines": 94,
|
||||
"branches": 86,
|
||||
"functions": 90,
|
||||
"statements": 94
|
||||
},
|
||||
"workspace/media-viewer.mjs": {
|
||||
"lines": 98,
|
||||
"branches": 93,
|
||||
@@ -485,12 +503,30 @@
|
||||
"functions": 98,
|
||||
"statements": 98
|
||||
},
|
||||
"workspace/system-backups.mjs": {
|
||||
"lines": 95,
|
||||
"branches": 88,
|
||||
"functions": 94,
|
||||
"statements": 95
|
||||
},
|
||||
"workspace/system-controls.mjs": {
|
||||
"lines": 98,
|
||||
"branches": 89,
|
||||
"functions": 89,
|
||||
"statements": 98
|
||||
},
|
||||
"workspace/system-integrations.mjs": {
|
||||
"lines": 96,
|
||||
"branches": 88,
|
||||
"functions": 91,
|
||||
"statements": 96
|
||||
},
|
||||
"workspace/system-maintenance.mjs": {
|
||||
"lines": 93,
|
||||
"branches": 80,
|
||||
"functions": 92,
|
||||
"statements": 93
|
||||
},
|
||||
"workspace/system-overview.mjs": {
|
||||
"lines": 96,
|
||||
"branches": 92,
|
||||
|
||||
+1
-1
@@ -6,7 +6,7 @@ if (host.href !== 'http://127.0.0.1:8196/') throw new Error('Execution tests req
|
||||
|
||||
export default defineConfig({
|
||||
testDir: '../../tests/comfyui',
|
||||
testMatch: ['execution-history.spec.mjs', 'execution-outputs.spec.mjs', 'library-location.spec.mjs'],
|
||||
testMatch: ['execution-history.spec.mjs', 'execution-outputs.spec.mjs', 'execution-text-link.spec.mjs', 'library-location.spec.mjs', 'image-attribution.spec.mjs'],
|
||||
outputDir: './test-results/executions/cases',
|
||||
timeout: 60000,
|
||||
workers: 1,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user